From b94beef451079a4dbfe897a94e05c7e0b32db7f9 Mon Sep 17 00:00:00 2001
From: yzlocal <local@local>
Date: Wed, 16 Nov 2022 09:54:35 +0800
Subject: [PATCH] update data loader

---
 src/data_loader.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/data_loader.py b/src/data_loader.py
index af9495d..b0495d8 100644
--- a/src/data_loader.py
+++ b/src/data_loader.py
@@ -29,7 +29,7 @@ def run_add_1day_code_init_minKline(date, code_list):
             tsl.get_mkt_min_k(date, date, code_list_filtered))
     if not df.empty:
         logger.info(
-            f'Getting a df of {df.shape}: {code_list[0][:-4]} on {date}')
+            f'Getting a df of {df.shape}: {"+".join(code_list)} on {date}')
         ddb.append_hft_table(ddb.ddf_hft_mink_tbname, df)
 
 
@@ -121,7 +121,7 @@ def run():
     too slow. depracated.
     """
     all_code_dict_by_init = {}
-    for c in code_list_pickel:
+    for c in code_list_pickel_from_file:
         init = c[:-4]
         if init in all_code_dict_by_init:
             all_code_dict_by_init[init].append(c)
@@ -146,7 +146,7 @@ def run():
 def run_pool_dates_by_code_init_n_group(typ='mink', code_gp_amt=10, date_gp_amt=10, start_date='20220101', end_date='20221031', if_check=1, code_dict_by='init'):
     logger.info("Running run_pool_dates_by_group")
     code_list_pickel=code_list_pickel_from_file
-    code_list_pickel=sorted(list(set()))
+    code_list_pickel=sorted(list(set(code_list_pickel)))
     all_code_dict_by_init = {}
     for c in code_list_pickel:
         if c[-4:] < start_date[2:6]:
@@ -194,8 +194,6 @@ def run_pool_dates_by_code_init_n_group(typ='mink', code_gp_amt=10, date_gp_amt=
             logger.info(code_list)
             if typ == 'mink':
                 # logger.info('Running mink')
-                logger.info(date_list)
-                logger.info(code_list)
                 run_pool_add_by_datelist_codeinit(
                     'mink', date_list, code_list, if_check)
 
@@ -226,12 +224,12 @@ if __name__ == '__main__':
         split_code_into_howmany_groups_no = int(split_code_into_howmany_groups_no)
         split_date_into_howmany_groups=int(split_date_into_howmany_groups)
     else:
-        typ = 'mink'
-        st_d = '20210104'
+        typ = 'tick'
+        st_d = '20211201'
         en_d = '20211231'
-        if_check = 0
-        split_code_into_howmany_groups_no = 5
-        split_date_into_howmany_groups = 20
+        if_check = 1
+        split_code_into_howmany_groups_no = 20
+        split_date_into_howmany_groups = 5
 
 
     logger.add(ROOT_DIR+"/logs/{time:YYYYMMDD-HHmmss}"+f"_{running_which_env}_{typ}_{st_d}_{en_d}_{if_check}_{split_code_into_howmany_groups_no}_{split_date_into_howmany_groups}.log",