From 85522c064c6110a995862b7deb6792990594a659 Mon Sep 17 00:00:00 2001 From: Guofu Li Date: Sat, 20 Aug 2022 20:34:33 +0800 Subject: [PATCH] reorganize the directory structure --- .gitignore | 2 + dolphindb/.workspace | 16 + dolphindb/project/scripts/backup_table.dos | 5 + dolphindb/{ => project/scripts}/demo.dos | 0 dolphindb/{ => project/scripts}/demo2.dos | 0 dolphindb/project/scripts/min_kurtosis.dos | 7 + dolphindb/project/scripts/pit.dos | 8 + dolphindb/project/scripts/pit2.dos | 36 + dolphindb/project/scripts/pit3.dos | 17 + dolphindb/project/scripts/pit4.dos | 3 + dolphindb/project/scripts/pit6.dos | 14 + dolphindb/project/scripts/pit_at_date.dos | 17 + dolphindb/project/scripts/pit_col_at_date.dos | 18 + dolphindb/project/scripts/restore_table.dos | 1 + dolphindb/project/scripts/show_databases.dos | 9 + .../project/scripts/stockMarketReplay.dos | 0 dolphindb/project/scripts/tail_vol_ratio.dos | 7 + dolphindb/{ => project/scripts}/test.dos | 0 dolphindb/project/scripts/test1.dos | 14 + dolphindb/project/scripts/test2.dos | 3 + dolphindb/test1.dos | 10 - dolphindb/test2.dos | 5 - ddb.ipynb => ipynb/ddb.ipynb | 371 +++++- ipynb/ddb_pit.ipynb | 1009 +++++++++++++++++ dos.ipynb => ipynb/dos.ipynb | 0 mssql.ipynb => ipynb/mssql.ipynb | 0 DDBFactor.py => src/DDBFactor.py | 0 DDBLoader.py => src/DDBLoader.py | 0 ReadTickFromDB.py => src/ReadTickFromDB.py | 0 make_hft.py => src/make_hft.py | 0 script.py => src/script.py | 0 31 files changed, 1556 insertions(+), 16 deletions(-) create mode 100755 dolphindb/.workspace create mode 100755 dolphindb/project/scripts/backup_table.dos rename dolphindb/{ => project/scripts}/demo.dos (100%) rename dolphindb/{ => project/scripts}/demo2.dos (100%) create mode 100755 dolphindb/project/scripts/min_kurtosis.dos create mode 100755 dolphindb/project/scripts/pit.dos create mode 100755 dolphindb/project/scripts/pit2.dos create mode 100755 dolphindb/project/scripts/pit3.dos create mode 100755 dolphindb/project/scripts/pit4.dos create mode 100755 dolphindb/project/scripts/pit6.dos create mode 100755 dolphindb/project/scripts/pit_at_date.dos create mode 100755 dolphindb/project/scripts/pit_col_at_date.dos create mode 100755 dolphindb/project/scripts/restore_table.dos create mode 100755 dolphindb/project/scripts/show_databases.dos rename stockMarketReplay.dos => dolphindb/project/scripts/stockMarketReplay.dos (100%) create mode 100755 dolphindb/project/scripts/tail_vol_ratio.dos rename dolphindb/{ => project/scripts}/test.dos (100%) create mode 100755 dolphindb/project/scripts/test1.dos create mode 100755 dolphindb/project/scripts/test2.dos delete mode 100644 dolphindb/test1.dos delete mode 100644 dolphindb/test2.dos rename ddb.ipynb => ipynb/ddb.ipynb (61%) create mode 100644 ipynb/ddb_pit.ipynb rename dos.ipynb => ipynb/dos.ipynb (100%) rename mssql.ipynb => ipynb/mssql.ipynb (100%) rename DDBFactor.py => src/DDBFactor.py (100%) rename DDBLoader.py => src/DDBLoader.py (100%) rename ReadTickFromDB.py => src/ReadTickFromDB.py (100%) rename make_hft.py => src/make_hft.py (100%) rename script.py => src/script.py (100%) diff --git a/.gitignore b/.gitignore index 9e8fd5a..f29e05e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.csv *.swp *.swo +.DS_Store +*.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/dolphindb/.workspace b/dolphindb/.workspace new file mode 100755 index 0000000..f205732 --- /dev/null +++ b/dolphindb/.workspace @@ -0,0 +1,16 @@ +Project,test2,/Users/guofu/Workspaces/dolphindb/test2 +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/test1.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit2.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/tail_vol_ratio.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/test2.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit3.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit4.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit_report_period_at_date.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit6.dos +Open,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit_series_report_period_at_date.dos +Server,.167,192.168.1.167,8848,,admin,123456 +Server,.7,192.168.1.7,8848,,admin,123456 +Server,local8848,localhost,8848,,, +ActiveFile,/Users/guofu/Workspaces/dolphindb/test2/scripts/pit6.dos +ActiveServer,.167 diff --git a/dolphindb/project/scripts/backup_table.dos b/dolphindb/project/scripts/backup_table.dos new file mode 100755 index 0000000..d8f6abd --- /dev/null +++ b/dolphindb/project/scripts/backup_table.dos @@ -0,0 +1,5 @@ +login('admin', '123456') +// init backup +backup('/data/dolphindb/backup/', ,,false, true) \ No newline at end of file diff --git a/dolphindb/demo.dos b/dolphindb/project/scripts/demo.dos similarity index 100% rename from dolphindb/demo.dos rename to dolphindb/project/scripts/demo.dos diff --git a/dolphindb/demo2.dos b/dolphindb/project/scripts/demo2.dos similarity index 100% rename from dolphindb/demo2.dos rename to dolphindb/project/scripts/demo2.dos diff --git a/dolphindb/project/scripts/min_kurtosis.dos b/dolphindb/project/scripts/min_kurtosis.dos new file mode 100755 index 0000000..e44ea82 --- /dev/null +++ b/dolphindb/project/scripts/min_kurtosis.dos @@ -0,0 +1,7 @@ +tbl = loadTable("dfs://hft_stock_ts", 'KLinePartitioned') + +// calculate the minute return +// `eachPre` applies the binary function (the 1st arg) to this row and its previous row +// TODO: m_nDate filter condition should be replaced by groupby +ret = select code, m_nDate, m_nTime, eachPre(\, m_nClose) - 1.0 as ret from tbl where m_nDate=2015.01.05 context by code limit 10; +kurto = select code, m_nDate, sqrt(239) * sum(pow(ret, 3)) / pow(sum(pow(ret, 2)), 1.5) as kurtosis from ret where group by code; diff --git a/dolphindb/project/scripts/pit.dos b/dolphindb/project/scripts/pit.dos new file mode 100755 index 0000000..4fd98a6 --- /dev/null +++ b/dolphindb/project/scripts/pit.dos @@ -0,0 +1,8 @@ +is_common_ori = loadTable("dfs://pit_stock_ts", "cfs_common_adj") +divident = loadTable("dfs://pit_stock_ts", "divident") + +//schema(is_common_ori) + +//sql1 = select code, report_period, year(report_period) as year, quarterOfYear(report_period) + 1 as base_for_quarter_accum, appear_at_date from is_common_ori where partition(code, 0) + +select top 100 * from divident diff --git a/dolphindb/project/scripts/pit2.dos b/dolphindb/project/scripts/pit2.dos new file mode 100755 index 0000000..4a82241 --- /dev/null +++ b/dolphindb/project/scripts/pit2.dos @@ -0,0 +1,36 @@ +is_common_adj = loadTable("dfs://pit_stock_ts", "is_common_adj") +is_common_ori = loadTable("dfs://pit_stock_ts", "is_common_ori") + +// schema(is_common_ori) + +/** + * 单季处理 + */ + +// 每年第一个季度 +tbl_quarter1 = select code, report_period, year(report_period) as year, quarterOfYear(report_period) as current_quarter_accum, appear_at_date from is_common_ori where partition(code, 0), quarterOfYear(report_period) == 1 + +// 当季累计数据，作为下一季基准 +tbl_quarter_accum_base = select code, report_period, year(report_period) as year, quarterOfYear(report_period) + 1 as base_for_quarter_accum, appear_at_date from is_common_ori where partition(code, 0), quarterOfYear(report_period) < 4 + +// 从第二季开始，去匹配前一季累计基数 +tbl_quarter_accum = select code, report_period, year(report_period) as year, quarterOfYear(report_period) as current_quarter_accum, appear_at_date from is_common_ori where partition(code, 0), quarterOfYear(report_period) > 1 + +tbl_quarter_flux = select tbl_quarter_accum.* from ej(tbl_quarter_accum_base, tbl_quarter_accum, `code`year`base_for_quarter_accum, `code`year`current_quarter_accum) +tbl_quarter_flux = unionAll(tbl_quarter1, tbl_quarter_flux).sortBy!(`code`report_period) + +tbl_quarter_flux + +/** + * 同比计算 + */ +/* +sql1 = select code, report_period, year(report_period) - 1 as last_year, quarterOfYear(report_period) as quarter_of_year from bs_common_ori where code='000001.SZ' +//sql1 + +sql2 = select code, report_period, year(report_period) as this_year, quarterOfYear(report_period) as quarter_of_year from bs_common_ori where code='000001.SZ' + +sql = select sql1.*, sql2.* from ej(sql1, sql2, `code`last_year`quarter_of_year, `code`this_year`quarter_of_year) + +sql +*/ diff --git a/dolphindb/project/scripts/pit3.dos b/dolphindb/project/scripts/pit3.dos new file mode 100755 index 0000000..b3bab32 --- /dev/null +++ b/dolphindb/project/scripts/pit3.dos @@ -0,0 +1,17 @@ +is_common_adj = loadTable("dfs://pit_stock_ts", "is_common_adj") +is_common_ori = loadTable("dfs://pit_stock_ts", "is_common_ori") + +// select * from is_common_ori where code='000677.SZ' and report_period=2009.12.31 +// select * from is_common_ori where report_period=2009.12.31 + +m_nDate = take(2011.10.01, 3); +report_period = [2008.12.31, 2009.12.31, 2010.12.31] +//code = `000001.SZ`000677.SZ`600001.SH; + +tbl_query = table(report_period, m_nDate); +tbl_query_exp = select code, report_period, m_nDate from ej(is_common_ori, tbl_query, `report_period) where partition(code, 0); + +is_common_ori_part = select * from ej(is_common_ori, tbl_query, `report_period) where partition(code, 0); +is_common_ori_part + +select top 100 * from aj(tbl_query_exp, is_common_ori_part, `code`report_period`m_nDate, `code`report_period`appear_at_date); \ No newline at end of file diff --git a/dolphindb/project/scripts/pit4.dos b/dolphindb/project/scripts/pit4.dos new file mode 100755 index 0000000..88517ac --- /dev/null +++ b/dolphindb/project/scripts/pit4.dos @@ -0,0 +1,3 @@ +login('admin', '123456'); +dropFunctionView('pit_report_period_at_date'); + diff --git a/dolphindb/project/scripts/pit6.dos b/dolphindb/project/scripts/pit6.dos new file mode 100755 index 0000000..5076be2 --- /dev/null +++ b/dolphindb/project/scripts/pit6.dos @@ -0,0 +1,14 @@ +select * from pit_report_period_at_date("is_common_ori", 2022.07.12, [2019.12.31], 0) order by code +/* +source_table = loadTable("dfs://pit_stock_ts", "is_common_adj"); +m_nDate = [2021.12.12] +report_period = [2019.12.31] + +query_table = table(report_period, m_nDate) +query_table_exp = select * from cj(query_table, select code from source_table where partition(code, 0) group by code map) +// query_table_exp + +source_table_part = select source_table.* from ej(source_table, query_table_exp, `code`report_period) where partition(code, 0) + +select top 100 * from aj(query_table_exp, source_table_part, `code`report_period`m_nDate, `code`report_period`appear_at_date) order by code +*/ \ No newline at end of file diff --git a/dolphindb/project/scripts/pit_at_date.dos b/dolphindb/project/scripts/pit_at_date.dos new file mode 100755 index 0000000..1e142e7 --- /dev/null +++ b/dolphindb/project/scripts/pit_at_date.dos @@ -0,0 +1,17 @@ +//dropFunctionView('pit_report_period_at_date') + +def pit_at_date(table_name, date, report_period_list, code_partition_id) { + source_table = loadTable("dfs://pit_stock_ts", table_name); + + m_nDate = take(date, size(report_period_list)); + report_period = report_period_list; + + query_table = table(report_period, m_nDate); + query_table_exp = select * from cj(query_table, select code from source_table where partition(code, code_partition_id) group by code map); + + source_table_part = select source_table.* from ej(source_table, query_table_exp, `code`report_period) where partition(code, code_partition_id); + + return select * from aj(query_table_exp, source_table_part, `code`report_period`m_nDate, `code`report_period`appear_at_date) +} + +addFunctionView(pit_at_date) diff --git a/dolphindb/project/scripts/pit_col_at_date.dos b/dolphindb/project/scripts/pit_col_at_date.dos new file mode 100755 index 0000000..8e6f32a --- /dev/null +++ b/dolphindb/project/scripts/pit_col_at_date.dos @@ -0,0 +1,18 @@ +def pit_col_at_date(table_name, col_name, date, report_period_list, code_partition_id){ + source_table = loadTable("dfs://pit_stock_ts", table_name); + + m_nDate = take(date, size(report_period_list)); + report_period = report_period_list; + + query_table = table(report_period, m_nDate); + query_table_exp = select code, report_period, m_nDate from ej(source_table, query_table, `report_period) where partition(code, code_partition_id); + + col_list = ['code', 'report_period', 'appear_in_period', 'appear_at_date', col_name] + from_tbl = + where_conditions = [] + source_table_part = sql(select=col_list, from=from_tbl, where=where_conditions); + + return select * from aj(query_table_exp, source_table_part, `report_period`m_nDate, `report_period`appear_at_date) +} + +addFunctionView(pit_col_at_date) diff --git a/dolphindb/project/scripts/restore_table.dos b/dolphindb/project/scripts/restore_table.dos new file mode 100755 index 0000000..f54cd32 --- /dev/null +++ b/dolphindb/project/scripts/restore_table.dos @@ -0,0 +1 @@ +migrate('/data/dolphindb/backup/', "dfs://hft_stock_ts", "KLinePartitioned") diff --git a/dolphindb/project/scripts/show_databases.dos b/dolphindb/project/scripts/show_databases.dos new file mode 100755 index 0000000..f890874 --- /dev/null +++ b/dolphindb/project/scripts/show_databases.dos @@ -0,0 +1,9 @@ +login('admin', '123456') +// schema(loadTable("dfs://daily_stock_ts", "kurto_partitioned")) +//listTables("dfs://daily_stock_ts") +//dropTable(database("dfs://daily_stock_ts"), "kurto_partitioned") +//listTables("dfs://daily_stock_ts") +//select max(m_nDate), min(m_nDate) from loadTable("dfs://daily_stock_ts", "hft_daily_factor") +// select count(*) from loadTable("dfs://daily_stock_ts", "daily_kline") +// select top 100 * from loadTable("dfs://daily_stock_ts", "hft_daily_factor") where code='000002.SZ' +select top 100 * from loadTable("dfs://hft_stock_ts", "OrderPartitioned") where code='000002.SZ' \ No newline at end of file diff --git a/stockMarketReplay.dos b/dolphindb/project/scripts/stockMarketReplay.dos similarity index 100% rename from stockMarketReplay.dos rename to dolphindb/project/scripts/stockMarketReplay.dos diff --git a/dolphindb/project/scripts/tail_vol_ratio.dos b/dolphindb/project/scripts/tail_vol_ratio.dos new file mode 100755 index 0000000..0af8558 --- /dev/null +++ b/dolphindb/project/scripts/tail_vol_ratio.dos @@ -0,0 +1,7 @@ +tbl = loadTable("dfs://hft_stock_ts", 'KLinePartitioned') + +// TODO: m_nDate filter in where clause need to be replaced by group by +total_vol = select code, m_nDate, sum(m_nVolume) as total_vol from tbl where m_nDate=2015.01.05 group by code +tail_vol = select code, m_nDate, sum(m_nVolume) as tail_vol from tbl where m_nDate=2015.01.05 and m_nTime>=14:30:00 group by code + +select code, m_nDate, tail_vol/total_val as tail_vol_ratio from ej(total_val, tail_vol, `code`m_nDate) \ No newline at end of file diff --git a/dolphindb/test.dos b/dolphindb/project/scripts/test.dos similarity index 100% rename from dolphindb/test.dos rename to dolphindb/project/scripts/test.dos diff --git a/dolphindb/project/scripts/test1.dos b/dolphindb/project/scripts/test1.dos new file mode 100755 index 0000000..51c433c --- /dev/null +++ b/dolphindb/project/scripts/test1.dos @@ -0,0 +1,14 @@ +login('admin', '123456') +tbl = loadTable("dfs://hft_stock_ts", 'TickPartitioned') +//select * from tbl where code='000666.SZ', m_nDate <= 2013.01.06 +// delete from tbl where code = '000666.SZ' //, '000667.SZ', '000668.SZ', '000669.SZ'] +for (code in ['000666.SZ' , '000667.SZ', '000668.SZ', '000669.SZ']) { + del_date = 2013.01.01 + //end_date = 2013.01.31 + do { + //print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date); + print('Deleting ' + code + ' of ' + del_date); + delete from tbl where m_nDate=del_date, code=code; + del_date = temporalAdd(del_date, 1, "d"); + } while (del_date <= 2022.12.31) +} diff --git a/dolphindb/project/scripts/test2.dos b/dolphindb/project/scripts/test2.dos new file mode 100755 index 0000000..6b77081 --- /dev/null +++ b/dolphindb/project/scripts/test2.dos @@ -0,0 +1,3 @@ +login('admin', '123456') +//select * from getClusterChunksStatus() where chunkId = "0367bcb4-4919-d4ad-5d46-b32a1fa6e724" +temporalAdd(2013.12.31, 2, "y") <= 2014.12.31 \ No newline at end of file diff --git a/dolphindb/test1.dos b/dolphindb/test1.dos deleted file mode 100644 index 730dc31..0000000 --- a/dolphindb/test1.dos +++ /dev/null @@ -1,10 +0,0 @@ - -login('admin', '123456') - -tt = table(100:0, `code`m_nDate`m_nTime`m_nSide`m_nPrice`m_nOrderItems`m_nABItems`m_nABVolume, [SYMBOL, DATE, TIME, INT, FLOAT, INT, INT, INT[]]) - -share t as gt - -share tt as gtt - -dropStreamTable(gt) \ No newline at end of file diff --git a/dolphindb/test2.dos b/dolphindb/test2.dos deleted file mode 100644 index a92ff6c..0000000 --- a/dolphindb/test2.dos +++ /dev/null @@ -1,5 +0,0 @@ - -login("admin","123456") -pt=loadTable("dfs://valuedb","pt") -getTabletsMeta("/valuedb/%", `pt, true); -select top 100 * from pt \ No newline at end of file diff --git a/ddb.ipynb b/ipynb/ddb.ipynb similarity index 61% rename from ddb.ipynb rename to ipynb/ddb.ipynb index 95ce37c..028dbd2 100644 --- a/ddb.ipynb +++ b/ipynb/ddb.ipynb @@ -847,9 +847,378 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "b1bd7605-c347-44f2-a90f-bc949cde0ef0", "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codereport_periodappear_in_periodappear_at_dateTOT_OPER_REVOPER_REVINT_INCINSUR_PREM_UNEARNEDHANDLING_CHRG_COMM_INCNET_INC_OTHER_OPS...OTHER_INCOMEMEMOASSET_DISPOSAL_INCOMECONTINUED_NET_PROFITEND_NET_PROFITCREDIT_IMPAIRMENT_LOSSRD_EXPENSESTMNOTE_FINEXPFIN_EXP_INT_INCTOT_OPER_COST2
0000677.SZ2009-12-312009-12-312010-04-263.514495e+093.514495e+090.00.00.026392229.68...0.0nan0.00.00.00.00.00.00.03.474455e+09
1000677.SZ2009-12-312009-12-312011-12-103.514495e+093.514495e+090.00.00.026392229.68...0.0nan0.00.00.00.00.00.00.03.460335e+09
\n", + "

2 rows × 69 columns

\n", + "
" + ], + "text/plain": [ + " code report_period appear_in_period appear_at_date TOT_OPER_REV \\\n", + "0 000677.SZ 2009-12-31 2009-12-31 2010-04-26 3.514495e+09 \n", + "1 000677.SZ 2009-12-31 2009-12-31 2011-12-10 3.514495e+09 \n", + "\n", + " OPER_REV INT_INC INSUR_PREM_UNEARNED HANDLING_CHRG_COMM_INC \\\n", + "0 3.514495e+09 0.0 0.0 0.0 \n", + "1 3.514495e+09 0.0 0.0 0.0 \n", + "\n", + " NET_INC_OTHER_OPS ... OTHER_INCOME MEMO ASSET_DISPOSAL_INCOME \\\n", + "0 26392229.68 ... 0.0 nan 0.0 \n", + "1 26392229.68 ... 0.0 nan 0.0 \n", + "\n", + " CONTINUED_NET_PROFIT END_NET_PROFIT CREDIT_IMPAIRMENT_LOSS RD_EXPENSE \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "\n", + " STMNOTE_FINEXP FIN_EXP_INT_INC TOT_OPER_COST2 \n", + "0 0.0 0.0 3.474455e+09 \n", + "1 0.0 0.0 3.460335e+09 \n", + "\n", + "[2 rows x 69 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import dolphindb as ddb \n", + "\n", + "sess = ddb.session('192.168.1.167', 8848)\n", + "sess.login('admin', '123456')\n", + "\n", + "sess.run(\"\"\"\n", + " is_common_adj = loadTable(\"dfs://pit_stock_ts\", \"is_common_adj\")\n", + " is_common_ori = loadTable(\"dfs://pit_stock_ts\", \"is_common_ori\")\n", + " select * from is_common_ori where code='000677.SZ' and report_period=2009.12.31\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f0b3928d-c06f-4976-b568-091d98293a6e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codereport_periodappear_in_periodappear_at_dateTOT_OPER_REVOPER_REVINT_INCINSUR_PREM_UNEARNEDHANDLING_CHRG_COMM_INCNET_INC_OTHER_OPS...OTHER_INCOMEMEMOASSET_DISPOSAL_INCOMECONTINUED_NET_PROFITEND_NET_PROFITCREDIT_IMPAIRMENT_LOSSRD_EXPENSESTMNOTE_FINEXPFIN_EXP_INT_INCTOT_OPER_COST2
0000677.SZ2009-12-312010-12-312011-04-293.514495e+093.514495e+090.00.00.026392229.68...0.0nan0.00.00.00.00.00.00.03.474455e+09
\n", + "

1 rows × 69 columns

\n", + "
" + ], + "text/plain": [ + " code report_period appear_in_period appear_at_date TOT_OPER_REV \\\n", + "0 000677.SZ 2009-12-31 2010-12-31 2011-04-29 3.514495e+09 \n", + "\n", + " OPER_REV INT_INC INSUR_PREM_UNEARNED HANDLING_CHRG_COMM_INC \\\n", + "0 3.514495e+09 0.0 0.0 0.0 \n", + "\n", + " NET_INC_OTHER_OPS ... OTHER_INCOME MEMO ASSET_DISPOSAL_INCOME \\\n", + "0 26392229.68 ... 0.0 nan 0.0 \n", + "\n", + " CONTINUED_NET_PROFIT END_NET_PROFIT CREDIT_IMPAIRMENT_LOSS RD_EXPENSE \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "\n", + " STMNOTE_FINEXP FIN_EXP_INT_INC TOT_OPER_COST2 \n", + "0 0.0 0.0 3.474455e+09 \n", + "\n", + "[1 rows x 69 columns]" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.run(\"select * from is_common_adj where code='000677.SZ' and report_period=2009.12.31\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d68ea326-82c3-4a7c-97cf-c04dd8aee56b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codem_nDatem_nTimem_nPricem_iVolumem_iTuroverm_nMatchItemsm_chTradeFlagm_chBSFlagm_iAccVolume...m_nOpenm_nPreClosem_nAskPricem_nAskVolumem_nBidPricem_nBidVolumem_nAskAvPricem_nBidAvPricem_iTotalAskVolumem_iTotalBidVolume
\n", + "

0 rows × 23 columns

\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [code, m_nDate, m_nTime, m_nPrice, m_iVolume, m_iTurover, m_nMatchItems, m_chTradeFlag, m_chBSFlag, m_iAccVolume, m_iAccTurover, m_nHigh, m_nLow, m_nOpen, m_nPreClose, m_nAskPrice, m_nAskVolume, m_nBidPrice, m_nBidVolume, m_nAskAvPrice, m_nBidAvPrice, m_iTotalAskVolume, m_iTotalBidVolume]\n", + "Index: []\n", + "\n", + "[0 rows x 23 columns]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess = ddb.session('192.168.1.7', 8848)\n", + "sess.login('admin', '123456')\n", + "sess.run(\"\"\"\n", + "tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\n", + "select * from tbl where code='000666.SZ', m_nDate <= 2013.02.26;\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "0e6e949b-ff25-40eb-a70a-72a3a6639aec", + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": " in run: Failed to read response header from the socket with IO error type 13", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Input \u001b[0;32mIn [31]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m code \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124mtbl = loadTable(\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdfs://hft_stock_ts\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTickPartitioned\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\n\u001b[1;32m 3\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124m}\u001b[39m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[0;32m---> 15\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcode\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:161\u001b[0m, in \u001b[0;36msession.run\u001b[0;34m(self, script, *args, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfetchSize\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m BlockReader(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcpp\u001b[38;5;241m.\u001b[39mrunBlock(script, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mRuntimeError\u001b[0m: in run: Failed to read response header from the socket with IO error type 13" + ] + } + ], + "source": [ + "code = \"\"\"\n", + "tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned')\n", + "\n", + "for (code in ['000666.SZ' , '000667.SZ', '000668.SZ', '000669.SZ']) {\n", + "\tdel_date = 2013.01.01\n", + "\t//end_date = 2013.01.31\n", + "\tdo {\n", + "\t\t//print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date);\n", + "\t\tprint('Deleting ' + code + ' of ' + del_date);\n", + "\t\tdelete from tbl where m_nDate=del_date, code=code;\n", + "\t\tdel_date = temporalAdd(del_date, 1, \"d\");\n", + "\t} while (del_date <= 2022.12.31)\n", + "}\n", + "\"\"\"\n", + "sess.run(code)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea0501e7-d416-45ce-add5-e443c55f158c", + "metadata": {}, "outputs": [], "source": [] } diff --git a/ipynb/ddb_pit.ipynb b/ipynb/ddb_pit.ipynb new file mode 100644 index 0000000..74a3632 --- /dev/null +++ b/ipynb/ddb_pit.ipynb @@ -0,0 +1,1009 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "139fd1cb-aedf-4186-8408-4d630ba69599", + "metadata": {}, + "outputs": [], + "source": [ + "import dolphindb as ddb\n", + "\n", + "sess = ddb.session(\"192.168.1.167\", 8848)\n", + "sess.login('admin', '123456')" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "86d6ce04-b5ae-410b-af8d-9d9b5d87a959", + "metadata": {}, + "outputs": [], + "source": [ + "sess.run(\"\"\"\n", + " dropFunctionView('pit_at_date')\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "5482fa63-23e7-48e7-9636-5628faef7b73", + "metadata": {}, + "outputs": [], + "source": [ + "sess.run(\"\"\"\n", + "def pit_at_date(table_name, date, report_period_list, code_partition_id) {\n", + "\tsource_table = loadTable(\"dfs://pit_stock_ts\", table_name);\n", + "\t\n", + "\tm_nDate = take(date, size(report_period_list));\n", + "\treport_period = report_period_list;\n", + "\t\n", + "\tquery_table = table(report_period, m_nDate);\n", + "\tquery_table_exp = select * from cj(query_table, select code from source_table where partition(code, code_partition_id) group by code map);\n", + "\t\n", + "\tsource_table_part = select source_table.* from ej(source_table, query_table_exp, `code`report_period) where partition(code, code_partition_id);\n", + "\t\n", + "\treturn select source_table_part.* from aj(query_table_exp, source_table_part, `code`report_period`m_nDate, `code`report_period`appear_at_date) where not isNull(source_table_part.code)\n", + "}\n", + "\n", + "addFunctionView(pit_at_date)\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "4e0eaa4d-80c9-4925-8496-a7d133fc0fda", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
appear_in_periodTOT_OPER_REVOPER_REVINT_INCINSUR_PREM_UNEARNEDHANDLING_CHRG_COMM_INCNET_INC_OTHER_OPSPLUS_NET_INC_OTHER_BUSPLUS_NET_GAIN_CHG_FVPLUS_NET_INVEST_INC...OTHER_INCOMEMEMOASSET_DISPOSAL_INCOMECONTINUED_NET_PROFITEND_NET_PROFITCREDIT_IMPAIRMENT_LOSSRD_EXPENSESTMNOTE_FINEXPFIN_EXP_INT_INCTOT_OPER_COST2
codereport_periodappear_at_date
000400.SZ2018-12-312019-03-292018-12-318.216559e+098.216559e+090.00.00.073932.940.00.073932.94...72773000.86nan0.002.737097e+080.00.03.350267e+0845075829.770.07.967406e+09
000558.SZ2018-12-312019-04-262018-12-317.024741e+087.024741e+080.00.00.023984753.520.00.023984753.52...10174473.66nan-12902036.23-5.424955e+070.00.00.000000e+0039309311.590.06.444387e+08
000677.SZ2018-12-312019-03-292018-12-317.068198e+087.068198e+080.00.00.0220497.760.00.0220497.76...1157100.00nan-1852690.976.403690e+060.00.03.514984e+055234360.370.06.821367e+08
000731.SZ2018-12-312019-04-162018-12-312.639920e+092.639920e+090.00.00.04467512.660.00.04467512.66...3477372.43nan1992627.872.228619e+080.00.01.653546e+0646581902.650.02.367904e+09
000752.SZ2018-12-312019-05-162018-12-313.231171e+083.231171e+080.00.00.0-43852162.570.00.0-43852162.57...136549.44nan0.00-3.830619e+080.00.00.000000e+000.000.03.135966e+08
........................................................................
688039.SH2018-12-312019-11-222018-12-312.035519e+082.035519e+080.00.00.0156599.670.00.0156599.67...15702563.93nan0.006.389806e+070.00.03.660990e+070.000.01.511902e+08
688368.SH2018-12-312019-09-172018-12-317.665912e+087.665912e+080.00.00.0972435.310.00.0972435.31...6568742.64nan0.000.000000e+000.00.06.081719e+070.000.06.890894e+08
688396.SH2018-12-312020-01-312018-12-316.270797e+096.270797e+090.00.00.010592096.660.00.010592096.66...91086261.29nan9434573.460.000000e+000.00.04.497610e+080.000.05.796301e+09
831010.NE2018-12-312020-07-012018-12-312.936331e+082.936331e+080.00.00.0262112.160.00.0262112.16...380000.00nan-258845.784.129632e+070.00.05.236440e+05254556.290.02.437095e+08
833874.NE2018-12-312020-06-292018-12-311.974978e+081.974978e+080.00.00.05841180.610.00.05841180.61...882100.00nan9554.969.051358e+070.00.01.092805e+070.000.01.021658e+08
\n", + "

91 rows × 66 columns

\n", + "
" + ], + "text/plain": [ + " appear_in_period TOT_OPER_REV \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 2018-12-31 8.216559e+09 \n", + "000558.SZ 2018-12-31 2019-04-26 2018-12-31 7.024741e+08 \n", + "000677.SZ 2018-12-31 2019-03-29 2018-12-31 7.068198e+08 \n", + "000731.SZ 2018-12-31 2019-04-16 2018-12-31 2.639920e+09 \n", + "000752.SZ 2018-12-31 2019-05-16 2018-12-31 3.231171e+08 \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 2018-12-31 2.035519e+08 \n", + "688368.SH 2018-12-31 2019-09-17 2018-12-31 7.665912e+08 \n", + "688396.SH 2018-12-31 2020-01-31 2018-12-31 6.270797e+09 \n", + "831010.NE 2018-12-31 2020-07-01 2018-12-31 2.936331e+08 \n", + "833874.NE 2018-12-31 2020-06-29 2018-12-31 1.974978e+08 \n", + "\n", + " OPER_REV INT_INC \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 8.216559e+09 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 7.024741e+08 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 7.068198e+08 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 2.639920e+09 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 3.231171e+08 0.0 \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 2.035519e+08 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 7.665912e+08 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 6.270797e+09 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 2.936331e+08 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 1.974978e+08 0.0 \n", + "\n", + " INSUR_PREM_UNEARNED \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 0.0 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 0.0 \n", + "\n", + " HANDLING_CHRG_COMM_INC \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 0.0 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 0.0 \n", + "\n", + " NET_INC_OTHER_OPS \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 73932.94 \n", + "000558.SZ 2018-12-31 2019-04-26 23984753.52 \n", + "000677.SZ 2018-12-31 2019-03-29 220497.76 \n", + "000731.SZ 2018-12-31 2019-04-16 4467512.66 \n", + "000752.SZ 2018-12-31 2019-05-16 -43852162.57 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 156599.67 \n", + "688368.SH 2018-12-31 2019-09-17 972435.31 \n", + "688396.SH 2018-12-31 2020-01-31 10592096.66 \n", + "831010.NE 2018-12-31 2020-07-01 262112.16 \n", + "833874.NE 2018-12-31 2020-06-29 5841180.61 \n", + "\n", + " PLUS_NET_INC_OTHER_BUS \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 0.0 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 0.0 \n", + "\n", + " PLUS_NET_GAIN_CHG_FV \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 0.0 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 0.0 \n", + "\n", + " PLUS_NET_INVEST_INC ... \\\n", + "code report_period appear_at_date ... \n", + "000400.SZ 2018-12-31 2019-03-29 73932.94 ... \n", + "000558.SZ 2018-12-31 2019-04-26 23984753.52 ... \n", + "000677.SZ 2018-12-31 2019-03-29 220497.76 ... \n", + "000731.SZ 2018-12-31 2019-04-16 4467512.66 ... \n", + "000752.SZ 2018-12-31 2019-05-16 -43852162.57 ... \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 156599.67 ... \n", + "688368.SH 2018-12-31 2019-09-17 972435.31 ... \n", + "688396.SH 2018-12-31 2020-01-31 10592096.66 ... \n", + "831010.NE 2018-12-31 2020-07-01 262112.16 ... \n", + "833874.NE 2018-12-31 2020-06-29 5841180.61 ... \n", + "\n", + " OTHER_INCOME MEMO \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 72773000.86 nan \n", + "000558.SZ 2018-12-31 2019-04-26 10174473.66 nan \n", + "000677.SZ 2018-12-31 2019-03-29 1157100.00 nan \n", + "000731.SZ 2018-12-31 2019-04-16 3477372.43 nan \n", + "000752.SZ 2018-12-31 2019-05-16 136549.44 nan \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 15702563.93 nan \n", + "688368.SH 2018-12-31 2019-09-17 6568742.64 nan \n", + "688396.SH 2018-12-31 2020-01-31 91086261.29 nan \n", + "831010.NE 2018-12-31 2020-07-01 380000.00 nan \n", + "833874.NE 2018-12-31 2020-06-29 882100.00 nan \n", + "\n", + " ASSET_DISPOSAL_INCOME \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.00 \n", + "000558.SZ 2018-12-31 2019-04-26 -12902036.23 \n", + "000677.SZ 2018-12-31 2019-03-29 -1852690.97 \n", + "000731.SZ 2018-12-31 2019-04-16 1992627.87 \n", + "000752.SZ 2018-12-31 2019-05-16 0.00 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.00 \n", + "688368.SH 2018-12-31 2019-09-17 0.00 \n", + "688396.SH 2018-12-31 2020-01-31 9434573.46 \n", + "831010.NE 2018-12-31 2020-07-01 -258845.78 \n", + "833874.NE 2018-12-31 2020-06-29 9554.96 \n", + "\n", + " CONTINUED_NET_PROFIT END_NET_PROFIT \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 2.737097e+08 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 -5.424955e+07 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 6.403690e+06 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 2.228619e+08 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 -3.830619e+08 0.0 \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 6.389806e+07 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.000000e+00 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.000000e+00 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 4.129632e+07 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 9.051358e+07 0.0 \n", + "\n", + " CREDIT_IMPAIRMENT_LOSS RD_EXPENSE \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 0.0 3.350267e+08 \n", + "000558.SZ 2018-12-31 2019-04-26 0.0 0.000000e+00 \n", + "000677.SZ 2018-12-31 2019-03-29 0.0 3.514984e+05 \n", + "000731.SZ 2018-12-31 2019-04-16 0.0 1.653546e+06 \n", + "000752.SZ 2018-12-31 2019-05-16 0.0 0.000000e+00 \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.0 3.660990e+07 \n", + "688368.SH 2018-12-31 2019-09-17 0.0 6.081719e+07 \n", + "688396.SH 2018-12-31 2020-01-31 0.0 4.497610e+08 \n", + "831010.NE 2018-12-31 2020-07-01 0.0 5.236440e+05 \n", + "833874.NE 2018-12-31 2020-06-29 0.0 1.092805e+07 \n", + "\n", + " STMNOTE_FINEXP FIN_EXP_INT_INC \\\n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 45075829.77 0.0 \n", + "000558.SZ 2018-12-31 2019-04-26 39309311.59 0.0 \n", + "000677.SZ 2018-12-31 2019-03-29 5234360.37 0.0 \n", + "000731.SZ 2018-12-31 2019-04-16 46581902.65 0.0 \n", + "000752.SZ 2018-12-31 2019-05-16 0.00 0.0 \n", + "... ... ... \n", + "688039.SH 2018-12-31 2019-11-22 0.00 0.0 \n", + "688368.SH 2018-12-31 2019-09-17 0.00 0.0 \n", + "688396.SH 2018-12-31 2020-01-31 0.00 0.0 \n", + "831010.NE 2018-12-31 2020-07-01 254556.29 0.0 \n", + "833874.NE 2018-12-31 2020-06-29 0.00 0.0 \n", + "\n", + " TOT_OPER_COST2 \n", + "code report_period appear_at_date \n", + "000400.SZ 2018-12-31 2019-03-29 7.967406e+09 \n", + "000558.SZ 2018-12-31 2019-04-26 6.444387e+08 \n", + "000677.SZ 2018-12-31 2019-03-29 6.821367e+08 \n", + "000731.SZ 2018-12-31 2019-04-16 2.367904e+09 \n", + "000752.SZ 2018-12-31 2019-05-16 3.135966e+08 \n", + "... ... \n", + "688039.SH 2018-12-31 2019-11-22 1.511902e+08 \n", + "688368.SH 2018-12-31 2019-09-17 6.890894e+08 \n", + "688396.SH 2018-12-31 2020-01-31 5.796301e+09 \n", + "831010.NE 2018-12-31 2020-07-01 2.437095e+08 \n", + "833874.NE 2018-12-31 2020-06-29 1.021658e+08 \n", + "\n", + "[91 rows x 66 columns]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#sess.run(\"\"\"\n", + "# is_common_ori = loadTable(\"dfs://pit_stock_ts\", \"is_common_ori\")\n", + "#\"\"\")\n", + "\n", + "# pit_report_period_at_date(table_name, date, report_period_list, code_partition_id)\n", + "sess.run(\"\"\"\n", + " select * from pit_at_date(\"is_common_ori\", 2021.03.14, [2018.12.31], 0) order by code\n", + "\"\"\").set_index(['code', 'report_period', 'appear_at_date'])" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "4cd4cd6e-f34e-43a8-98de-c468a54d8081", + "metadata": {}, + "outputs": [], + "source": [ + "sess.run(\"\"\"\n", + " dropFunctionView('pit_col_at_date');\n", + "\"\"\")\n", + "\n", + "sess.run(\"\"\"\n", + "def pit_col_at_date(table_name, col_name, date, report_period_list, code_partition_id){\n", + "\tsource_table = loadTable(\"dfs://pit_stock_ts\", table_name);\n", + "\t\n", + "\tm_nDate = take(date, size(report_period_list));\n", + "\treport_period = report_period_list;\n", + "\t\n", + "\tquery_table = table(report_period, m_nDate);\n", + "\tquery_table_exp = select * from cj(query_table, select code from source_table where partition(code, code_partition_id) group by code map);\n", + "\t\n", + "\tcol_list = sqlCol(['code', 'report_period', 'appear_at_date', col_name]);\n", + " from_tbl = ;\n", + " where_conditions = [];\n", + " source_table_part = sql(select=col_list, from=from_tbl, where=where_conditions).eval();\n", + "\t\n", + "\treturn select source_table_part.* from aj(query_table_exp, source_table_part, `code`report_period`m_nDate, `code`report_period`appear_at_date) where not isNull(source_table_part.code)\n", + "}\n", + "\n", + "addFunctionView(pit_col_at_date);\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "943b760a-ab39-4291-8a93-81b3e38a70b7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TOT_OPER_REV
codereport_periodappear_at_date
000400.SZ2019-09-302020-10-295.658881e+09
000558.SZ2019-09-302020-10-298.322244e+07
000677.SZ2019-09-302020-10-216.020088e+08
000731.SZ2019-09-302020-10-272.311672e+09
000752.SZ2019-09-302020-10-292.330136e+08
............
688039.SH2019-09-302020-10-281.185143e+08
688368.SH2019-09-302020-10-286.139719e+08
688396.SH2019-09-302020-10-204.131915e+09
831010.NE2019-09-302020-10-292.276040e+08
833874.NE2019-09-302020-10-261.303494e+08
\n", + "

91 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " TOT_OPER_REV\n", + "code report_period appear_at_date \n", + "000400.SZ 2019-09-30 2020-10-29 5.658881e+09\n", + "000558.SZ 2019-09-30 2020-10-29 8.322244e+07\n", + "000677.SZ 2019-09-30 2020-10-21 6.020088e+08\n", + "000731.SZ 2019-09-30 2020-10-27 2.311672e+09\n", + "000752.SZ 2019-09-30 2020-10-29 2.330136e+08\n", + "... ...\n", + "688039.SH 2019-09-30 2020-10-28 1.185143e+08\n", + "688368.SH 2019-09-30 2020-10-28 6.139719e+08\n", + "688396.SH 2019-09-30 2020-10-20 4.131915e+09\n", + "831010.NE 2019-09-30 2020-10-29 2.276040e+08\n", + "833874.NE 2019-09-30 2020-10-26 1.303494e+08\n", + "\n", + "[91 rows x 1 columns]" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.run(\"\"\"\n", + " select * from pit_col_at_date(\"is_common_adj\", \"TOT_OPER_REV\", 2021.03.14, [2019.09.30], 0) order by code\n", + "\"\"\").set_index(['code', 'report_period', 'appear_at_date'])" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "2fea0f1e-5105-4d28-9c36-c5542a5389ba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codereport_periodappear_in_periodappear_at_dateTOT_OPER_REV
0000400.SZ2019-12-312019-12-312020-04-101.015608e+10
1000558.SZ2019-12-312019-12-312020-04-271.378479e+08
2000558.SZ2019-12-312019-12-312020-08-251.378479e+08
3000677.SZ2019-12-312019-12-312020-04-257.906742e+08
4000731.SZ2019-12-312019-12-312020-04-242.929131e+09
..................
109688396.SH2019-12-312019-12-312020-04-235.742784e+09
110831010.NE2019-12-312019-12-312020-04-133.552313e+08
111831010.NE2019-12-312019-12-312020-07-013.552313e+08
112833874.NE2019-12-312019-12-312020-03-181.785113e+08
113833874.NE2019-12-312019-12-312020-06-291.785113e+08
\n", + "

114 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " code report_period appear_in_period appear_at_date TOT_OPER_REV\n", + "0 000400.SZ 2019-12-31 2019-12-31 2020-04-10 1.015608e+10\n", + "1 000558.SZ 2019-12-31 2019-12-31 2020-04-27 1.378479e+08\n", + "2 000558.SZ 2019-12-31 2019-12-31 2020-08-25 1.378479e+08\n", + "3 000677.SZ 2019-12-31 2019-12-31 2020-04-25 7.906742e+08\n", + "4 000731.SZ 2019-12-31 2019-12-31 2020-04-24 2.929131e+09\n", + ".. ... ... ... ... ...\n", + "109 688396.SH 2019-12-31 2019-12-31 2020-04-23 5.742784e+09\n", + "110 831010.NE 2019-12-31 2019-12-31 2020-04-13 3.552313e+08\n", + "111 831010.NE 2019-12-31 2019-12-31 2020-07-01 3.552313e+08\n", + "112 833874.NE 2019-12-31 2019-12-31 2020-03-18 1.785113e+08\n", + "113 833874.NE 2019-12-31 2019-12-31 2020-06-29 1.785113e+08\n", + "\n", + "[114 rows x 5 columns]" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.run(\"\"\"\n", + " source_table = loadTable(\"dfs://pit_stock_ts\", \"is_common_ori\");\n", + " m_nDate = [2022.03.04]\n", + " report_period = [2019.12.31]\n", + "\tquery_table = table(report_period, m_nDate);\n", + "\tquery_table_exp = select * from cj(query_table, select code from source_table where partition(code, 0) group by code map);\n", + "\t\n", + " \n", + " col_list = sqlCol(['code', 'report_period', 'appear_in_period', 'appear_at_date', 'TOT_OPER_REV']);\n", + " from_tbl = ;\n", + " where_conditions = [];\n", + " source_table_part = sql(select=col_list, from=from_tbl, where=where_conditions).eval();\n", + "\tsource_table_part\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c246940-1ad6-414f-b461-2d8ca7cd87f1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dos.ipynb b/ipynb/dos.ipynb similarity index 100% rename from dos.ipynb rename to ipynb/dos.ipynb diff --git a/mssql.ipynb b/ipynb/mssql.ipynb similarity index 100% rename from mssql.ipynb rename to ipynb/mssql.ipynb diff --git a/DDBFactor.py b/src/DDBFactor.py similarity index 100% rename from DDBFactor.py rename to src/DDBFactor.py diff --git a/DDBLoader.py b/src/DDBLoader.py similarity index 100% rename from DDBLoader.py rename to src/DDBLoader.py diff --git a/ReadTickFromDB.py b/src/ReadTickFromDB.py similarity index 100% rename from ReadTickFromDB.py rename to src/ReadTickFromDB.py diff --git a/make_hft.py b/src/make_hft.py similarity index 100% rename from make_hft.py rename to src/make_hft.py diff --git a/script.py b/src/script.py similarity index 100% rename from script.py rename to src/script.py