From aed5e94395544910e65922909db99f22c1ef1728 Mon Sep 17 00:00:00 2001 From: yzlocal Date: Thu, 10 Nov 2022 16:48:20 +0800 Subject: [PATCH] somehow is running eeeee --- dolphindb-test-clean.ipynb | 1467 ++++++++++++++++++++++++++++++++---- src/DDBfm.py | 82 +- src/TSLfm.py | 11 +- src/data_loader.py | 167 +++- test/test.dos | 9 +- 5 files changed, 1538 insertions(+), 198 deletions(-) diff --git a/dolphindb-test-clean.ipynb b/dolphindb-test-clean.ipynb index 909a8df..72474f9 100644 --- a/dolphindb-test-clean.ipynb +++ b/dolphindb-test-clean.ipynb @@ -21,6 +21,26 @@ "sess.login('admin','123456')\n" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.clearAllCache()" + ] + }, { "cell_type": "code", "execution_count": 27, @@ -826,9 +846,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-11-09 14:48:51.524 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", - "2022-11-09 14:48:51.708 | DEBUG | src.TSLfm:get_mkt_min_k:209 - \n", - " SetSysParam(pn_cycle(), cy_1m());\n", + "2022-11-09 15:44:17.606 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", + "2022-11-09 15:44:17.762 | DEBUG | src.TSLfm:get_trade_tick:136 - \n", " stock_list := Array('CF2211', 'CF2301', 'CF2303', 'CF2305', 'CF2307', 'CF2309');\n", " r := select \n", " ['StockID'] as 'code',\n", @@ -881,70 +900,69 @@ " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", " ['lb'] as 'm_nMItemsVolRate'//量比\n", "\n", - "\n", - " from markettable\n", - " datekey 20221001T to 20221101T+0.999 \n", + " from tradetable\n", + " datekey 20221101T to 20221101T+0.999 \n", " of stock_list\n", " end;\n", " \n", " return r; \n", " \n", - "2022-11-09 14:49:05.395 | INFO | src.TSLfm:process_result_data_type:215 - Processing new df of shape (35910, 37), which looks like\n", + "2022-11-09 15:44:37.198 | INFO | src.TSLfm:process_result_data_type:215 - Processing new df of shape (112175, 37), which looks like\n", " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", - "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "0 CF2303 2022-11-01 09:00:00 12365.0 12365.0 12365.0 12365.0 12365.0 \n", + "1 CF2303 2022-11-01 09:00:01 12375.0 12375.0 12375.0 12375.0 12375.0 \n", + "2 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "3 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "4 CF2303 2022-11-01 09:00:03 12350.0 12350.0 12350.0 12350.0 12350.0 \n", "\n", " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", - "0 13445.0 13445.0 0.0 0.0 0.0 \n", - "1 13445.0 13445.0 0.0 0.0 0.0 \n", - "2 13445.0 13445.0 0.0 0.0 0.0 \n", - "3 13445.0 13445.0 0.0 0.0 0.0 \n", - "4 13445.0 13445.0 0.0 0.0 0.0 \n", + "0 12370.0 12250.0 13.0 20959.0 -7.0 \n", + "1 12375.0 12250.0 6.0 20965.0 2.0 \n", + "2 12375.0 12250.0 2.0 20967.0 -2.0 \n", + "3 12375.0 12250.0 1.0 20968.0 -1.0 \n", + "4 12375.0 12250.0 21.0 20989.0 0.0 \n", "\n", " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", - "0 96132 0.0 0.0 13445.0 13445.0 \n", - "1 96132 0.0 0.0 13445.0 13445.0 \n", - "2 96132 0.0 0.0 13445.0 13445.0 \n", - "3 96132 0.0 0.0 13445.0 13445.0 \n", - "4 96132 0.0 0.0 13445.0 13445.0 \n", + "0 163532 159965.0 257900495.0 12270.0 12390.0 \n", + "1 163534 73830.0 257974325.0 12365.0 12390.0 \n", + "2 163532 24610.0 257998935.0 12375.0 12390.0 \n", + "3 163531 12305.0 258011240.0 12370.0 12390.0 \n", + "4 163531 258405.0 258269645.0 12370.0 12390.0 \n", "\n", " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", - "0 0.0 0 0.0 0 0 \n", - "1 0.0 0 0.0 0 0 \n", - "2 0.0 0 0.0 0 0 \n", - "3 0.0 0 0.0 0 0 \n", - "4 0.0 0 0.0 0 0 \n", + "0 12365.0 2 12370.0 1 1 \n", + "1 12360.0 2 12370.0 111 1 \n", + "2 12345.0 3 12355.0 98 1 \n", + "3 12340.0 1 12350.0 97 1 \n", + "4 12340.0 10 12350.0 19 1 \n", "\n", " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", - "0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 \n", + "0 13.0 10529.0 159965.0 \n", + "1 6.0 10535.0 73830.0 \n", + "2 2.0 10537.0 24610.0 \n", + "3 1.0 10538.0 12305.0 \n", + "4 21.0 10559.0 258405.0 \n", "\n", " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", - "0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 \n", + "0 129574875.0 0.0 10430.0 \n", + "1 129648705.0 0.0 10430.0 \n", + "2 129673315.0 0.0 10430.0 \n", + "3 129685620.0 0.0 10430.0 \n", + "4 129944025.0 0.0 10430.0 \n", "\n", " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", + "0 0.0 128325620.0 2.0 405687.0 \n", + "1 0.0 128325620.0 2.0 405689.0 \n", + "2 0.0 128325620.0 3.0 405692.0 \n", + "3 0.0 128325620.0 1.0 405693.0 \n", + "4 0.0 128325620.0 10.0 405703.0 \n", "\n", " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", + "0 1.0 352340.0 2.000000 1.151408 \n", + "1 111.0 352451.0 0.018018 1.151051 \n", + "2 98.0 352549.0 0.030612 1.150739 \n", + "3 97.0 352646.0 0.010309 1.150426 \n", + "4 19.0 352665.0 0.526316 1.150392 \n", "\n", " m_nMItemsVolRate \n", "0 0.0 \n", @@ -952,139 +970,139 @@ "2 0.0 \n", "3 0.0 \n", "4 0.0 \n", - "2022-11-09 14:49:05.493 | INFO | src.TSLfm:process_result_data_type:229 - Processing done, new df looks like\n", + "2022-11-09 15:44:37.288 | INFO | src.TSLfm:process_result_data_type:229 - Processing done, new df looks like\n", " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", - "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "0 CF2303 2022-11-01 09:00:00 12365.0 12365.0 12365.0 12365.0 12365.0 \n", + "1 CF2303 2022-11-01 09:00:01 12375.0 12375.0 12375.0 12375.0 12375.0 \n", + "2 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "3 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "4 CF2303 2022-11-01 09:00:03 12350.0 12350.0 12350.0 12350.0 12350.0 \n", "\n", " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", - "0 13445.0 13445.0 0 0 0 \n", - "1 13445.0 13445.0 0 0 0 \n", - "2 13445.0 13445.0 0 0 0 \n", - "3 13445.0 13445.0 0 0 0 \n", - "4 13445.0 13445.0 0 0 0 \n", + "0 12370.0 12250.0 13 20959 -7 \n", + "1 12375.0 12250.0 6 20965 2 \n", + "2 12375.0 12250.0 2 20967 -2 \n", + "3 12375.0 12250.0 1 20968 -1 \n", + "4 12375.0 12250.0 21 20989 0 \n", "\n", " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", - "0 30596 0.0 0.0 13445.0 13445.0 \n", - "1 30596 0.0 0.0 13445.0 13445.0 \n", - "2 30596 0.0 0.0 13445.0 13445.0 \n", - "3 30596 0.0 0.0 13445.0 13445.0 \n", - "4 30596 0.0 0.0 13445.0 13445.0 \n", + "0 32460 159965.0 257900495.0 12270.0 12390.0 \n", + "1 32462 73830.0 257974325.0 12365.0 12390.0 \n", + "2 32460 24610.0 257998935.0 12375.0 12390.0 \n", + "3 32459 12305.0 258011240.0 12370.0 12390.0 \n", + "4 32459 258405.0 258269645.0 12370.0 12390.0 \n", "\n", " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", - "0 0.0 0 0.0 0 0 \n", - "1 0.0 0 0.0 0 0 \n", - "2 0.0 0 0.0 0 0 \n", - "3 0.0 0 0.0 0 0 \n", - "4 0.0 0 0.0 0 0 \n", + "0 12365.0 2 12370.0 1 1 \n", + "1 12360.0 2 12370.0 111 1 \n", + "2 12345.0 3 12355.0 98 1 \n", + "3 12340.0 1 12350.0 97 1 \n", + "4 12340.0 10 12350.0 19 1 \n", "\n", " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", - "0 0 0 0.0 \n", - "1 0 0 0.0 \n", - "2 0 0 0.0 \n", - "3 0 0 0.0 \n", - "4 0 0 0.0 \n", + "0 13 10529 159965.0 \n", + "1 6 10535 73830.0 \n", + "2 2 10537 24610.0 \n", + "3 1 10538 12305.0 \n", + "4 21 10559 258405.0 \n", "\n", " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", - "0 0.0 0 0 \n", - "1 0.0 0 0 \n", - "2 0.0 0 0 \n", - "3 0.0 0 0 \n", - "4 0.0 0 0 \n", + "0 129574875.0 0 10430 \n", + "1 129648705.0 0 10430 \n", + "2 129673315.0 0 10430 \n", + "3 129685620.0 0 10430 \n", + "4 129944025.0 0 10430 \n", "\n", " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", + "0 0.0 128325620.0 2.0 405687.0 \n", + "1 0.0 128325620.0 2.0 405689.0 \n", + "2 0.0 128325620.0 3.0 405692.0 \n", + "3 0.0 128325620.0 1.0 405693.0 \n", + "4 0.0 128325620.0 10.0 405703.0 \n", "\n", " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", - "\n", - " m_nMItemsVolRate m_nDate code_init \n", - "0 0.0 2022-10-07 CF \n", - "1 0.0 2022-10-07 CF \n", - "2 0.0 2022-10-07 CF \n", - "3 0.0 2022-10-07 CF \n", - "4 0.0 2022-10-07 CF \n", - "2022-11-09 14:49:05.501 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n" + "0 1.0 352340.0 2.000000 1.151408 \n", + "1 111.0 352451.0 0.018018 1.151051 \n", + "2 98.0 352549.0 0.030612 1.150739 \n", + "3 97.0 352646.0 0.010309 1.150426 \n", + "4 19.0 352665.0 0.526316 1.150392 \n", + "\n", + " m_nMItemsVolRate code_init \n", + "0 0.0 CF \n", + "1 0.0 CF \n", + "2 0.0 CF \n", + "3 0.0 CF \n", + "4 0.0 CF \n", + "2022-11-09 15:44:37.295 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "(35910, 39)\n", + "(112175, 38)\n", " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", - "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", - "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "0 CF2303 2022-11-01 09:00:00 12365.0 12365.0 12365.0 12365.0 12365.0 \n", + "1 CF2303 2022-11-01 09:00:01 12375.0 12375.0 12375.0 12375.0 12375.0 \n", + "2 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "3 CF2303 2022-11-01 09:00:02 12370.0 12370.0 12370.0 12370.0 12370.0 \n", + "4 CF2303 2022-11-01 09:00:03 12350.0 12350.0 12350.0 12350.0 12350.0 \n", "\n", " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", - "0 13445.0 13445.0 0 0 0 \n", - "1 13445.0 13445.0 0 0 0 \n", - "2 13445.0 13445.0 0 0 0 \n", - "3 13445.0 13445.0 0 0 0 \n", - "4 13445.0 13445.0 0 0 0 \n", + "0 12370.0 12250.0 13 20959 -7 \n", + "1 12375.0 12250.0 6 20965 2 \n", + "2 12375.0 12250.0 2 20967 -2 \n", + "3 12375.0 12250.0 1 20968 -1 \n", + "4 12375.0 12250.0 21 20989 0 \n", "\n", " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", - "0 30596 0.0 0.0 13445.0 13445.0 \n", - "1 30596 0.0 0.0 13445.0 13445.0 \n", - "2 30596 0.0 0.0 13445.0 13445.0 \n", - "3 30596 0.0 0.0 13445.0 13445.0 \n", - "4 30596 0.0 0.0 13445.0 13445.0 \n", + "0 32460 159965.0 257900495.0 12270.0 12390.0 \n", + "1 32462 73830.0 257974325.0 12365.0 12390.0 \n", + "2 32460 24610.0 257998935.0 12375.0 12390.0 \n", + "3 32459 12305.0 258011240.0 12370.0 12390.0 \n", + "4 32459 258405.0 258269645.0 12370.0 12390.0 \n", "\n", " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", - "0 0.0 0 0.0 0 0 \n", - "1 0.0 0 0.0 0 0 \n", - "2 0.0 0 0.0 0 0 \n", - "3 0.0 0 0.0 0 0 \n", - "4 0.0 0 0.0 0 0 \n", + "0 12365.0 2 12370.0 1 1 \n", + "1 12360.0 2 12370.0 111 1 \n", + "2 12345.0 3 12355.0 98 1 \n", + "3 12340.0 1 12350.0 97 1 \n", + "4 12340.0 10 12350.0 19 1 \n", "\n", " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", - "0 0 0 0.0 \n", - "1 0 0 0.0 \n", - "2 0 0 0.0 \n", - "3 0 0 0.0 \n", - "4 0 0 0.0 \n", + "0 13 10529 159965.0 \n", + "1 6 10535 73830.0 \n", + "2 2 10537 24610.0 \n", + "3 1 10538 12305.0 \n", + "4 21 10559 258405.0 \n", "\n", " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", - "0 0.0 0 0 \n", - "1 0.0 0 0 \n", - "2 0.0 0 0 \n", - "3 0.0 0 0 \n", - "4 0.0 0 0 \n", + "0 129574875.0 0 10430 \n", + "1 129648705.0 0 10430 \n", + "2 129673315.0 0 10430 \n", + "3 129685620.0 0 10430 \n", + "4 129944025.0 0 10430 \n", "\n", " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", + "0 0.0 128325620.0 2.0 405687.0 \n", + "1 0.0 128325620.0 2.0 405689.0 \n", + "2 0.0 128325620.0 3.0 405692.0 \n", + "3 0.0 128325620.0 1.0 405693.0 \n", + "4 0.0 128325620.0 10.0 405703.0 \n", "\n", " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", - "0 0.0 0.0 0.0 0.0 \n", - "1 0.0 0.0 0.0 0.0 \n", - "2 0.0 0.0 0.0 0.0 \n", - "3 0.0 0.0 0.0 0.0 \n", - "4 0.0 0.0 0.0 0.0 \n", - "\n", - " m_nMItemsVolRate m_nDate code_init \n", - "0 0.0 2022-10-07 CF \n", - "1 0.0 2022-10-07 CF \n", - "2 0.0 2022-10-07 CF \n", - "3 0.0 2022-10-07 CF \n", - "4 0.0 2022-10-07 CF \n" + "0 1.0 352340.0 2.000000 1.151408 \n", + "1 111.0 352451.0 0.018018 1.151051 \n", + "2 98.0 352549.0 0.030612 1.150739 \n", + "3 97.0 352646.0 0.010309 1.150426 \n", + "4 19.0 352665.0 0.526316 1.150392 \n", + "\n", + " m_nMItemsVolRate code_init \n", + "0 0.0 CF \n", + "1 0.0 CF \n", + "2 0.0 CF \n", + "3 0.0 CF \n", + "4 0.0 CF \n" ] } ], @@ -1101,12 +1119,12 @@ " t_list=['CF2211', 'CF2301', 'CF2303', 'CF2305', 'CF2307', 'CF2309']\n", " # t_list=['AP2212']\n", " # t_list=['CF2211']\n", - " df = tsl.process_result_data_type(tsl.get_mkt_min_k('20221001','20221101',t_list))\n", + " df = tsl.process_result_data_type(tsl.get_trade_tick('20221101','20221101',t_list))\n", " \n", " print(df.shape)\n", " print(df.head())\n", "\n", - "df.to_csv('data/CF202210.csv')\n", + "df.to_csv('data/CF20221101tick.csv')\n", "\n" ] }, @@ -1777,22 +1795,132 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codem_nDatetimem_nPricem_nOpenm_nHighm_nLowm_nClosem_nAccHighm_nAccLowm_iVolume...m_nActAskTurnoverm_nAccActAskTurnoverm_nBidOrderm_nAccBidOrderm_nAskOrderm_nAccAskOrderm_nABOrderRatem_nAccABOrderRatem_nMItemsVolRatecode_init
0CF22112022-10-11 09:01:0014480.014460.014480.014450.014480.014490.014315.0412...1019205.0212045740.01233.068627.01178.054072.00.251.2691780.0CF
\n", + "

1 rows × 38 columns

\n", + "
" + ], + "text/plain": [ + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 CF2211 2022-10-11 09:01:00 14480.0 14460.0 14480.0 14450.0 14480.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", + "0 14490.0 14315.0 412 ... 1019205.0 \n", + "\n", + " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + "0 212045740.0 1233.0 68627.0 1178.0 \n", + "\n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + "0 54072.0 0.25 1.269178 0.0 \n", + "\n", + " code_init \n", + "0 CF \n", + "\n", + "[1 rows x 38 columns]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from usrc.DDBfm import DDBfm\n", + "from src.DDBfm import DDBfm\n", "ddb=DDBfm('dev')\n", + "tb = ddb.sess.loadTable(dbPath=ddb.ddb_hft_dbPath, tableName=ddb.ddf_hft_mink_tbname)\n", + "tb.select('*').where(f\"code=`CF2211\").where(f\"m_nDatetime>=2022.10.11d\").top(1).toDF()\n", "\n", - "db = ddb.create_ddb_database(ddb.ddb_hft_path,ddb.ddb_hft_mink_dbname)\n" + "# tb.toDF()\n", + "\n", + "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ + "db = ddb.create_ddb_database(ddb.ddb_hft_path,ddb.ddb_hft_mink_dbname)\n", "ddb.add_new_hft_table(db,ddb.ddf_hft_mink_tbname,df)" ] }, @@ -1821,26 +1949,1057 @@ "# ))" ] }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'01': ['2022-01-01',\n", + " '2022-02-01',\n", + " '2022-03-01',\n", + " '2022-04-01',\n", + " '2022-05-01',\n", + " '2022-06-01',\n", + " '2022-07-01',\n", + " '2022-08-01',\n", + " '2022-09-01',\n", + " '2022-10-01',\n", + " '2022-11-01'],\n", + " '02': ['2022-01-02',\n", + " '2022-02-02',\n", + " '2022-03-02',\n", + " '2022-04-02',\n", + " '2022-05-02',\n", + " '2022-06-02',\n", + " '2022-07-02',\n", + " '2022-08-02',\n", + " '2022-09-02',\n", + " '2022-10-02',\n", + " '2022-11-02'],\n", + " '03': ['2022-01-03',\n", + " '2022-02-03',\n", + " '2022-03-03',\n", + " '2022-04-03',\n", + " '2022-05-03',\n", + " '2022-06-03',\n", + " '2022-07-03',\n", + " '2022-08-03',\n", + " '2022-09-03',\n", + " '2022-10-03',\n", + " '2022-11-03'],\n", + " '04': ['2022-01-04',\n", + " '2022-02-04',\n", + " '2022-03-04',\n", + " '2022-04-04',\n", + " '2022-05-04',\n", + " '2022-06-04',\n", + " '2022-07-04',\n", + " '2022-08-04',\n", + " '2022-09-04',\n", + " '2022-10-04',\n", + " '2022-11-04'],\n", + " '05': ['2022-01-05',\n", + " '2022-02-05',\n", + " '2022-03-05',\n", + " '2022-04-05',\n", + " '2022-05-05',\n", + " '2022-06-05',\n", + " '2022-07-05',\n", + " '2022-08-05',\n", + " '2022-09-05',\n", + " '2022-10-05',\n", + " '2022-11-05'],\n", + " '06': ['2022-01-06',\n", + " '2022-02-06',\n", + " '2022-03-06',\n", + " '2022-04-06',\n", + " '2022-05-06',\n", + " '2022-06-06',\n", + " '2022-07-06',\n", + " '2022-08-06',\n", + " '2022-09-06',\n", + " '2022-10-06',\n", + " '2022-11-06'],\n", + " '07': ['2022-01-07',\n", + " '2022-02-07',\n", + " '2022-03-07',\n", + " '2022-04-07',\n", + " '2022-05-07',\n", + " '2022-06-07',\n", + " '2022-07-07',\n", + " '2022-08-07',\n", + " '2022-09-07',\n", + " '2022-10-07',\n", + " '2022-11-07'],\n", + " '08': ['2022-01-08',\n", + " '2022-02-08',\n", + " '2022-03-08',\n", + " '2022-04-08',\n", + " '2022-05-08',\n", + " '2022-06-08',\n", + " '2022-07-08',\n", + " '2022-08-08',\n", + " '2022-09-08',\n", + " '2022-10-08',\n", + " '2022-11-08'],\n", + " '09': ['2022-01-09',\n", + " '2022-02-09',\n", + " '2022-03-09',\n", + " '2022-04-09',\n", + " '2022-05-09',\n", + " '2022-06-09',\n", + " '2022-07-09',\n", + " '2022-08-09',\n", + " '2022-09-09',\n", + " '2022-10-09'],\n", + " '10': ['2022-01-10',\n", + " '2022-02-10',\n", + " '2022-03-10',\n", + " '2022-04-10',\n", + " '2022-05-10',\n", + " '2022-06-10',\n", + " '2022-07-10',\n", + " '2022-08-10',\n", + " '2022-09-10',\n", + " '2022-10-10'],\n", + " '11': ['2022-01-11',\n", + " '2022-02-11',\n", + " '2022-03-11',\n", + " '2022-04-11',\n", + " '2022-05-11',\n", + " '2022-06-11',\n", + " '2022-07-11',\n", + " '2022-08-11',\n", + " '2022-09-11',\n", + " '2022-10-11'],\n", + " '12': ['2022-01-12',\n", + " '2022-02-12',\n", + " '2022-03-12',\n", + " '2022-04-12',\n", + " '2022-05-12',\n", + " '2022-06-12',\n", + " '2022-07-12',\n", + " '2022-08-12',\n", + " '2022-09-12',\n", + " '2022-10-12'],\n", + " '13': ['2022-01-13',\n", + " '2022-02-13',\n", + " '2022-03-13',\n", + " '2022-04-13',\n", + " '2022-05-13',\n", + " '2022-06-13',\n", + " '2022-07-13',\n", + " '2022-08-13',\n", + " '2022-09-13',\n", + " '2022-10-13'],\n", + " '14': ['2022-01-14',\n", + " '2022-02-14',\n", + " '2022-03-14',\n", + " '2022-04-14',\n", + " '2022-05-14',\n", + " '2022-06-14',\n", + " '2022-07-14',\n", + " '2022-08-14',\n", + " '2022-09-14',\n", + " '2022-10-14'],\n", + " '15': ['2022-01-15',\n", + " '2022-02-15',\n", + " '2022-03-15',\n", + " '2022-04-15',\n", + " '2022-05-15',\n", + " '2022-06-15',\n", + " '2022-07-15',\n", + " '2022-08-15',\n", + " '2022-09-15',\n", + " '2022-10-15'],\n", + " '16': ['2022-01-16',\n", + " '2022-02-16',\n", + " '2022-03-16',\n", + " '2022-04-16',\n", + " '2022-05-16',\n", + " '2022-06-16',\n", + " '2022-07-16',\n", + " '2022-08-16',\n", + " '2022-09-16',\n", + " '2022-10-16'],\n", + " '17': ['2022-01-17',\n", + " '2022-02-17',\n", + " '2022-03-17',\n", + " '2022-04-17',\n", + " '2022-05-17',\n", + " '2022-06-17',\n", + " '2022-07-17',\n", + " '2022-08-17',\n", + " '2022-09-17',\n", + " '2022-10-17'],\n", + " '18': ['2022-01-18',\n", + " '2022-02-18',\n", + " '2022-03-18',\n", + " '2022-04-18',\n", + " '2022-05-18',\n", + " '2022-06-18',\n", + " '2022-07-18',\n", + " '2022-08-18',\n", + " '2022-09-18',\n", + " '2022-10-18'],\n", + " '19': ['2022-01-19',\n", + " '2022-02-19',\n", + " '2022-03-19',\n", + " '2022-04-19',\n", + " '2022-05-19',\n", + " '2022-06-19',\n", + " '2022-07-19',\n", + " '2022-08-19',\n", + " '2022-09-19',\n", + " '2022-10-19'],\n", + " '20': ['2022-01-20',\n", + " '2022-02-20',\n", + " '2022-03-20',\n", + " '2022-04-20',\n", + " '2022-05-20',\n", + " '2022-06-20',\n", + " '2022-07-20',\n", + " '2022-08-20',\n", + " '2022-09-20',\n", + " '2022-10-20'],\n", + " '21': ['2022-01-21',\n", + " '2022-02-21',\n", + " '2022-03-21',\n", + " '2022-04-21',\n", + " '2022-05-21',\n", + " '2022-06-21',\n", + " '2022-07-21',\n", + " '2022-08-21',\n", + " '2022-09-21',\n", + " '2022-10-21'],\n", + " '22': ['2022-01-22',\n", + " '2022-02-22',\n", + " '2022-03-22',\n", + " '2022-04-22',\n", + " '2022-05-22',\n", + " '2022-06-22',\n", + " '2022-07-22',\n", + " '2022-08-22',\n", + " '2022-09-22',\n", + " '2022-10-22'],\n", + " '23': ['2022-01-23',\n", + " '2022-02-23',\n", + " '2022-03-23',\n", + " '2022-04-23',\n", + " '2022-05-23',\n", + " '2022-06-23',\n", + " '2022-07-23',\n", + " '2022-08-23',\n", + " '2022-09-23',\n", + " '2022-10-23'],\n", + " '24': ['2022-01-24',\n", + " '2022-02-24',\n", + " '2022-03-24',\n", + " '2022-04-24',\n", + " '2022-05-24',\n", + " '2022-06-24',\n", + " '2022-07-24',\n", + " '2022-08-24',\n", + " '2022-09-24',\n", + " '2022-10-24'],\n", + " '25': ['2022-01-25',\n", + " '2022-02-25',\n", + " '2022-03-25',\n", + " '2022-04-25',\n", + " '2022-05-25',\n", + " '2022-06-25',\n", + " '2022-07-25',\n", + " '2022-08-25',\n", + " '2022-09-25',\n", + " '2022-10-25'],\n", + " '26': ['2022-01-26',\n", + " '2022-02-26',\n", + " '2022-03-26',\n", + " '2022-04-26',\n", + " '2022-05-26',\n", + " '2022-06-26',\n", + " '2022-07-26',\n", + " '2022-08-26',\n", + " '2022-09-26',\n", + " '2022-10-26'],\n", + " '27': ['2022-01-27',\n", + " '2022-02-27',\n", + " '2022-03-27',\n", + " '2022-04-27',\n", + " '2022-05-27',\n", + " '2022-06-27',\n", + " '2022-07-27',\n", + " '2022-08-27',\n", + " '2022-09-27',\n", + " '2022-10-27'],\n", + " '28': ['2022-01-28',\n", + " '2022-02-28',\n", + " '2022-03-28',\n", + " '2022-04-28',\n", + " '2022-05-28',\n", + " '2022-06-28',\n", + " '2022-07-28',\n", + " '2022-08-28',\n", + " '2022-09-28',\n", + " '2022-10-28'],\n", + " '29': ['2022-01-29',\n", + " '2022-03-29',\n", + " '2022-04-29',\n", + " '2022-05-29',\n", + " '2022-06-29',\n", + " '2022-07-29',\n", + " '2022-08-29',\n", + " '2022-09-29',\n", + " '2022-10-29'],\n", + " '30': ['2022-01-30',\n", + " '2022-03-30',\n", + " '2022-04-30',\n", + " '2022-05-30',\n", + " '2022-06-30',\n", + " '2022-07-30',\n", + " '2022-08-30',\n", + " '2022-09-30',\n", + " '2022-10-30'],\n", + " '31': ['2022-01-31',\n", + " '2022-03-31',\n", + " '2022-05-31',\n", + " '2022-07-31',\n", + " '2022-08-31',\n", + " '2022-10-31']}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "allDates = pd.date_range('20220101', '20221108', freq ='D')\n", + "dates_dict_by_day={}\n", + "\n", + "for d in list(allDates.astype('str')):\n", + " day = d[-2:]\n", + " if day not in dates_dict_by_day:\n", + " dates_dict_by_day[day] = [d]\n", + " else:\n", + " dates_dict_by_day[day].append(d)\n", + "dates_dict_by_day" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "\n", + "start_date='2022-10-01'\n", + "end_date='2022-11-08'\n", + "allDates = pd.date_range(start_date, end_date, freq ='D')\n", + "allDates = [i.replace('-','') for i in list(allDates.astype('str'))]\n", + "allDates" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'AP': ['AP2211', 'AP2212', 'AP2301', 'AP2303', 'AP2304', 'AP2305', 'AP2310'], 'CF': ['CF2211', 'CF2301', 'CF2303', 'CF2305', 'CF2307', 'CF2309'], 'CJ': ['CJ2212', 'CJ2301', 'CJ2303', 'CJ2305', 'CJ2307', 'CJ2309'], 'CY': ['CY2211', 'CY2212', 'CY2301', 'CY2302', 'CY2303', 'CY2304', 'CY2305', 'CY2306', 'CY2307', 'CY2308', 'CY2309', 'CY2310'], 'FG': ['FG2211', 'FG2212', 'FG2301', 'FG2302', 'FG2303', 'FG2304', 'FG2305', 'FG2306', 'FG2307', 'FG2308', 'FG2309', 'FG2310'], 'IC': ['IC2211', 'IC2212', 'IC2303', 'IC2306'], 'IF': ['IF2211', 'IF2212', 'IF2303', 'IF2306'], 'IH': ['IH2211', 'IH2212', 'IH2303', 'IH2306'], 'IM': ['IM2211', 'IM2212', 'IM2303', 'IM2306'], 'JR': ['JR2211', 'JR2301', 'JR2303', 'JR2305', 'JR2307', 'JR2309'], 'LR': ['LR2211', 'LR2301', 'LR2303', 'LR2305', 'LR2307', 'LR2309'], 'MA': ['MA2211', 'MA2212', 'MA2301', 'MA2302', 'MA2303', 'MA2304', 'MA2305', 'MA2306', 'MA2307', 'MA2308', 'MA2309', 'MA2310'], 'OI': ['OI2211', 'OI2301', 'OI2303', 'OI2305', 'OI2307', 'OI2309'], 'PF': ['PF2211', 'PF2212', 'PF2301', 'PF2302', 'PF2303', 'PF2304', 'PF2305', 'PF2306', 'PF2307', 'PF2308', 'PF2309', 'PF2310'], 'PK': ['PK2211', 'PK2212', 'PK2301', 'PK2303', 'PK2304', 'PK2310'], 'PM': ['PM2211', 'PM2301', 'PM2303', 'PM2305', 'PM2307', 'PM2309'], 'RI': ['RI2211', 'RI2301', 'RI2303', 'RI2305', 'RI2307', 'RI2309'], 'RM': ['RM2211', 'RM2301', 'RM2303', 'RM2305', 'RM2307', 'RM2308', 'RM2309'], 'RS': ['RS2211', 'RS2307', 'RS2308', 'RS2309'], 'SA': ['SA2211', 'SA2212', 'SA2301', 'SA2302', 'SA2303', 'SA2304', 'SA2305', 'SA2306', 'SA2307', 'SA2308', 'SA2309', 'SA2310'], 'SF': ['SF2211', 'SF2212', 'SF2301', 'SF2302', 'SF2303', 'SF2304', 'SF2305', 'SF2306', 'SF2307', 'SF2308', 'SF2309', 'SF2310'], 'SM': ['SM2211', 'SM2212', 'SM2301', 'SM2302', 'SM2303', 'SM2304', 'SM2305', 'SM2306', 'SM2307', 'SM2308', 'SM2309', 'SM2310'], 'SR': ['SR2211', 'SR2301', 'SR2303', 'SR2305', 'SR2307', 'SR2309'], 'T': ['T2212', 'T2303', 'T2306'], 'TA': ['TA2211', 'TA2212', 'TA2301', 'TA2302', 'TA2303', 'TA2304', 'TA2305', 'TA2306', 'TA2307', 'TA2308', 'TA2309', 'TA2310'], 'TF': ['TF2212', 'TF2303', 'TF2306'], 'TS': ['TS2212', 'TS2303', 'TS2306'], 'UR': ['UR2211', 'UR2212', 'UR2301', 'UR2302', 'UR2303', 'UR2304', 'UR2305', 'UR2306', 'UR2307', 'UR2308', 'UR2309', 'UR2310'], 'WH': ['WH2211', 'WH2301', 'WH2303', 'WH2305', 'WH2307', 'WH2309'], 'ZC': ['ZC2212', 'ZC2301', 'ZC2302', 'ZC2303', 'ZC2304', 'ZC2305', 'ZC2306', 'ZC2307', 'ZC2308', 'ZC2309', 'ZC2310', 'ZC2311'], 'a': ['a2211', 'a2301', 'a2303', 'a2305', 'a2307', 'a2309'], 'ag': ['ag2211', 'ag2212', 'ag2301', 'ag2302', 'ag2303', 'ag2304', 'ag2305', 'ag2306', 'ag2307', 'ag2308', 'ag2309', 'ag2310'], 'al': ['al2211', 'al2212', 'al2301', 'al2302', 'al2303', 'al2304', 'al2305', 'al2306', 'al2307', 'al2308', 'al2309', 'al2310'], 'au': ['au2211', 'au2212', 'au2301', 'au2302', 'au2304', 'au2306', 'au2308', 'au2310'], 'b': ['b2211', 'b2212', 'b2301', 'b2302', 'b2303', 'b2304', 'b2305', 'b2306', 'b2307', 'b2308', 'b2309', 'b2310'], 'bb': ['bb2211', 'bb2212', 'bb2301', 'bb2302', 'bb2303', 'bb2304', 'bb2305', 'bb2306', 'bb2307', 'bb2308', 'bb2309', 'bb2310'], 'bc': ['bc2211', 'bc2212', 'bc2301', 'bc2302', 'bc2303', 'bc2304', 'bc2305', 'bc2306', 'bc2307', 'bc2308', 'bc2309', 'bc2310'], 'bu': ['bu2211', 'bu2212', 'bu2301', 'bu2302', 'bu2303', 'bu2304', 'bu2305', 'bu2306', 'bu2307', 'bu2308', 'bu2309', 'bu2310', 'bu2312', 'bu2403', 'bu2406', 'bu2409'], 'c': ['c2211', 'c2301', 'c2303', 'c2305', 'c2307', 'c2309'], 'cs': ['cs2211', 'cs2301', 'cs2303', 'cs2305', 'cs2307', 'cs2309'], 'cu': ['cu2211', 'cu2212', 'cu2301', 'cu2302', 'cu2303', 'cu2304', 'cu2305', 'cu2306', 'cu2307', 'cu2308', 'cu2309', 'cu2310'], 'eb': ['eb2211', 'eb2212', 'eb2301', 'eb2302', 'eb2303', 'eb2304', 'eb2305', 'eb2306', 'eb2307', 'eb2308', 'eb2309', 'eb2310'], 'eg': ['eg2211', 'eg2212', 'eg2301', 'eg2302', 'eg2303', 'eg2304', 'eg2305', 'eg2306', 'eg2307', 'eg2308', 'eg2309', 'eg2310'], 'fb': ['fb2211', 'fb2212', 'fb2301', 'fb2302', 'fb2303', 'fb2304', 'fb2305', 'fb2306', 'fb2307', 'fb2308', 'fb2309', 'fb2310'], 'fu': ['fu2212', 'fu2301', 'fu2302', 'fu2303', 'fu2304', 'fu2305', 'fu2306', 'fu2307', 'fu2308', 'fu2309', 'fu2310', 'fu2311'], 'hc': ['hc2211', 'hc2212', 'hc2301', 'hc2302', 'hc2303', 'hc2304', 'hc2305', 'hc2306', 'hc2307', 'hc2308', 'hc2309', 'hc2310'], 'i': ['i2211', 'i2212', 'i2301', 'i2302', 'i2303', 'i2304', 'i2305', 'i2306', 'i2307', 'i2308', 'i2309', 'i2310'], 'j': ['j2211', 'j2212', 'j2301', 'j2302', 'j2303', 'j2304', 'j2305', 'j2306', 'j2307', 'j2308', 'j2309', 'j2310'], 'jd': ['jd2211', 'jd2212', 'jd2301', 'jd2302', 'jd2303', 'jd2304', 'jd2305', 'jd2306', 'jd2307', 'jd2308', 'jd2309', 'jd2310'], 'jm': ['jm2211', 'jm2212', 'jm2301', 'jm2302', 'jm2303', 'jm2304', 'jm2305', 'jm2306', 'jm2307', 'jm2308', 'jm2309', 'jm2310'], 'l': ['l2211', 'l2212', 'l2301', 'l2302', 'l2303', 'l2304', 'l2305', 'l2306', 'l2307', 'l2308', 'l2309', 'l2310'], 'lh': ['lh2211', 'lh2301', 'lh2303', 'lh2305', 'lh2307', 'lh2309'], 'lu': ['lu2212', 'lu2301', 'lu2302', 'lu2303', 'lu2304', 'lu2305', 'lu2306', 'lu2307', 'lu2308', 'lu2309', 'lu2310', 'lu2311'], 'm': ['m2211', 'm2212', 'm2301', 'm2303', 'm2305', 'm2307', 'm2308', 'm2309'], 'ni': ['ni2211', 'ni2212', 'ni2301', 'ni2302', 'ni2303', 'ni2304', 'ni2305', 'ni2306', 'ni2307', 'ni2308', 'ni2309', 'ni2310'], 'nr': ['nr2211', 'nr2212', 'nr2301', 'nr2302', 'nr2303', 'nr2304', 'nr2305', 'nr2306', 'nr2307', 'nr2308', 'nr2309', 'nr2310'], 'p': ['p2211', 'p2212', 'p2301', 'p2302', 'p2303', 'p2304', 'p2305', 'p2306', 'p2307', 'p2308', 'p2309', 'p2310'], 'pb': ['pb2211', 'pb2212', 'pb2301', 'pb2302', 'pb2303', 'pb2304', 'pb2305', 'pb2306', 'pb2307', 'pb2308', 'pb2309', 'pb2310'], 'pg': ['pg2211', 'pg2212', 'pg2301', 'pg2302', 'pg2303', 'pg2304', 'pg2305', 'pg2306', 'pg2307', 'pg2308', 'pg2309', 'pg2310'], 'pp': ['pp2211', 'pp2212', 'pp2301', 'pp2302', 'pp2303', 'pp2304', 'pp2305', 'pp2306', 'pp2307', 'pp2308', 'pp2309', 'pp2310'], 'rb': ['rb2211', 'rb2212', 'rb2301', 'rb2302', 'rb2303', 'rb2304', 'rb2305', 'rb2306', 'rb2307', 'rb2308', 'rb2309', 'rb2310'], 'rr': ['rr2211', 'rr2212', 'rr2301', 'rr2302', 'rr2303', 'rr2304', 'rr2305', 'rr2306', 'rr2307', 'rr2308', 'rr2309', 'rr2310'], 'ru': ['ru2211', 'ru2301', 'ru2303', 'ru2304', 'ru2305', 'ru2306', 'ru2307', 'ru2308', 'ru2309', 'ru2310'], 'sc': ['sc2212', 'sc2301', 'sc2302', 'sc2303', 'sc2304', 'sc2305', 'sc2306', 'sc2307', 'sc2308', 'sc2309', 'sc2310', 'sc2311', 'sc2312', 'sc2403', 'sc2406', 'sc2409', 'sc2412', 'sc2503', 'sc2506', 'sc2509'], 'sn': ['sn2211', 'sn2212', 'sn2301', 'sn2302', 'sn2303', 'sn2304', 'sn2305', 'sn2306', 'sn2307', 'sn2308', 'sn2309', 'sn2310'], 'sp': ['sp2211', 'sp2212', 'sp2301', 'sp2302', 'sp2303', 'sp2304', 'sp2305', 'sp2306', 'sp2307', 'sp2308', 'sp2309', 'sp2310'], 'ss': ['ss2211', 'ss2212', 'ss2301', 'ss2302', 'ss2303', 'ss2304', 'ss2305', 'ss2306', 'ss2307', 'ss2308', 'ss2309', 'ss2310'], 'v': ['v2211', 'v2212', 'v2301', 'v2302', 'v2303', 'v2304', 'v2305', 'v2306', 'v2307', 'v2308', 'v2309', 'v2310'], 'wr': ['wr2211', 'wr2212', 'wr2301', 'wr2302', 'wr2303', 'wr2304', 'wr2305', 'wr2306', 'wr2307', 'wr2308', 'wr2309', 'wr2310'], 'y': ['y2211', 'y2212', 'y2301', 'y2303', 'y2305', 'y2307', 'y2308', 'y2309'], 'zn': ['zn2211', 'zn2212', 'zn2301', 'zn2302', 'zn2303', 'zn2304', 'zn2305', 'zn2306', 'zn2307', 'zn2308', 'zn2309', 'zn2310']}\n" + ] + } + ], + "source": [ + "from data.code_list import code_list_pickel\n", + "# print(code_list_pickel)\n", + "all_fm_init=['sc', 'v', 'TS', 'MA', 'AP', 'jm', 'bc', 'bb', 'fu', 'IM', 'IF', 'a', 'lu', 'FG', 'cu', 'al', 'IH', 'RS', 'pg', 'CF', 'SF', 'ni', 'hc', 'UR', 'm', 'SR', 'j', 'PF', 'RM', 'T', 'c', 'JR', 'l', 'p', 'sp', 'CY', 'pb', 'TF', 'b', 'eg', 'rb', 'PK', 'sn', 'nr', 'pp', 'CJ', 'eb', 'SA', 'y', 'RI', 'lh', 'jd', 'OI', 'WH', 'ss', 'ru', 'zn', 'fb', 'rr', 'PM', 'au', 'TA', 'ZC', 'IC', 'bu', 'SM', 'wr', 'cs', 'LR', 'ag', 'i']\n", + "all_code_dict_by_init={}\n", + "for c in code_list_pickel:\n", + " init = c[:-4]\n", + " if init in all_code_dict_by_init:\n", + " all_code_dict_by_init[init].append(c)\n", + " else:\n", + " all_code_dict_by_init[init]=[c]\n", + "print(all_code_dict_by_init)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-11-10 13:16:00.361292: [2499840] Error: unmarshall failed\n" + ] + }, + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import dolphindb as ddb\n", + "from src.DDBfm import DDBfm\n", + "ddb = DDBfm('prd')\n", + "# df = ddb.sess.run(f'select top 1 * from pt where code=`CF2301 and m_nDatetime.date()>=2022.09.30d')\n", + "# df = ddb.sess.run(f'select top 1 * from pt where code=`CF2301 and m_nDatetime.date()>={curr_date_formatted}d')\n", + "tb = ddb.sess.loadTable(dbPath=ddb.ddb_hft_dbPath, tableName=ddb.ddf_hft_mink_tbname)\n", + "\n", + "curr_code='CF2211'\n", + "cuff_code='T2212'\n", + "curr_date_formatted='2022.11.01'\n", + "# df = tb.select('*').top(1).toDF() \n", + "\n", + "cond=f\"code_init=`{curr_code[:2]}, m_nDatetime.date()={curr_date_formatted}d\"\n", + "# print(cond)\n", + "df = tb.select('distinct code').where(cond).toDF()\n", + "# df\n", + "# df = tb.select('*').top(1).toDF() \n", + "# df = tb.select('*').where(f\"code=`AP2301, m_nDatetime.date()=2022.09.30d\").toDF() \n", + "# df[df['m_nPrice']==8274]\n", + "# pool = ddb.DBConnectionPool(\"192.168.1.88\", 8848, 20, \"admin\", \"123456\")\n", + "df['distinct_code'].to_list()" + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import dolphindb as ddb\n", + "from src.DDBfm import DDBfm\n", + "ddb = DDBfm('prd')\n", + "# df = ddb.sess.run(f'select top 1 * from pt where code=`CF2301 and m_nDatetime.date()>=2022.09.30d')\n", + "# df = ddb.sess.run(f'select top 1 * from pt where code=`CF2301 and m_nDatetime.date()>={curr_date_formatted}d')\n", + "tb = ddb.sess.loadTable(dbPath=ddb.ddb_hft_dbPath, tableName=ddb.ddf_hft_mink_tbname)\n", + "\n", + "# curr_code='CF2211'\n", + "cuff_code='T2212'\n", + "curr_date_formatted='2022.11.01'\n", + "# df = tb.select('*').top(1).toDF() \n", + "\n", + "cond=f\"code_init=`{curr_code[:2]}, m_nDatetime.date()={curr_date_formatted}d\"\n", + "# print(cond)\n", + "df = tb.select('distinct code').where(cond).toDF()\n", + "# df\n", + "# df = tb.select('*').top(1).toDF() \n", + "# df = tb.select('*').where(f\"code=`AP2301, m_nDatetime.date()=2022.09.30d\").toDF() \n", + "# df[df['m_nPrice']==8274]\n", + "# pool = ddb.DBConnectionPool(\"192.168.1.88\", 8848, 20, \"admin\", \"123456\")\n", + "df['distinct_code'].to_list()" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'DBConnectionPool' object has no attribute 'loadTable'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [3], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m pool \u001b[39m=\u001b[39m ddb\u001b[39m.\u001b[39mDBConnectionPool(\u001b[39m\"\u001b[39m\u001b[39m192.168.1.88\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m8848\u001b[39m, \u001b[39m20\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39madmin\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m123456\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 5\u001b[0m \u001b[39m# pool.run('getAllDBs()').value()\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m pool\u001b[39m.\u001b[39;49mloadTable(dbPath\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mdfs://hft_futuremarket_ts\u001b[39m\u001b[39m\"\u001b[39m, tableName\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mMinKlinePartitioned\u001b[39m\u001b[39m'\u001b[39m)\n", + "\u001b[0;31mAttributeError\u001b[0m: 'DBConnectionPool' object has no attribute 'loadTable'" + ] + } + ], + "source": [ + "import dolphindb as ddb\n", + "from src.DDBfm import DDBfm\n", + "\n", + "pool = ddb.DBConnectionPool(\"192.168.1.88\", 8848, 20, \"admin\", \"123456\")\n", + "# pool.run('getAllDBs()').value()\n", + "pool.loadTable(dbPath=\"dfs://hft_futuremarket_ts\", tableName='MinKlinePartitioned')\n", + "# appender = ddb.PartitionedTableAppender(\"dfs://hft_futuremarket_ts\", \"MinKlinePartitioned\", 'm_nDatetime', pool)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd\n", + "df_all = pd.concat(df_list)\n", + "r= appender.append(df_all)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import dolphindb as ddb\n", + "from src.DDBfm import DDBfm\n", + "ddbfm=DDBfm('dev',pool=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-11-10 10:58:40.946 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", + "2022-11-10 10:58:41.103 | DEBUG | src.TSLfm:get_mkt_min_k:209 - \n", + " SetSysParam(pn_cycle(), cy_1m());\n", + " stock_list := Array('AP2211', 'AP2212', 'AP2301');\n", + " r := select \n", + " ['StockID'] as 'code',\n", + " DateTimeToStr(['date']) as 'm_nDatetime',\n", + "\n", + " ['price'] as 'm_nPrice',\n", + " ['open'] as 'm_nOpen',\n", + " ['high'] as 'm_nHigh',\n", + " ['low'] as 'm_nLow',\n", + " ['close'] as 'm_nClose',\n", + "\n", + " ['sectional_high'] as 'm_nAccHigh',\n", + " ['sectional_low'] as 'm_nAccLow',\n", + "\n", + " ['vol'] as 'm_iVolume', //成交量\n", + " ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量\n", + "\n", + " ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量\n", + " ['sectional_cjbs'] as 'm_nAccMatchItems',\n", + "\n", + " ['amount'] as 'm_iTurnover', //成交金额\n", + " ['sectional_amount'] as 'm_iAccTurnover', \n", + "\n", + " ['yclose'] as 'm_nPreClose', //上一周期的收盘价\n", + " ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘\n", + " \n", + " ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?\n", + " ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量\n", + " ['sale1'] as 'm_nAskPrice', \n", + " ['sc1'] as 'm_nAskVolume', \n", + "\n", + " ['zmm'] as 'm_iABFlag', //买卖标识\n", + "\n", + " ['buy_vol'] as 'm_nActBidVolume', //主买量\n", + " ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量\n", + " ['buy_amount'] as 'm_nActBidTurnover', //主买金额\n", + " ['sectional_buy_amount'] as 'm_nAccActBidTurnover', \n", + "\n", + " ['sale_vol'] as 'm_nActAskVolume', \n", + " ['sectional_sale_vol'] as 'm_nAccActAskVolume', \n", + " ['sale_amount'] as 'm_nActAskTurnover',\n", + " ['sectional_sale_amount'] as 'm_nAccActAskTurnover', \n", + "\n", + " ['w_buy'] as 'm_nBidOrder', //委买\n", + " ['sectional_w_buy'] as 'm_nAccBidOrder', \n", + " ['w_sale'] as 'm_nAskOrder',\n", + " ['sectional_w_sale'] as 'm_nAccAskOrder',\n", + "\n", + " ['wb'] as 'm_nABOrderRate', //委比\n", + " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", + " ['lb'] as 'm_nMItemsVolRate'//量比\n", + "\n", + "\n", + " from markettable\n", + " datekey 20220902T to 20220902T+0.999 \n", + " of stock_list\n", + " end;\n", + " \n", + " return r; \n", + " \n", + "2022-11-10 10:58:41.551 | INFO | src.TSLfm:process_result_data_type:218 - Processing new df of shape (675, 37), which looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 AP2211 2022-09-02 09:01:00 8950.0 8966.0 8970.0 8950.0 8950.0 \n", + "1 AP2211 2022-09-02 09:02:00 8918.0 8950.0 8950.0 8917.0 8918.0 \n", + "2 AP2211 2022-09-02 09:03:00 8912.0 8914.0 8914.0 8903.0 8912.0 \n", + "3 AP2211 2022-09-02 09:04:00 8920.0 8924.0 8924.0 8912.0 8920.0 \n", + "4 AP2211 2022-09-02 09:05:00 8908.0 8920.0 8920.0 8908.0 8908.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nAccActAskVolume \\\n", + "0 8970.0 8950.0 3.0 ... 0.0 \n", + "1 8970.0 8917.0 8.0 ... 6.0 \n", + "2 8970.0 8903.0 20.0 ... 17.0 \n", + "3 8970.0 8903.0 17.0 ... 29.0 \n", + "4 8970.0 8903.0 3.0 ... 30.0 \n", + "\n", + " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", + "0 0.0 0.0 65.0 65.0 \n", + "1 53522.0 53522.0 65.0 130.0 \n", + "2 98069.0 151591.0 77.0 207.0 \n", + "3 106951.0 258542.0 64.0 271.0 \n", + "4 8918.0 267460.0 67.0 338.0 \n", + "\n", + " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", + "0 61.0 61.0 1.0 1.065574 \n", + "1 57.0 118.0 1.0 1.101695 \n", + "2 64.0 182.0 0.5 1.137363 \n", + "3 63.0 245.0 1.0 1.106122 \n", + "4 49.0 294.0 1.0 1.149660 \n", + "\n", + " m_nMItemsVolRate \n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "\n", + "[5 rows x 37 columns]\n", + "2022-11-10 10:58:41.559 | INFO | src.TSLfm:process_result_data_type:232 - Processing done, new df looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 AP2211 2022-09-02 09:01:00 8950.0 8966.0 8970.0 8950.0 8950.0 \n", + "1 AP2211 2022-09-02 09:02:00 8918.0 8950.0 8950.0 8917.0 8918.0 \n", + "2 AP2211 2022-09-02 09:03:00 8912.0 8914.0 8914.0 8903.0 8912.0 \n", + "3 AP2211 2022-09-02 09:04:00 8920.0 8924.0 8924.0 8912.0 8920.0 \n", + "4 AP2211 2022-09-02 09:05:00 8908.0 8920.0 8920.0 8908.0 8908.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", + "0 8970.0 8950.0 3 ... 0.0 \n", + "1 8970.0 8917.0 8 ... 53522.0 \n", + "2 8970.0 8903.0 20 ... 98069.0 \n", + "3 8970.0 8903.0 17 ... 106951.0 \n", + "4 8970.0 8903.0 3 ... 8918.0 \n", + "\n", + " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + "0 0.0 65.0 65.0 61.0 \n", + "1 53522.0 65.0 130.0 57.0 \n", + "2 151591.0 77.0 207.0 64.0 \n", + "3 258542.0 64.0 271.0 63.0 \n", + "4 267460.0 67.0 338.0 49.0 \n", + "\n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + "0 61.0 1.0 1.065574 0.0 \n", + "1 118.0 1.0 1.101695 0.0 \n", + "2 182.0 0.5 1.137363 0.0 \n", + "3 245.0 1.0 1.106122 0.0 \n", + "4 294.0 1.0 1.149660 0.0 \n", + "\n", + " code_init \n", + "0 AP \n", + "1 AP \n", + "2 AP \n", + "3 AP \n", + "4 AP \n", + "\n", + "[5 rows x 38 columns]\n", + "2022-11-10 10:58:41.559 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n", + "2022-11-10 10:58:41.559 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", + "2022-11-10 10:58:41.711 | DEBUG | src.TSLfm:get_mkt_min_k:209 - \n", + " SetSysParam(pn_cycle(), cy_1m());\n", + " stock_list := Array('AP2211', 'AP2212', 'AP2301');\n", + " r := select \n", + " ['StockID'] as 'code',\n", + " DateTimeToStr(['date']) as 'm_nDatetime',\n", + "\n", + " ['price'] as 'm_nPrice',\n", + " ['open'] as 'm_nOpen',\n", + " ['high'] as 'm_nHigh',\n", + " ['low'] as 'm_nLow',\n", + " ['close'] as 'm_nClose',\n", + "\n", + " ['sectional_high'] as 'm_nAccHigh',\n", + " ['sectional_low'] as 'm_nAccLow',\n", + "\n", + " ['vol'] as 'm_iVolume', //成交量\n", + " ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量\n", + "\n", + " ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量\n", + " ['sectional_cjbs'] as 'm_nAccMatchItems',\n", + "\n", + " ['amount'] as 'm_iTurnover', //成交金额\n", + " ['sectional_amount'] as 'm_iAccTurnover', \n", + "\n", + " ['yclose'] as 'm_nPreClose', //上一周期的收盘价\n", + " ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘\n", + " \n", + " ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?\n", + " ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量\n", + " ['sale1'] as 'm_nAskPrice', \n", + " ['sc1'] as 'm_nAskVolume', \n", + "\n", + " ['zmm'] as 'm_iABFlag', //买卖标识\n", + "\n", + " ['buy_vol'] as 'm_nActBidVolume', //主买量\n", + " ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量\n", + " ['buy_amount'] as 'm_nActBidTurnover', //主买金额\n", + " ['sectional_buy_amount'] as 'm_nAccActBidTurnover', \n", + "\n", + " ['sale_vol'] as 'm_nActAskVolume', \n", + " ['sectional_sale_vol'] as 'm_nAccActAskVolume', \n", + " ['sale_amount'] as 'm_nActAskTurnover',\n", + " ['sectional_sale_amount'] as 'm_nAccActAskTurnover', \n", + "\n", + " ['w_buy'] as 'm_nBidOrder', //委买\n", + " ['sectional_w_buy'] as 'm_nAccBidOrder', \n", + " ['w_sale'] as 'm_nAskOrder',\n", + " ['sectional_w_sale'] as 'm_nAccAskOrder',\n", + "\n", + " ['wb'] as 'm_nABOrderRate', //委比\n", + " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", + " ['lb'] as 'm_nMItemsVolRate'//量比\n", + "\n", + "\n", + " from markettable\n", + " datekey 20220903T to 20220903T+0.999 \n", + " of stock_list\n", + " end;\n", + " \n", + " return r; \n", + " \n", + "2022-11-10 10:58:41.789 | INFO | src.TSLfm:process_result_data_type:216 - No data on this day.\n", + "2022-11-10 10:58:41.790 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n", + "2022-11-10 10:58:41.790 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", + "2022-11-10 10:58:41.940 | DEBUG | src.TSLfm:get_mkt_min_k:209 - \n", + " SetSysParam(pn_cycle(), cy_1m());\n", + " stock_list := Array('AP2211', 'AP2212', 'AP2301');\n", + " r := select \n", + " ['StockID'] as 'code',\n", + " DateTimeToStr(['date']) as 'm_nDatetime',\n", + "\n", + " ['price'] as 'm_nPrice',\n", + " ['open'] as 'm_nOpen',\n", + " ['high'] as 'm_nHigh',\n", + " ['low'] as 'm_nLow',\n", + " ['close'] as 'm_nClose',\n", + "\n", + " ['sectional_high'] as 'm_nAccHigh',\n", + " ['sectional_low'] as 'm_nAccLow',\n", + "\n", + " ['vol'] as 'm_iVolume', //成交量\n", + " ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量\n", + "\n", + " ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量\n", + " ['sectional_cjbs'] as 'm_nAccMatchItems',\n", + "\n", + " ['amount'] as 'm_iTurnover', //成交金额\n", + " ['sectional_amount'] as 'm_iAccTurnover', \n", + "\n", + " ['yclose'] as 'm_nPreClose', //上一周期的收盘价\n", + " ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘\n", + " \n", + " ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?\n", + " ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量\n", + " ['sale1'] as 'm_nAskPrice', \n", + " ['sc1'] as 'm_nAskVolume', \n", + "\n", + " ['zmm'] as 'm_iABFlag', //买卖标识\n", + "\n", + " ['buy_vol'] as 'm_nActBidVolume', //主买量\n", + " ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量\n", + " ['buy_amount'] as 'm_nActBidTurnover', //主买金额\n", + " ['sectional_buy_amount'] as 'm_nAccActBidTurnover', \n", + "\n", + " ['sale_vol'] as 'm_nActAskVolume', \n", + " ['sectional_sale_vol'] as 'm_nAccActAskVolume', \n", + " ['sale_amount'] as 'm_nActAskTurnover',\n", + " ['sectional_sale_amount'] as 'm_nAccActAskTurnover', \n", + "\n", + " ['w_buy'] as 'm_nBidOrder', //委买\n", + " ['sectional_w_buy'] as 'm_nAccBidOrder', \n", + " ['w_sale'] as 'm_nAskOrder',\n", + " ['sectional_w_sale'] as 'm_nAccAskOrder',\n", + "\n", + " ['wb'] as 'm_nABOrderRate', //委比\n", + " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", + " ['lb'] as 'm_nMItemsVolRate'//量比\n", + "\n", + "\n", + " from markettable\n", + " datekey 20221102T to 20221102T+0.999 \n", + " of stock_list\n", + " end;\n", + " \n", + " return r; \n", + " \n", + "2022-11-10 10:58:42.352 | INFO | src.TSLfm:process_result_data_type:218 - Processing new df of shape (675, 37), which looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 AP2211 2022-11-02 09:01:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "1 AP2211 2022-11-02 09:02:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "2 AP2211 2022-11-02 09:03:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "3 AP2211 2022-11-02 09:04:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "4 AP2211 2022-11-02 09:05:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nAccActAskVolume \\\n", + "0 8860.0 8860.0 0.0 ... 0.0 \n", + "1 8860.0 8860.0 0.0 ... 0.0 \n", + "2 8860.0 8860.0 0.0 ... 0.0 \n", + "3 8860.0 8860.0 0.0 ... 0.0 \n", + "4 8860.0 8860.0 0.0 ... 0.0 \n", + "\n", + " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", + "0 0.0 0.0 1.0 1.0 \n", + "1 0.0 0.0 5.0 6.0 \n", + "2 0.0 0.0 0.0 6.0 \n", + "3 0.0 0.0 0.0 6.0 \n", + "4 0.0 0.0 0.0 6.0 \n", + "\n", + " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", + "0 1.0 1.0 1.0 1.0 \n", + "1 5.0 6.0 1.0 1.0 \n", + "2 0.0 6.0 1.0 1.0 \n", + "3 0.0 6.0 1.0 1.0 \n", + "4 0.0 6.0 1.0 1.0 \n", + "\n", + " m_nMItemsVolRate \n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "\n", + "[5 rows x 37 columns]\n", + "2022-11-10 10:58:42.358 | INFO | src.TSLfm:process_result_data_type:232 - Processing done, new df looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 AP2211 2022-11-02 09:01:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "1 AP2211 2022-11-02 09:02:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "2 AP2211 2022-11-02 09:03:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "3 AP2211 2022-11-02 09:04:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "4 AP2211 2022-11-02 09:05:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", + "0 8860.0 8860.0 0 ... 0.0 \n", + "1 8860.0 8860.0 0 ... 0.0 \n", + "2 8860.0 8860.0 0 ... 0.0 \n", + "3 8860.0 8860.0 0 ... 0.0 \n", + "4 8860.0 8860.0 0 ... 0.0 \n", + "\n", + " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + "0 0.0 1.0 1.0 1.0 \n", + "1 0.0 5.0 6.0 5.0 \n", + "2 0.0 0.0 6.0 0.0 \n", + "3 0.0 0.0 6.0 0.0 \n", + "4 0.0 0.0 6.0 0.0 \n", + "\n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + "0 1.0 1.0 1.0 0.0 \n", + "1 6.0 1.0 1.0 0.0 \n", + "2 6.0 1.0 1.0 0.0 \n", + "3 6.0 1.0 1.0 0.0 \n", + "4 6.0 1.0 1.0 0.0 \n", + "\n", + " code_init \n", + "0 AP \n", + "1 AP \n", + "2 AP \n", + "3 AP \n", + "4 AP \n", + "\n", + "[5 rows x 38 columns]\n", + "2022-11-10 10:58:42.359 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n" + ] + } + ], + "source": [ + "from src.TSLfm import TSLfm\n", + "# ddbfm = DDBfm('dev')\n", + "df_list = []\n", + "for date in ['20220902','20220903','20221102']:\n", + " with TSLfm() as tsl:\n", + " df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,['AP2211','AP2212','AP2301']))\n", + " df_list.append(df)\n", + " # logger.info(f'Getting a df of {df.shape}: {code_list[0][:-4]} on {date}')\n", + " # ddbfm.append_hft_table(ddb.ddf_hft_mink_tbname,df)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[ code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + " 0 AP2211 2022-09-02 09:01:00 8950.0 8966.0 8970.0 8950.0 8950.0 \n", + " 1 AP2211 2022-09-02 09:02:00 8918.0 8950.0 8950.0 8917.0 8918.0 \n", + " 2 AP2211 2022-09-02 09:03:00 8912.0 8914.0 8914.0 8903.0 8912.0 \n", + " 3 AP2211 2022-09-02 09:04:00 8920.0 8924.0 8924.0 8912.0 8920.0 \n", + " 4 AP2211 2022-09-02 09:05:00 8908.0 8920.0 8920.0 8908.0 8908.0 \n", + " .. ... ... ... ... ... ... ... \n", + " 670 AP2301 2022-09-02 14:56:00 9193.0 9194.0 9198.0 9190.0 9193.0 \n", + " 671 AP2301 2022-09-02 14:57:00 9204.0 9192.0 9204.0 9192.0 9204.0 \n", + " 672 AP2301 2022-09-02 14:58:00 9210.0 9202.0 9210.0 9200.0 9210.0 \n", + " 673 AP2301 2022-09-02 14:59:00 9209.0 9206.0 9213.0 9206.0 9209.0 \n", + " 674 AP2301 2022-09-02 15:00:00 9211.0 9210.0 9215.0 9207.0 9211.0 \n", + " \n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", + " 0 8970.0 8950.0 3 ... 0.0 \n", + " 1 8970.0 8917.0 8 ... 53522.0 \n", + " 2 8970.0 8903.0 20 ... 98069.0 \n", + " 3 8970.0 8903.0 17 ... 106951.0 \n", + " 4 8970.0 8903.0 3 ... 8918.0 \n", + " .. ... ... ... ... ... \n", + " 670 9220.0 9080.0 368 ... 1154034.0 \n", + " 671 9220.0 9080.0 568 ... 952568.0 \n", + " 672 9220.0 9080.0 707 ... 1951080.0 \n", + " 673 9220.0 9080.0 763 ... 3968840.0 \n", + " 674 9220.0 9080.0 918 ... 3979399.0 \n", + " \n", + " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + " 0 0.0 65.0 65.0 61.0 \n", + " 1 53522.0 65.0 130.0 57.0 \n", + " 2 151591.0 77.0 207.0 64.0 \n", + " 3 258542.0 64.0 271.0 63.0 \n", + " 4 267460.0 67.0 338.0 49.0 \n", + " .. ... ... ... ... \n", + " 670 223212226.0 739.0 106767.0 873.0 \n", + " 671 224164794.0 1250.0 108017.0 3668.0 \n", + " 672 226115874.0 631.0 108648.0 1723.0 \n", + " 673 230084714.0 2361.0 111009.0 1079.0 \n", + " 674 234064113.0 1555.0 112564.0 1902.0 \n", + " \n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + " 0 61.0 1.000000 1.065574 0.0 \n", + " 1 118.0 1.000000 1.101695 0.0 \n", + " 2 182.0 0.500000 1.137363 0.0 \n", + " 3 245.0 1.000000 1.106122 0.0 \n", + " 4 294.0 1.000000 1.149660 0.0 \n", + " .. ... ... ... ... \n", + " 670 173656.0 0.100000 0.614819 0.0 \n", + " 671 177324.0 5.000000 0.609150 0.0 \n", + " 672 179047.0 3.000000 0.606813 0.0 \n", + " 673 180126.0 2.666667 0.616285 0.0 \n", + " 674 182028.0 0.271605 0.618388 0.0 \n", + " \n", + " code_init \n", + " 0 AP \n", + " 1 AP \n", + " 2 AP \n", + " 3 AP \n", + " 4 AP \n", + " .. ... \n", + " 670 AP \n", + " 671 AP \n", + " 672 AP \n", + " 673 AP \n", + " 674 AP \n", + " \n", + " [675 rows x 38 columns],\n", + " Empty DataFrame\n", + " Columns: []\n", + " Index: [],\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + " 0 AP2211 2022-11-02 09:01:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + " 1 AP2211 2022-11-02 09:02:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + " 2 AP2211 2022-11-02 09:03:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + " 3 AP2211 2022-11-02 09:04:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + " 4 AP2211 2022-11-02 09:05:00 8860.0 8860.0 8860.0 8860.0 8860.0 \n", + " .. ... ... ... ... ... ... ... \n", + " 670 AP2301 2022-11-02 14:56:00 8264.0 8260.0 8269.0 8258.0 8264.0 \n", + " 671 AP2301 2022-11-02 14:57:00 8271.0 8264.0 8275.0 8263.0 8271.0 \n", + " 672 AP2301 2022-11-02 14:58:00 8269.0 8272.0 8276.0 8268.0 8269.0 \n", + " 673 AP2301 2022-11-02 14:59:00 8269.0 8270.0 8277.0 8269.0 8269.0 \n", + " 674 AP2301 2022-11-02 15:00:00 8265.0 8268.0 8270.0 8260.0 8265.0 \n", + " \n", + " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", + " 0 8860.0 8860.0 0 ... 0.0 \n", + " 1 8860.0 8860.0 0 ... 0.0 \n", + " 2 8860.0 8860.0 0 ... 0.0 \n", + " 3 8860.0 8860.0 0 ... 0.0 \n", + " 4 8860.0 8860.0 0 ... 0.0 \n", + " .. ... ... ... ... ... \n", + " 670 8325.0 8190.0 523 ... 2124105.0 \n", + " 671 8325.0 8190.0 998 ... 3355590.0 \n", + " 672 8325.0 8190.0 435 ... 1603410.0 \n", + " 673 8325.0 8190.0 942 ... 3190290.0 \n", + " 674 8325.0 8190.0 1626 ... 6190485.0 \n", + " \n", + " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + " 0 0.0 1.0 1.0 1.0 \n", + " 1 0.0 5.0 6.0 5.0 \n", + " 2 0.0 0.0 6.0 0.0 \n", + " 3 0.0 0.0 6.0 0.0 \n", + " 4 0.0 0.0 6.0 0.0 \n", + " .. ... ... ... ... \n", + " 670 682298903.0 1040.0 285907.0 1062.0 \n", + " 671 685654493.0 1512.0 287419.0 2259.0 \n", + " 672 687257903.0 696.0 288115.0 1567.0 \n", + " 673 690448193.0 2008.0 290123.0 2516.0 \n", + " 674 696638678.0 2027.0 292150.0 2496.0 \n", + " \n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + " 0 1.0 1.000000 1.000000 0.0 \n", + " 1 6.0 1.000000 1.000000 0.0 \n", + " 2 6.0 1.000000 1.000000 0.0 \n", + " 3 6.0 1.000000 1.000000 0.0 \n", + " 4 6.0 1.000000 1.000000 0.0 \n", + " .. ... ... ... ... \n", + " 670 278256.0 4.600000 1.027496 0.0 \n", + " 671 280515.0 9.000000 1.024612 0.0 \n", + " 672 282082.0 6.666667 1.021387 0.0 \n", + " 673 284598.0 7.000000 1.019413 0.0 \n", + " 674 287094.0 1.941176 1.017611 0.0 \n", + " \n", + " code_init \n", + " 0 AP \n", + " 1 AP \n", + " 2 AP \n", + " 3 AP \n", + " 4 AP \n", + " .. ... \n", + " 670 AP \n", + " 671 AP \n", + " 672 AP \n", + " 673 AP \n", + " 674 AP \n", + " \n", + " [675 rows x 38 columns]]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.Asyn worker closed peacefully.\n", + "\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n", + "Asyn worker closed peacefully.\n" + ] + } + ], + "source": [ + "df_list\n" + ] } ], "metadata": { diff --git a/src/DDBfm.py b/src/DDBfm.py index 2b7e3b1..ab9e601 100644 --- a/src/DDBfm.py +++ b/src/DDBfm.py @@ -3,7 +3,7 @@ import dolphindb.settings as keys import numpy as np import pandas as pd from loguru import logger - +import time class DDBfm(): ddb_config_servers = { @@ -36,14 +36,41 @@ class DDBfm(): ddf_hft_mink_tbname="MinKlinePartitioned" - def __init__(self, which_server='local', **kwargs): - + def __init__(self, which_server='local', pool=False): + self.pool=None + self.sess =None self.hft_db=None self.which_server=which_server self.ddb_config = self.ddb_config_servers[self.which_server] - self.sess = ddb.session(self.ddb_config['host'], 8848) - self.sess.login(self.ddb_config['username'], self.ddb_config['password']) + if not pool: + self.sess = ddb.session(self.ddb_config['host'], 8848) + self.sess.login(self.ddb_config['username'], self.ddb_config['password']) + + else: + self.pool = ddb.DBConnectionPool(self.ddb_config['host'], 8848, 12, self.ddb_config['username'], self.ddb_config['password']) + + def clear_pool(self): + if self.pool: + while not self.pool.isFinished: + logger.info("pool not finished,sleep for 5 seconds.") + time.sleep(5) + print("pool finished") + logger.debug("pool finished") + self.pool.shutDown() + + def close_sess(self): + if self.pool: + while not self.pool.isFinished: + logger.info("pool not finished,sleep for 5 seconds.") + time.sleep(5) + print("pool finished") + logger.debug("pool finished") + self.pool.shutDown() + + if self.sess: + self.sess.clearAllCache() + self.sess.close() def drop_dbpath(self,dbPath): if self.sess.existsDatabase(dbPath): @@ -59,7 +86,7 @@ class DDBfm(): db_init = self.sess.database(dbName='db_init', partitionType=keys.VALUE, partitions=self.all_fm_init, dbPath='') months=np.array(pd.date_range(start='2000-01', end='2050-12', freq="M"), dtype="datetime64[M]") - logger.debug(months) + logger.debug(f'created months len: {len(months)}') db_date = self.sess.database('db_date', partitionType=keys.VALUE, partitions=months, dbPath='') @@ -76,7 +103,7 @@ class DDBfm(): def create_hft_table(self, tbName, df): t = self.sess.table(data=df, tableAliasName=tbName) if self.which_server == 'prd': - pt = self.hft_db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDatetime', 'code_init'],sortColumns=["code","m_nDatetime"]) + pt = self.hft_db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDatetime', 'code_init'],sortColumns=["code","m_nDatetime"],compressMethods={"m_nDatetime":"delta"}) else: pt = self.hft_db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDatetime', 'code_init']) pt.append(t) @@ -85,7 +112,14 @@ class DDBfm(): def append_hft_table(self, tbName, df): appender = ddb.tableAppender(tableName=tbName, ddbSession=self.sess,dbPath=self.ddb_hft_dbPath) appender.append(df) - logger.info(f"sucessfully append some df of {df.shape}") + logger.info(f"sucessfully append some df of {df.shape} to {tbName}") + + def append_pool_hft_table(self, tbName, df): + + appender = ddb.PartitionedTableAppender(self.ddb_hft_dbPath, tbName, 'm_nDatetime', self.pool) + appender.append(df) + logger.info(f"Appending some df of {df.shape} to {tbName}") + def search_code_date_in_tb(self,tbName,curr_date,curr_code): curr_date_formatted = curr_date[:4]+'.'+curr_date[4:6]+'.'+curr_date[6:] @@ -96,6 +130,7 @@ class DDBfm(): try: # doing this cuz there's no method to check if a table is empty lol cond=f"code=`{curr_code}, m_nDatetime.date()={curr_date_formatted}d" + cond=f"code_init=`{curr_code[:2]}, m_nDatetime.date()={curr_date_formatted}d" # print(cond) df = tb.select('*').where(cond).top(1).toDF() if df.empty or df.shape[0]==0: @@ -104,7 +139,36 @@ class DDBfm(): except: return 0 return 1 - + + def load_tb(self,tableName): + return self.sess.loadTable(dbPath=self.ddb_hft_dbPath, tableName=tableName) + + def get_missing_code_date_in_tb(self,tb,curr_date,code_list): # code list has to be like all CF + curr_date_formatted = curr_date[:4]+'.'+curr_date[4:6]+'.'+curr_date[6:] + # print('?did i split this right') + # print(curr_date_formatted) + # tb = self.sess.loadTable(dbPath=self.ddb_hft_dbPath, tableName=tbName) + logger.info(f"Quickly checking if data on {'+'.join(code_list)} {curr_date} exists...") # could do a slow checking of num of data + try: + # doing this cuz there's no method to check if a table is empty lol + # cond=f"code=`{curr_code}, m_nDatetime.date()={curr_date_formatted}d" + cond=f"code_init=`{code_list[0][:2]}, m_nDatetime.date()={curr_date_formatted}d" + # print(cond) + df = tb.select('distinct code').where(cond).toDF() + + if df.empty or df.shape[0]==0: + # print(df) + return code_list + except: + return code_list + ddb_code_list = df['distinct_code'].to_list() + print(ddb_code_list) + return_code_list=[] + for c in code_list: + if c not in ddb_code_list: + return_code_list.append(c) + return return_code_list + if __name__ == '__main__': pass \ No newline at end of file diff --git a/src/TSLfm.py b/src/TSLfm.py index f2d156a..a02869d 100644 --- a/src/TSLfm.py +++ b/src/TSLfm.py @@ -15,13 +15,13 @@ class TSLfm: self.timeout_default=100000000 def __enter__(self): - logger.info('Logging in TSL.') + logger.debug('Logging in TSL.') self.c = pyTSL.Client(TINYSOFT_USERNAME, TINYSOFT_PASSWORD, TINYSOFT_HOSTNAME, 443) self.c.login() return self def __exit__(self, exc_type, exc_val, exc_tb): - logger.info('Logging out TSL.') + logger.debug('Logging out TSL.') self.c.logout() del(self.c) @@ -215,7 +215,7 @@ class TSLfm: if df.empty: logger.info('No data on this day.') return pd.DataFrame() - logger.info(f"Processing new df of shape {df.shape}, which looks like\n{df.head(5)}") + logger.debug(f"Processing new df of shape {df.shape}, which looks like\n{df.head(5)}") # new = df["m_nDatetime"].str.split(" ", n = 1, expand = True) df["m_nDatetime"]=df["m_nDatetime"].astype('datetime64') @@ -229,7 +229,8 @@ class TSLfm: for col in ['m_iABFlag']: df[col]=df[col].astype(np.int8) - logger.info(f"Processing done, new df looks like\n{df.head(5)}") + logger.info(f"Processing result df of shape {df.shape} done") + logger.debug(f"New df looks like\n{df.head(5)}") return df @@ -238,6 +239,6 @@ if __name__ == '__main__': logger.add("../logs/{time:YYYYMMDD-HHmmss}_TSLfm.log", rotation="10 MB", compression="zip", level="INFO") with TSLfm() as tsl: - t_list=['T2212'] + t_list=['CF2211'] df = tsl.process_result_data_type(tsl.get_mkt_min_k('20221031','20221101',t_list)) print(df) diff --git a/src/data_loader.py b/src/data_loader.py index 42f0ea4..09eccd0 100644 --- a/src/data_loader.py +++ b/src/data_loader.py @@ -8,8 +8,7 @@ ROOT_DIR = abspath(join(dirname(abspath(__file__)), "..")) from loguru import logger logger.remove() logger.add(sys.stderr, level="INFO") -logger.add(ROOT_DIR+"/logs/{time:YYYYMMDD-HHmmss}"+f"_{running_which_env}.log", rotation="10 MB", compression="zip", level="INFO") - +logger.add(ROOT_DIR+"/logs/{time:YYYYMMDD-HHmmss}"+f"_{running_which_env}.log", rotation="10 MB", compression="zip", level="DEBUG") import pandas as pd @@ -37,15 +36,84 @@ def run_add_1day_code_init_minKline(date,code_list): logger.info(f'Getting a df of {df.shape}: {code_list[0][:-4]} on {date}') ddb.append_hft_table(ddb.ddf_hft_mink_tbname,df) -def run_create_db_minKline(): - date = '20221101' +# def run_pool_add_byday_code_init_minKline(date_list,code_list): +# df_list=[] +# code_list_filtered=code_list +# ddb1 = DDBfm(running_which_env) + +# tb=ddb1.load_tb(tableName=ddb1.ddf_hft_mink_tbname) +# # tb=ddb1.sess.loadTable(dbPath=ddb1.ddb_hft_dbPath, tableName=ddb1.ddf_hft_mink_tbname) +# for date in date_list: +# with TSLfm() as tsl: +# df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list)) +# if df.empty: +# continue + +# code_list_filtered = ddb1.get_missing_code_date_in_tb(tb,date,code_list) +# if len(code_list_filtered)==0: +# continue +# logger.info(f"getting {'+'.join(code_list_filtered)} on {date}") + +# df=df[df['code'].isin(code_list_filtered)] +# df_list.append(df) +# ddb1.close_sess() +# del ddb1 + +# if df_list: +# df_all = pd.concat(df_list) + +# ddb2 = DDBfm(running_which_env,pool=True) +# logger.info(f'Getting a df of {df_all.shape}: {code_list[0][:-4]} on {"+".join(date_list)}') +# ddb2.append_pool_hft_table(ddb2.ddf_hft_mink_tbname,df_all) +# ddb2.clear_pool() +# del ddb2 + + +def run_pool_add_byday_code_init_tick(date_list,code_list): + df_list=[] + code_list_filtered=code_list + for date in date_list: + + ddb1 = DDBfm(running_which_env) + code_list_filtered = ddb1.get_missing_code_date_in_tb(ddb1.ddf_hft_mink_tbname,date,code_list) + if len(code_list_filtered)==0: + continue + logger.info(f"getting {'+'.join(code_list_filtered)} on {date}") + ddb1.close_sess() + del ddb1 + + with TSLfm() as tsl: + df = tsl.process_result_data_type(tsl.get_trade_tick(date,date,code_list_filtered)) + if not df.empty: + df_list.append(df) + df_all = pd.concat(df_list) + + ddb2 = DDBfm(running_which_env,pool=True) + logger.info(f'Getting a df of {df_all.shape}: {code_list[0][:-4]} on {"+".join(date_list)}') + ddb2.append_pool_hft_table(ddb2.ddf_hft_tick_tbname,df_all) + ddb2.close_sess() + del ddb2 + + +def run_create_hft_db(date = '20221101'): + + ddb = DDBfm(running_which_env) + ddb.create_hft_database() + with TSLfm() as tsl: code_list=['T2212'] - df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list)) + df_mink = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list)) # print(df) - ddb = DDBfm(running_which_env) - ddb.create_hft_database() - ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df) + ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df_mink) + + with TSLfm() as tsl: + code_list=['T2212'] + df_tick = tsl.process_result_data_type(tsl.get_trade_tick(date,date,code_list)) + # print(df) + + ddb.create_hft_table(ddb.ddf_hft_tick_tbname,df_tick) + + def run(): @@ -60,38 +128,79 @@ def run(): # print(all_code_dict_by_init) start_date='2022-09-30' - end_date='2022-11-09' + end_date='2022-10-31' allDates = pd.date_range(start_date, end_date, freq ='D') allDates = [i.replace('-','') for i in list(allDates.astype('str'))] for date in allDates: - for code_init in all_code_dict_by_init: + for ind,code_init in enumerate(all_code_dict_by_init): + logger.info(f"Getting {code_init} (no.{ind})") code_list = all_code_dict_by_init[code_init] run_add_1day_code_init_minKline(date,code_list) - # date = '20221101' - # with TSLfm() as tsl: - # # code_list = tsl.get_code_list("国债期货") - # # code_list += tsl.get_code_list("股指期货") - # # code_list += tsl.get_code_list("上市期货") - # # code_list=sorted(list(set(code_list))) - - # # print(code_list_pickel) - # code_list=['CF2211'] - # df = tsl.process_result_data_type(tsl.get_mkt_min_k(date,date,code_list)) - # print(df) - # ddb = DDBfm('prd') - # ddb.create_hft_database() - # ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df) - # if ddb.search_code_date_in_tb(ddb.ddf_hft_mink_tbname,date,'CF2211'): - # logger.warning(f"Possible duplicates on {date} and ") +def run_pool_dates_by_code_init_n_group(typ='mink',group_amount=10,start_date='20220101',end_date='20221031'): + logger.info("Running run_pool_dates_by_group") + all_code_dict_by_init={} + for c in code_list_pickel: + init = c[:-4] + if init in all_code_dict_by_init: + all_code_dict_by_init[init].append(c) + else: + all_code_dict_by_init[init]=[c] + + # print(all_code_dict_by_init) + + + allDates = pd.date_range(start_date, end_date, freq ='D') + dates_dict_by_day={} + + for d in list(allDates.astype('str')): + group_no = int(d[-2:])%group_amount + if group_no not in dates_dict_by_day: + dates_dict_by_day[group_no] = [d.replace('-','')] + else: + dates_dict_by_day[group_no].append(d.replace('-','')) + + logger.debug(dates_dict_by_day) + + + for group_no in dates_dict_by_day: + date_list=dates_dict_by_day[group_no] + num_of_init = len(all_code_dict_by_init) + for ind,code_init in enumerate(all_code_dict_by_init): + # done: 'T','TS','TS','TF' + if code_init in ['T']: # todo filtered this ,,'TF', 'IC','IF','IH','IM' + logger.info(f"Getting {code_init} (no.{ind}/{num_of_init} of group {group_no}/{group_amount})") + code_list = all_code_dict_by_init[code_init] + if typ=='mink': + # logger.info('Running mink') + logger.error('mink by day to be fixed') + + # run_pool_add_byday_code_init_minKline(date_list,code_list) + elif typ=='tick': + logger.info('Running tick') + run_pool_add_byday_code_init_tick(date_list,code_list) + - # ddb.append_hft_table(ddb.ddf_hft_mink_tbname,df) if __name__ == '__main__': - run() - # run_create_db_minKline() \ No newline at end of file + # run() + + # run_create_hft_db() # including two tables + + import time + tic = time.perf_counter() + run_pool_dates_by_code_init_n_group(typ='tick') + # run_pool_dates_by_code_init_n_group(typ='mink',group_amount=5) + + toc = time.perf_counter() + + logger.info(f"Running used {toc - tic:0.4f} seconds") + + # all t taks Running used 588.5782 seconds for 10 months + # 600/60=10 min 12min for take code_init + # 12* 71 = 850 min / 60 = 15 hr for all code for each year \ No newline at end of file diff --git a/test/test.dos b/test/test.dos index 9579f1f..1fa3fda 100644 --- a/test/test.dos +++ b/test/test.dos @@ -7,9 +7,16 @@ loadTable("dfs://hft_stock_ts","KLinePartitioned") listTables("dfs:/hft_futuremarket_ts"); pt=loadTable("dfs://hft_futuremarket_ts","MinKlinePartitioned") -select top 40 * from pt +pt=loadTable("dfs://hft_futuremarket_ts","TickPartitioned") + +select top 40 * from pt where code=`T2212 and m_nDatetime.date() >=2022.01.01d +select top 4 * from pt where code_init=`TS and m_nDatetime.date()>=2022.01.01d +select distinct m_nDatetime.date() from pt where code=`AP2211 select count(*) from loadTable("dfs://hft_futuremarket_ts", "MinKlinePartitioned") +select count(*) from loadTable("dfs://hft_futuremarket_ts", "TickPartitioned") + +select top 40 * from pt schema(pt) n=1000000