{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# dolphindb read data\n", "\n", "last update: 2022-12-13\n", "\n", "tick数据：2020到2022-11-24日左右，minute kline有2022到2022-11-10左右" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Failed to connect to host = 192.168.1.7 port = 8848 with error code 111\n" ] }, { "ename": "RuntimeError", "evalue": " in login: Couldn't send script/function to the remote host because the connection has been closed", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn [2], line 7\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mpandas\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mpd\u001b[39;00m\n\u001b[1;32m 6\u001b[0m sess \u001b[39m=\u001b[39m ddb\u001b[39m.\u001b[39msession(\u001b[39m'\u001b[39m\u001b[39m192.168.1.7\u001b[39m\u001b[39m'\u001b[39m,\u001b[39m8848\u001b[39m)\n\u001b[0;32m----> 7\u001b[0m sess\u001b[39m.\u001b[39;49mlogin(\u001b[39m'\u001b[39;49m\u001b[39madmin\u001b[39;49m\u001b[39m'\u001b[39;49m,\u001b[39m'\u001b[39;49m\u001b[39m123456\u001b[39;49m\u001b[39m'\u001b[39;49m)\n", "File \u001b[0;32m~/.local/lib/python3.9/site-packages/dolphindb/session.py:365\u001b[0m, in \u001b[0;36msession.login\u001b[0;34m(self, userid, password, enableEncryption)\u001b[0m\n\u001b[1;32m 363\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpassword \u001b[39m=\u001b[39m password\n\u001b[1;32m 364\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39menableEncryption \u001b[39m=\u001b[39m enableEncryption\n\u001b[0;32m--> 365\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcpp\u001b[39m.\u001b[39;49mlogin(userid, password, enableEncryption)\n\u001b[1;32m 366\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 367\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmutex\u001b[39m.\u001b[39mrelease()\n", "\u001b[0;31mRuntimeError\u001b[0m: in login: Couldn't send script/function to the remote host because the connection has been closed" ] } ], "source": [ "import dolphindb as ddb\n", "import dolphindb.settings as keys\n", "import numpy as np\n", "import pandas as pd\n", "\n", "# sess = ddb.session('192.168.1.7',8848)\n", "sess = ddb.session('localhost',8848)\n", "sess.login('admin','123456')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'/hft_stock_ts': 'TransePartitioned:7:0:TransePartitioned_7; TickQueuePartitioned:6:0:TickQueuePartitioned_6; TickPartitioned:5:0:TickPartitioned_5; OrderPartitioned:4:0:OrderPartitioned_4; KLinePartitioned:3:0:KLinePartitioned_3; Calendar:2:0:Calendar_2; ',\n", " '/daily_stock_ts': 'daily_kline:2:0:daily_kline_2; hft_daily_factor:16:0:hft_daily_factor_16; idx_daily_concept:6:0:idx_daily_concept_6; idx_daily_kline:11:0:idx_daily_kline_11; ',\n", " '/hft_futuremarket_ts': 'MinKlinePartitioned:2:0:MinKlinePartitioned_2; TickPartitioned:3:0:TickPartitioned_3; DailyFutureInfoPartitioned:6:0:DailyFutureInfoPartitioned_6; '}" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sess.run('getAllDBs()')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## 读数据" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 读部分数据查看" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
code_domcode_initcode_contcode_cont1code_cont2code_cont3code_cont4m_nDate
0AP2105APAP2101AP2103AP2105AP2110AP21112021-01-01
1AP2105APAP2101AP2103AP2105AP2110AP21112021-01-02
2AP2105APAP2101AP2103AP2105AP2110AP21112021-01-03
3AP2105APAP2101AP2103AP2105AP2110AP21112021-01-04
\n", "
" ], "text/plain": [ " code_dom code_init code_cont code_cont1 code_cont2 code_cont3 code_cont4 \\\n", "0 AP2105 AP AP2101 AP2103 AP2105 AP2110 AP2111 \n", "1 AP2105 AP AP2101 AP2103 AP2105 AP2110 AP2111 \n", "2 AP2105 AP AP2101 AP2103 AP2105 AP2110 AP2111 \n", "3 AP2105 AP AP2101 AP2103 AP2105 AP2110 AP2111 \n", "\n", " m_nDate \n", "0 2021-01-01 \n", "1 2021-01-02 \n", "2 2021-01-03 \n", "3 2021-01-04 " ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = sess.run(\"\"\"select top 4 * from loadTable(\"dfs://hft_futuremarket_ts\", \"DailyFutureInfoPartitioned\")\"\"\")\n", "df" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 读少量数据" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'sess' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn [1], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m tbName\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mTickPartitioned\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m# MinKlinePartitioned\u001b[39;00m\n\u001b[1;32m 3\u001b[0m code_list\u001b[39m=\u001b[39m[\u001b[39m'\u001b[39m\u001b[39mAP2211\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m----> 4\u001b[0m df \u001b[39m=\u001b[39m sess\u001b[39m.\u001b[39mrun(\u001b[39mf\u001b[39m\u001b[39m\"\"\"\u001b[39m\u001b[39mpt=loadTable(\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mddb_hft_dbPath\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m,\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mtbName\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m);\u001b[39m\n\u001b[1;32m 5\u001b[0m \u001b[39m select * from pt where m_nDatetime.date()=2022.06.01d and code in (\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mjoin([\u001b[39m\"\u001b[39m\u001b[39m`\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m+\u001b[39mc \u001b[39mfor\u001b[39;00m c \u001b[39min\u001b[39;00m code_list])\u001b[39m}\u001b[39;00m\u001b[39m);\u001b[39m\n\u001b[1;32m 6\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39m)\n", "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined" ] } ], "source": [ "ddb_hft_dbPath=\"dfs://hft_futuremarket_ts\"\n", "tbName=\"TickPartitioned\" # MinKlinePartitioned\n", "code_list=['AP2211']\n", "df = sess.run(f\"\"\"pt=loadTable(\"{ddb_hft_dbPath}\",\"{tbName}\");\n", " select * from pt where m_nDatetime.date()=2022.06.01d and code in ({\" \".join([\"`\"+c for c in code_list])});\n", " \"\"\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(7687, 38)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codem_nDatetimem_nPricem_nOpenm_nHighm_nLowm_nClosem_nAccHighm_nAccLowm_iVolume...m_nActAskTurnoverm_nAccActAskTurnoverm_nBidOrderm_nAccBidOrderm_nAskOrderm_nAccAskOrderm_nABOrderRatem_nAccABOrderRatem_nMItemsVolRatecode_init
0AP22112022-06-01 08:55:008701.08701.08701.08701.08701.00.00.00...0.00.00.00.00.00.00.0000000.0000000.0AP
1AP22112022-06-01 08:59:008623.08623.08623.08623.08623.08623.08623.0505...2177307.52177307.52.02.01.01.02.0000002.0000000.0AP
2AP22112022-06-01 09:00:018691.08691.08691.08691.08691.08691.08623.04...0.02177307.51.03.02.03.00.5000001.0000000.0AP
3AP22112022-06-01 09:00:028691.08691.08691.08691.08691.08691.08623.00...0.02177307.51.04.01.04.01.0000001.0000000.0AP
4AP22112022-06-01 09:00:038691.08691.08691.08691.08691.08691.08623.00...0.02177307.51.05.03.07.00.3333330.7142860.0AP
\n", "

5 rows × 38 columns

\n", "
" ], "text/plain": [ " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", "0 AP2211 2022-06-01 08:55:00 8701.0 8701.0 8701.0 8701.0 8701.0 \n", "1 AP2211 2022-06-01 08:59:00 8623.0 8623.0 8623.0 8623.0 8623.0 \n", "2 AP2211 2022-06-01 09:00:01 8691.0 8691.0 8691.0 8691.0 8691.0 \n", "3 AP2211 2022-06-01 09:00:02 8691.0 8691.0 8691.0 8691.0 8691.0 \n", "4 AP2211 2022-06-01 09:00:03 8691.0 8691.0 8691.0 8691.0 8691.0 \n", "\n", " m_nAccHigh m_nAccLow m_iVolume ... m_nActAskTurnover \\\n", "0 0.0 0.0 0 ... 0.0 \n", "1 8623.0 8623.0 505 ... 2177307.5 \n", "2 8691.0 8623.0 4 ... 0.0 \n", "3 8691.0 8623.0 0 ... 0.0 \n", "4 8691.0 8623.0 0 ... 0.0 \n", "\n", " m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", "0 0.0 0.0 0.0 0.0 \n", "1 2177307.5 2.0 2.0 1.0 \n", "2 2177307.5 1.0 3.0 2.0 \n", "3 2177307.5 1.0 4.0 1.0 \n", "4 2177307.5 1.0 5.0 3.0 \n", "\n", " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", "0 0.0 0.000000 0.000000 0.0 \n", "1 1.0 2.000000 2.000000 0.0 \n", "2 3.0 0.500000 1.000000 0.0 \n", "3 4.0 1.000000 1.000000 0.0 \n", "4 7.0 0.333333 0.714286 0.0 \n", "\n", " code_init \n", "0 AP \n", "1 AP \n", "2 AP \n", "3 AP \n", "4 AP \n", "\n", "[5 rows x 38 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(df.shape)\n", "df.head()" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### **🌟读较多数据**" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "#### 读主力" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
code_domcode_initcode_contcode_cont1code_cont2code_cont3code_cont4m_nDate
0IC2206ICIC2205IC2206IC2209IC22122022-05-01
1IC2206ICIC2205IC2206IC2209IC22122022-05-02
2IC2206ICIC2205IC2206IC2209IC22122022-05-03
3IC2206ICIC2205IC2206IC2209IC22122022-05-04
4IC2206ICIC2205IC2206IC2209IC22122022-05-05
...........................
204IC2212ICIC2211IC2212IC2303IC23062022-11-20
205IC2212ICIC2212IC2301IC2303IC23062022-11-21
206IC2212ICIC2212IC2301IC2303IC23062022-11-22
207IC2212ICIC2212IC2301IC2303IC23062022-11-23
208IC2212ICIC2212IC2301IC2303IC23062022-11-24
\n", "

209 rows × 8 columns

\n", "
" ], "text/plain": [ " code_dom code_init code_cont code_cont1 code_cont2 code_cont3 code_cont4 \\\n", "0 IC2206 IC IC2205 IC2206 IC2209 IC2212 \n", "1 IC2206 IC IC2205 IC2206 IC2209 IC2212 \n", "2 IC2206 IC IC2205 IC2206 IC2209 IC2212 \n", "3 IC2206 IC IC2205 IC2206 IC2209 IC2212 \n", "4 IC2206 IC IC2205 IC2206 IC2209 IC2212 \n", ".. ... ... ... ... ... ... ... \n", "204 IC2212 IC IC2211 IC2212 IC2303 IC2306 \n", "205 IC2212 IC IC2212 IC2301 IC2303 IC2306 \n", "206 IC2212 IC IC2212 IC2301 IC2303 IC2306 \n", "207 IC2212 IC IC2212 IC2301 IC2303 IC2306 \n", "208 IC2212 IC IC2212 IC2301 IC2303 IC2306 \n", "\n", " m_nDate \n", "0 2022-05-01 \n", "1 2022-05-02 \n", "2 2022-05-03 \n", "3 2022-05-04 \n", "4 2022-05-05 \n", ".. ... \n", "204 2022-11-20 \n", "205 2022-11-21 \n", "206 2022-11-22 \n", "207 2022-11-23 \n", "208 2022-11-24 \n", "\n", "[209 rows x 8 columns]" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "st_d = '2022.05.01'\n", "en_d = '2022.11.24'\n", "code_init = 'IC'\n", "\n", "spt=\"\"\"select * from loadTable(\"dfs://hft_futuremarket_ts\", \"DailyFutureInfoPartitioned\") where m_nDate.date()>={st_d}d and m_nDate.date()<={en_d}d and code_init=`{code_init}\n", "\"\"\".format(st_d=st_d,en_d=en_d,code_init=code_init)\n", "\n", "block= sess.run(spt, fetchSize = 8192)\n", "\n", "total = []\n", "while block.hasNext():\n", " tem = block.read()\n", " total.append(tem)\n", "\n", "import pandas as pd\n", "df_dom=pd.concat(total)\n", "df_dom" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "#### 读tick数据" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codem_nDatetimem_nPricem_iVolumem_iAccVolumem_nMatchItemsm_nAccMatchItemsm_nBidPricem_nBidVolumem_nAskPricem_nAskVolume
0IC22102022-10-10 09:29:005730.2109109.0-76-29454.05730.0115730.21
1IC22102022-10-10 09:30:005730.025134.0-16-29470.05730.015732.010
2IC22102022-10-10 09:30:005730.030164.0-20-29490.05730.215730.82
3IC22102022-10-10 09:30:015730.626190.0-17-29507.05730.225732.07
4IC22102022-10-10 09:30:015730.229219.0-19-29526.05730.215731.41
....................................
5833IC23062022-11-23 14:59:576026.016468.00-23469.06019.016026.09
5834IC23062022-11-23 14:59:586026.006468.00-23469.06019.216026.09
5835IC23062022-11-23 14:59:596026.056473.00-23469.06019.216026.04
5836IC23062022-11-23 15:00:006019.216474.00-23469.06019.016019.29
5837IC23062022-11-23 15:25:246019.206474.00-23469.06019.016019.29
\n", "

2881230 rows × 11 columns

\n", "
" ], "text/plain": [ " code m_nDatetime m_nPrice m_iVolume m_iAccVolume \\\n", "0 IC2210 2022-10-10 09:29:00 5730.2 109 109.0 \n", "1 IC2210 2022-10-10 09:30:00 5730.0 25 134.0 \n", "2 IC2210 2022-10-10 09:30:00 5730.0 30 164.0 \n", "3 IC2210 2022-10-10 09:30:01 5730.6 26 190.0 \n", "4 IC2210 2022-10-10 09:30:01 5730.2 29 219.0 \n", "... ... ... ... ... ... \n", "5833 IC2306 2022-11-23 14:59:57 6026.0 1 6468.0 \n", "5834 IC2306 2022-11-23 14:59:58 6026.0 0 6468.0 \n", "5835 IC2306 2022-11-23 14:59:59 6026.0 5 6473.0 \n", "5836 IC2306 2022-11-23 15:00:00 6019.2 1 6474.0 \n", "5837 IC2306 2022-11-23 15:25:24 6019.2 0 6474.0 \n", "\n", " m_nMatchItems m_nAccMatchItems m_nBidPrice m_nBidVolume m_nAskPrice \\\n", "0 -76 -29454.0 5730.0 11 5730.2 \n", "1 -16 -29470.0 5730.0 1 5732.0 \n", "2 -20 -29490.0 5730.2 1 5730.8 \n", "3 -17 -29507.0 5730.2 2 5732.0 \n", "4 -19 -29526.0 5730.2 1 5731.4 \n", "... ... ... ... ... ... \n", "5833 0 -23469.0 6019.0 1 6026.0 \n", "5834 0 -23469.0 6019.2 1 6026.0 \n", "5835 0 -23469.0 6019.2 1 6026.0 \n", "5836 0 -23469.0 6019.0 1 6019.2 \n", "5837 0 -23469.0 6019.0 1 6019.2 \n", "\n", " m_nAskVolume \n", "0 1 \n", "1 10 \n", "2 2 \n", "3 7 \n", "4 1 \n", "... ... \n", "5833 9 \n", "5834 9 \n", "5835 4 \n", "5836 9 \n", "5837 9 \n", "\n", "[2881230 rows x 11 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "st_d = '2022.10.01'\n", "en_d = '2022.11.24'\n", "code_init = 'IC'\n", "\n", "spt=\"\"\"pt=loadTable(\"dfs://hft_futuremarket_ts\",\"TickPartitioned\")\n", "select code,m_nDatetime,m_nPrice,m_iVolume,m_iAccVolume,m_nMatchItems,m_nAccMatchItems,m_nBidPrice,m_nBidVolume,m_nAskPrice,m_nAskVolume from pt where m_nDatetime.date()>={st_d}d and m_nDatetime.date()<={en_d}d and code_init=`{code_init}\n", "\"\"\".format(st_d=st_d,en_d=en_d,code_init=code_init)\n", "\n", "block= sess.run(spt, fetchSize = 8192)\n", "\n", "total = []\n", "while block.hasNext():\n", " tem = block.read()\n", " total.append(tem)\n", "\n", "import pandas as pd\n", "df_ddb=pd.concat(total)\n", "df_ddb" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## 表结构和字段解释\n", "\n", "### DailyFutureInfoPartitioned\n" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'partitionType': array([1, 1], dtype=int32),\n", " 'partitionColumnType': array([ 7, 17], dtype=int32),\n", " 'partitionColumnIndex': array([7, 1], dtype=int32),\n", " 'chunkPath': None,\n", " 'colDefs': name typeString typeInt comment\n", " 0 code_dom STRING 18 \n", " 1 code_init STRING 18 \n", " 2 code_cont STRING 18 \n", " 3 code_cont1 STRING 18 \n", " 4 code_cont2 STRING 18 \n", " 5 code_cont3 STRING 18 \n", " 6 code_cont4 STRING 18 \n", " 7 m_nDate NANOTIMESTAMP 14 ,\n", " 'chunkGranularity': 'TABLE',\n", " 'partitionTypeName': array(['VALUE', 'VALUE'], dtype=object),\n", " 'keepDuplicates': 'ALL',\n", " 'engineType': 'TSDB',\n", " 'partitionColumnName': array(['m_nDate', 'code_init'], dtype=object),\n", " 'partitionSchema': [array(['2050-11', '2050-10', '2050-09', '2050-08', '2050-07', '2050-06',\n", " '2050-05', '2050-04', '2050-03', '2050-02', '2050-01', '2049-12',\n", " '2049-11', '2049-10', '2049-09', '2049-08', '2049-07', '2049-06',\n", " '2049-05', '2049-04', '2049-03', '2049-02', '2049-01', '2048-12',\n", " '2048-11', '2048-10', '2048-09', '2048-08', '2048-07', '2048-06',\n", " '2048-05', '2048-04', '2048-03', '2048-02', '2048-01', '2047-12',\n", " '2047-11', '2047-10', '2047-09', '2047-08', '2047-07', '2047-06',\n", " '2047-05', '2047-04', '2047-03', '2047-02', '2047-01', '2046-12',\n", " '2046-11', '2046-10', '2046-09', '2046-08', '2046-07', '2046-06',\n", " '2046-05', '2046-04', '2046-03', '2046-02', '2046-01', '2045-12',\n", " '2045-11', '2045-10', '2045-09', '2045-08', '2045-07', '2045-06',\n", " '2045-05', '2045-04', '2045-03', '2045-02', '2045-01', '2044-12',\n", " '2044-11', '2044-10', '2044-09', '2044-08', '2044-07', '2044-06',\n", " '2044-05', '2044-04', '2044-03', '2044-02', '2044-01', '2043-12',\n", " '2043-11', '2043-10', '2043-09', '2043-08', '2043-07', '2043-06',\n", " '2043-05', '2043-04', '2043-03', '2043-02', '2043-01', '2042-12',\n", " '2042-11', '2042-10', '2042-09', '2042-08', '2042-07', '2042-06',\n", " '2042-05', '2042-04', '2042-03', '2042-02', '2042-01', '2041-12',\n", " '2041-11', '2041-10', '2041-09', '2041-08', '2041-07', '2041-06',\n", " '2041-05', '2041-04', '2041-03', '2041-02', '2041-01', '2040-12',\n", " '2040-11', '2040-10', '2040-09', '2040-08', '2040-07', '2040-06',\n", " '2040-05', '2040-04', '2040-03', '2040-02', '2040-01', '2039-12',\n", " '2039-11', '2039-10', '2039-09', '2039-08', '2039-07', '2039-06',\n", " '2039-05', '2039-04', '2039-03', '2039-02', '2039-01', '2038-12',\n", " '2038-11', '2038-10', '2038-09', '2038-08', '2038-07', '2038-06',\n", " '2038-05', '2038-04', '2038-03', '2038-02', '2038-01', '2037-12',\n", " '2037-11', '2037-10', '2037-09', '2037-08', '2037-07', '2037-06',\n", " '2037-05', '2037-04', '2037-03', '2037-01', '2036-12', '2036-11',\n", " '2036-10', '2036-09', '2036-08', '2036-07', '2036-06', '2036-05',\n", " '2036-04', '2036-03', '2036-02', '2036-01', '2035-12', '2035-11',\n", " '2035-10', '2035-09', '2035-08', '2035-07', '2035-06', '2035-05',\n", " '2035-04', '2035-03', '2035-02', '2035-01', '2034-12', '2034-11',\n", " '2034-10', '2034-09', '2034-08', '2034-07', '2034-06', '2034-05',\n", " '2034-04', '2034-03', '2034-02', '2034-01', '2033-12', '2033-11',\n", " '2033-10', '2033-09', '2033-08', '2033-07', '2033-06', '2033-05',\n", " '2033-04', '2033-03', '2033-02', '2033-01', '2032-12', '2032-11',\n", " '2032-10', '2032-09', '2032-08', '2032-07', '2032-06', '2032-05',\n", " '2032-04', '2032-03', '2032-02', '2032-01', '2031-12', '2031-11',\n", " '2031-10', '2031-09', '2031-08', '2031-07', '2031-06', '2031-05',\n", " '2031-04', '2031-03', '2031-02', '2031-01', '2030-12', '2030-11',\n", " '2030-10', '2030-09', '2030-08', '2030-07', '2030-06', '2030-05',\n", " '2030-04', '2030-03', '2030-02', '2030-01', '2029-12', '2029-11',\n", " '2029-10', '2029-09', '2029-08', '2029-07', '2029-06', '2029-05',\n", " '2029-04', '2029-03', '2029-02', '2029-01', '2028-12', '2028-11',\n", " '2028-10', '2028-09', '2028-08', '2028-07', '2028-06', '2028-05',\n", " '2028-04', '2028-03', '2028-02', '2028-01', '2027-12', '2027-11',\n", " '2027-10', '2027-09', '2027-08', '2027-07', '2027-06', '2027-05',\n", " '2027-04', '2027-03', '2027-02', '2027-01', '2026-12', '2026-11',\n", " '2026-10', '2026-09', '2026-08', '2026-07', '2026-06', '2026-05',\n", " '2026-04', '2026-03', '2026-02', '2026-01', '2025-12', '2025-11',\n", " '2025-10', '2025-09', '2025-08', '2025-07', '2025-06', '2025-05',\n", " '2025-04', '2025-03', '2025-02', '2025-01', '2024-12', '2024-11',\n", " '2024-10', '2024-09', '2024-08', '2024-07', '2024-06', '2024-05',\n", " '2024-04', '2024-03', '2024-02', '2024-01', '2023-12', '2023-11',\n", " '2023-10', '2023-09', '2023-08', '2023-07', '2023-06', '2023-05',\n", " '2023-04', '2023-03', '2023-02', '2023-01', '2022-12', '2022-11',\n", " '2022-10', '2022-09', '2022-08', '2022-07', '2022-06', '2022-05',\n", " '2022-04', '2022-03', '2022-02', '2022-01', '2021-12', '2021-11',\n", " '2021-10', '2021-09', '2021-08', '2021-07', '2021-06', '2021-05',\n", " '2021-04', '2021-03', '2021-02', '2021-01', '2020-12', '2020-11',\n", " '2020-10', '2020-09', '2020-08', '2020-07', '2020-06', '2020-05',\n", " '2020-04', '2020-03', '2020-02', '2020-01', '2019-12', '2019-11',\n", " '2019-10', '2019-09', '2019-08', '2019-07', '2019-06', '2019-05',\n", " '2019-04', '2019-03', '2019-02', '2019-01', '2018-12', '2018-11',\n", " '2018-10', '2018-09', '2018-08', '2018-07', '2018-06', '2018-05',\n", " '2018-04', '2018-03', '2018-02', '2018-01', '2017-12', '2017-11',\n", " '2017-10', '2017-09', '2017-08', '2017-07', '2017-06', '2017-05',\n", " '2017-04', '2017-03', '2017-02', '2017-01', '2016-12', '2016-11',\n", " '2016-10', '2016-09', '2016-08', '2016-07', '2016-06', '2016-05',\n", " '2016-04', '2016-03', '2016-02', '2016-01', '2015-12', '2015-11',\n", " '2015-10', '2015-09', '2015-08', '2015-07', '2015-06', '2015-05',\n", " '2015-04', '2015-03', '2015-02', '2015-01', '2014-12', '2014-11',\n", " '2014-10', '2014-09', '2014-08', '2014-07', '2014-06', '2014-05',\n", " '2014-04', '2014-03', '2014-02', '2014-01', '2013-12', '2013-11',\n", " '2013-10', '2013-09', '2013-08', '2013-07', '2013-06', '2013-05',\n", " '2013-04', '2013-03', '2013-02', '2013-01', '2012-12', '2012-11',\n", " '2012-10', '2012-09', '2012-08', '2012-07', '2012-06', '2012-05',\n", " '2012-04', '2012-03', '2012-02', '2012-01', '2011-12', '2011-11',\n", " '2011-10', '2011-09', '2011-08', '2011-07', '2011-06', '2011-05',\n", " '2011-04', '2011-03', '2011-02', '2011-01', '2010-12', '2010-11',\n", " '2010-10', '2010-09', '2010-08', '2010-07', '2010-06', '2010-05',\n", " '2010-04', '2010-03', '2010-02', '2010-01', '2009-12', '2009-11',\n", " '2009-10', '2009-09', '2009-08', '2009-07', '2009-06', '2009-05',\n", " '2009-04', '2009-03', '2009-02', '2009-01', '2008-12', '2008-11',\n", " '2008-10', '2008-09', '2008-08', '2008-07', '2008-06', '2008-05',\n", " '2008-04', '2008-03', '2008-02', '2008-01', '2007-12', '2007-11',\n", " '2007-10', '2007-09', '2007-08', '2007-07', '2007-06', '2007-05',\n", " '2007-04', '2007-03', '2007-02', '2007-01', '2006-12', '2006-11',\n", " '2006-10', '2006-09', '2006-08', '2006-07', '2006-06', '2006-05',\n", " '2006-04', '2006-03', '2006-02', '2006-01', '2005-12', '2005-11',\n", " '2005-10', '2005-09', '2005-08', '2005-07', '2005-06', '2005-05',\n", " '2005-04', '2005-03', '2005-02', '2005-01', '2004-12', '2004-11',\n", " '2004-10', '2004-09', '2004-08', '2004-07', '2004-06', '2004-05',\n", " '2004-04', '2004-03', '2004-02', '2004-01', '2003-12', '2003-11',\n", " '2003-10', '2003-09', '2003-08', '2003-07', '2003-06', '2003-05',\n", " '2003-04', '2003-03', '2003-02', '2003-01', '2002-12', '2002-11',\n", " '2002-10', '2002-09', '2002-08', '2002-07', '2002-06', '2002-05',\n", " '2002-04', '2002-03', '2002-02', '2002-01', '2001-12', '2001-11',\n", " '2001-10', '2001-09', '2001-08', '2001-07', '2001-06', '2001-05',\n", " '2001-04', '2001-03', '2001-02', '2001-01', '2000-12', '2000-11',\n", " '2000-10', '2000-09', '2000-08', '2000-07', '2000-06', '2000-05',\n", " '2000-04', '2000-03', '2000-02', '2000-01', '2037-02', '1899-12'],\n", " dtype='datetime64[M]'),\n", " array(['zn', 'y', 'wr', 'v', 'ss', 'sp', 'sn', 'sc', 'ru', 'rr', 'rb',\n", " 'pp', 'pg', 'pb', 'p', 'nr', 'ni', 'm', 'lu', 'lh', 'l', 'jm',\n", " 'jd', 'j', 'i', 'hc', 'fu', 'fb', 'eg', 'eb', 'cu', 'cs', 'c',\n", " 'bu', 'bc', 'bb', 'b', 'au', 'al', 'ag', 'a', 'ZC', 'WH', 'UR',\n", " 'TS', 'TF', 'TA', 'T', 'SR', 'SM', 'SF', 'SA', 'RS', 'RM', 'RI',\n", " 'PM', 'PK', 'PF', 'OI', 'MA', 'LR', 'JR', 'IM', 'IH', 'IF', 'IC',\n", " 'FG', 'CY', 'CJ', 'CF', 'AP'], dtype=object)],\n", " 'sortColumns': array(['code_init', 'm_nDate'], dtype=object),\n", " 'partitionSites': None}" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sess.run('loadTable(\"dfs://hft_futuremarket_ts\",\"DailyFutureInfoPartitioned\").schema()')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### MinKlinePartitioned" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'partitionType': array([1, 1], dtype=int32),\n", " 'partitionColumnType': array([ 7, 17], dtype=int32),\n", " 'partitionColumnIndex': array([ 1, 37], dtype=int32),\n", " 'chunkPath': None,\n", " 'colDefs': name typeString typeInt comment\n", " 0 code STRING 18 \n", " 1 m_nDatetime NANOTIMESTAMP 14 \n", " 2 m_nPrice DOUBLE 16 \n", " 3 m_nOpen DOUBLE 16 \n", " 4 m_nHigh DOUBLE 16 \n", " 5 m_nLow DOUBLE 16 \n", " 6 m_nClose DOUBLE 16 \n", " 7 m_nAccHigh DOUBLE 16 \n", " 8 m_nAccLow DOUBLE 16 \n", " 9 m_iVolume SHORT 3 \n", " 10 m_iAccVolume SHORT 3 \n", " 11 m_nMatchItems SHORT 3 \n", " 12 m_nAccMatchItems SHORT 3 \n", " 13 m_iTurnover DOUBLE 16 \n", " 14 m_iAccTurnover DOUBLE 16 \n", " 15 m_nPreClose DOUBLE 16 \n", " 16 m_nAccPreClose DOUBLE 16 \n", " 17 m_nBidPrice DOUBLE 16 \n", " 18 m_nBidVolume SHORT 3 \n", " 19 m_nAskPrice DOUBLE 16 \n", " 20 m_nAskVolume SHORT 3 \n", " 21 m_iABFlag CHAR 2 \n", " 22 m_nActBidVolume SHORT 3 \n", " 23 m_nAccActBidVolume SHORT 3 \n", " 24 m_nActBidTurnover DOUBLE 16 \n", " 25 m_nAccActBidTurnover DOUBLE 16 \n", " 26 m_nActAskVolume SHORT 3 \n", " 27 m_nAccActAskVolume SHORT 3 \n", " 28 m_nActAskTurnover DOUBLE 16 \n", " 29 m_nAccActAskTurnover DOUBLE 16 \n", " 30 m_nBidOrder DOUBLE 16 \n", " 31 m_nAccBidOrder DOUBLE 16 \n", " 32 m_nAskOrder DOUBLE 16 \n", " 33 m_nAccAskOrder DOUBLE 16 \n", " 34 m_nABOrderRate DOUBLE 16 \n", " 35 m_nAccABOrderRate DOUBLE 16 \n", " 36 m_nMItemsVolRate DOUBLE 16 \n", " 37 code_init STRING 18 ,\n", " 'chunkGranularity': 'TABLE',\n", " 'partitionTypeName': array(['VALUE', 'VALUE'], dtype=object),\n", " 'keepDuplicates': 'ALL',\n", " 'engineType': 'TSDB',\n", " 'partitionColumnName': array(['m_nDatetime', 'code_init'], dtype=object),\n", " 'partitionSchema': [array(['2050-11', '2050-10', '2050-09', '2050-08', '2050-07', '2050-06',\n", " '2050-05', '2050-04', '2050-03', '2050-02', '2050-01', '2049-12',\n", " '2049-11', '2049-10', '2049-09', '2049-08', '2049-07', '2049-06',\n", " '2049-05', '2049-04', '2049-03', '2049-02', '2049-01', '2048-12',\n", " '2048-11', '2048-10', '2048-09', '2048-08', '2048-07', '2048-06',\n", " '2048-05', '2048-04', '2048-03', '2048-02', '2048-01', '2047-12',\n", " '2047-11', '2047-10', '2047-09', '2047-08', '2047-07', '2047-06',\n", " '2047-05', '2047-04', '2047-03', '2047-02', '2047-01', '2046-12',\n", " '2046-11', '2046-10', '2046-09', '2046-08', '2046-07', '2046-06',\n", " '2046-05', '2046-04', '2046-03', '2046-02', '2046-01', '2045-12',\n", " '2045-11', '2045-10', '2045-09', '2045-08', '2045-07', '2045-06',\n", " '2045-05', '2045-04', '2045-03', '2045-02', '2045-01', '2044-12',\n", " '2044-11', '2044-10', '2044-09', '2044-08', '2044-07', '2044-06',\n", " '2044-05', '2044-04', '2044-03', '2044-02', '2044-01', '2043-12',\n", " '2043-11', '2043-10', '2043-09', '2043-08', '2043-07', '2043-06',\n", " '2043-05', '2043-04', '2043-03', '2043-02', '2043-01', '2042-12',\n", " '2042-11', '2042-10', '2042-09', '2042-08', '2042-07', '2042-06',\n", " '2042-05', '2042-04', '2042-03', '2042-02', '2042-01', '2041-12',\n", " '2041-11', '2041-10', '2041-09', '2041-08', '2041-07', '2041-06',\n", " '2041-05', '2041-04', '2041-03', '2041-02', '2041-01', '2040-12',\n", " '2040-11', '2040-10', '2040-09', '2040-08', '2040-07', '2040-06',\n", " '2040-05', '2040-04', '2040-03', '2040-02', '2040-01', '2039-12',\n", " '2039-11', '2039-10', '2039-09', '2039-08', '2039-07', '2039-06',\n", " '2039-05', '2039-04', '2039-03', '2039-02', '2039-01', '2038-12',\n", " '2038-11', '2038-10', '2038-09', '2038-08', '2038-07', '2038-06',\n", " '2038-05', '2038-04', '2038-03', '2038-02', '2038-01', '2037-12',\n", " '2037-11', '2037-10', '2037-09', '2037-08', '2037-07', '2037-06',\n", " '2037-05', '2037-04', '2037-03', '2037-01', '2036-12', '2036-11',\n", " '2036-10', '2036-09', '2036-08', '2036-07', '2036-06', '2036-05',\n", " '2036-04', '2036-03', '2036-02', '2036-01', '2035-12', '2035-11',\n", " '2035-10', '2035-09', '2035-08', '2035-07', '2035-06', '2035-05',\n", " '2035-04', '2035-03', '2035-02', '2035-01', '2034-12', '2034-11',\n", " '2034-10', '2034-09', '2034-08', '2034-07', '2034-06', '2034-05',\n", " '2034-04', '2034-03', '2034-02', '2034-01', '2033-12', '2033-11',\n", " '2033-10', '2033-09', '2033-08', '2033-07', '2033-06', '2033-05',\n", " '2033-04', '2033-03', '2033-02', '2033-01', '2032-12', '2032-11',\n", " '2032-10', '2032-09', '2032-08', '2032-07', '2032-06', '2032-05',\n", " '2032-04', '2032-03', '2032-02', '2032-01', '2031-12', '2031-11',\n", " '2031-10', '2031-09', '2031-08', '2031-07', '2031-06', '2031-05',\n", " '2031-04', '2031-03', '2031-02', '2031-01', '2030-12', '2030-11',\n", " '2030-10', '2030-09', '2030-08', '2030-07', '2030-06', '2030-05',\n", " '2030-04', '2030-03', '2030-02', '2030-01', '2029-12', '2029-11',\n", " '2029-10', '2029-09', '2029-08', '2029-07', '2029-06', '2029-05',\n", " '2029-04', '2029-03', '2029-02', '2029-01', '2028-12', '2028-11',\n", " '2028-10', '2028-09', '2028-08', '2028-07', '2028-06', '2028-05',\n", " '2028-04', '2028-03', '2028-02', '2028-01', '2027-12', '2027-11',\n", " '2027-10', '2027-09', '2027-08', '2027-07', '2027-06', '2027-05',\n", " '2027-04', '2027-03', '2027-02', '2027-01', '2026-12', '2026-11',\n", " '2026-10', '2026-09', '2026-08', '2026-07', '2026-06', '2026-05',\n", " '2026-04', '2026-03', '2026-02', '2026-01', '2025-12', '2025-11',\n", " '2025-10', '2025-09', '2025-08', '2025-07', '2025-06', '2025-05',\n", " '2025-04', '2025-03', '2025-02', '2025-01', '2024-12', '2024-11',\n", " '2024-10', '2024-09', '2024-08', '2024-07', '2024-06', '2024-05',\n", " '2024-04', '2024-03', '2024-02', '2024-01', '2023-12', '2023-11',\n", " '2023-10', '2023-09', '2023-08', '2023-07', '2023-06', '2023-05',\n", " '2023-04', '2023-03', '2023-02', '2023-01', '2022-12', '2022-11',\n", " '2022-10', '2022-09', '2022-08', '2022-07', '2022-06', '2022-05',\n", " '2022-04', '2022-03', '2022-02', '2022-01', '2021-12', '2021-11',\n", " '2021-10', '2021-09', '2021-08', '2021-07', '2021-06', '2021-05',\n", " '2021-04', '2021-03', '2021-02', '2021-01', '2020-12', '2020-11',\n", " '2020-10', '2020-09', '2020-08', '2020-07', '2020-06', '2020-05',\n", " '2020-04', '2020-03', '2020-02', '2020-01', '2019-12', '2019-11',\n", " '2019-10', '2019-09', '2019-08', '2019-07', '2019-06', '2019-05',\n", " '2019-04', '2019-03', '2019-02', '2019-01', '2018-12', '2018-11',\n", " '2018-10', '2018-09', '2018-08', '2018-07', '2018-06', '2018-05',\n", " '2018-04', '2018-03', '2018-02', '2018-01', '2017-12', '2017-11',\n", " '2017-10', '2017-09', '2017-08', '2017-07', '2017-06', '2017-05',\n", " '2017-04', '2017-03', '2017-02', '2017-01', '2016-12', '2016-11',\n", " '2016-10', '2016-09', '2016-08', '2016-07', '2016-06', '2016-05',\n", " '2016-04', '2016-03', '2016-02', '2016-01', '2015-12', '2015-11',\n", " '2015-10', '2015-09', '2015-08', '2015-07', '2015-06', '2015-05',\n", " '2015-04', '2015-03', '2015-02', '2015-01', '2014-12', '2014-11',\n", " '2014-10', '2014-09', '2014-08', '2014-07', '2014-06', '2014-05',\n", " '2014-04', '2014-03', '2014-02', '2014-01', '2013-12', '2013-11',\n", " '2013-10', '2013-09', '2013-08', '2013-07', '2013-06', '2013-05',\n", " '2013-04', '2013-03', '2013-02', '2013-01', '2012-12', '2012-11',\n", " '2012-10', '2012-09', '2012-08', '2012-07', '2012-06', '2012-05',\n", " '2012-04', '2012-03', '2012-02', '2012-01', '2011-12', '2011-11',\n", " '2011-10', '2011-09', '2011-08', '2011-07', '2011-06', '2011-05',\n", " '2011-04', '2011-03', '2011-02', '2011-01', '2010-12', '2010-11',\n", " '2010-10', '2010-09', '2010-08', '2010-07', '2010-06', '2010-05',\n", " '2010-04', '2010-03', '2010-02', '2010-01', '2009-12', '2009-11',\n", " '2009-10', '2009-09', '2009-08', '2009-07', '2009-06', '2009-05',\n", " '2009-04', '2009-03', '2009-02', '2009-01', '2008-12', '2008-11',\n", " '2008-10', '2008-09', '2008-08', '2008-07', '2008-06', '2008-05',\n", " '2008-04', '2008-03', '2008-02', '2008-01', '2007-12', '2007-11',\n", " '2007-10', '2007-09', '2007-08', '2007-07', '2007-06', '2007-05',\n", " '2007-04', '2007-03', '2007-02', '2007-01', '2006-12', '2006-11',\n", " '2006-10', '2006-09', '2006-08', '2006-07', '2006-06', '2006-05',\n", " '2006-04', '2006-03', '2006-02', '2006-01', '2005-12', '2005-11',\n", " '2005-10', '2005-09', '2005-08', '2005-07', '2005-06', '2005-05',\n", " '2005-04', '2005-03', '2005-02', '2005-01', '2004-12', '2004-11',\n", " '2004-10', '2004-09', '2004-08', '2004-07', '2004-06', '2004-05',\n", " '2004-04', '2004-03', '2004-02', '2004-01', '2003-12', '2003-11',\n", " '2003-10', '2003-09', '2003-08', '2003-07', '2003-06', '2003-05',\n", " '2003-04', '2003-03', '2003-02', '2003-01', '2002-12', '2002-11',\n", " '2002-10', '2002-09', '2002-08', '2002-07', '2002-06', '2002-05',\n", " '2002-04', '2002-03', '2002-02', '2002-01', '2001-12', '2001-11',\n", " '2001-10', '2001-09', '2001-08', '2001-07', '2001-06', '2001-05',\n", " '2001-04', '2001-03', '2001-02', '2001-01', '2000-12', '2000-11',\n", " '2000-10', '2000-09', '2000-08', '2000-07', '2000-06', '2000-05',\n", " '2000-04', '2000-03', '2000-02', '2000-01', '2037-02', '1899-12'],\n", " dtype='datetime64[M]'),\n", " array(['zn', 'y', 'wr', 'v', 'ss', 'sp', 'sn', 'sc', 'ru', 'rr', 'rb',\n", " 'pp', 'pg', 'pb', 'p', 'nr', 'ni', 'm', 'lu', 'lh', 'l', 'jm',\n", " 'jd', 'j', 'i', 'hc', 'fu', 'fb', 'eg', 'eb', 'cu', 'cs', 'c',\n", " 'bu', 'bc', 'bb', 'b', 'au', 'al', 'ag', 'a', 'ZC', 'WH', 'UR',\n", " 'TS', 'TF', 'TA', 'T', 'SR', 'SM', 'SF', 'SA', 'RS', 'RM', 'RI',\n", " 'PM', 'PK', 'PF', 'OI', 'MA', 'LR', 'JR', 'IM', 'IH', 'IF', 'IC',\n", " 'FG', 'CY', 'CJ', 'CF', 'AP'], dtype=object)],\n", " 'sortColumns': array(['code', 'm_nDatetime'], dtype=object),\n", " 'partitionSites': None}" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sess.run('loadTable(\"dfs://hft_futuremarket_ts\",\"MinKlinePartitioned\").schema()')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### TickPartitioned" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'partitionType': array([1, 1], dtype=int32),\n", " 'partitionColumnType': array([ 7, 17], dtype=int32),\n", " 'partitionColumnIndex': array([ 1, 37], dtype=int32),\n", " 'chunkPath': None,\n", " 'colDefs': name typeString typeInt comment\n", " 0 code STRING 18 \n", " 1 m_nDatetime NANOTIMESTAMP 14 \n", " 2 m_nPrice DOUBLE 16 \n", " 3 m_nOpen DOUBLE 16 \n", " 4 m_nHigh DOUBLE 16 \n", " 5 m_nLow DOUBLE 16 \n", " 6 m_nClose DOUBLE 16 \n", " 7 m_nAccHigh DOUBLE 16 \n", " 8 m_nAccLow DOUBLE 16 \n", " 9 m_iVolume SHORT 3 \n", " 10 m_iAccVolume SHORT 3 \n", " 11 m_nMatchItems SHORT 3 \n", " 12 m_nAccMatchItems SHORT 3 \n", " 13 m_iTurnover DOUBLE 16 \n", " 14 m_iAccTurnover DOUBLE 16 \n", " 15 m_nPreClose DOUBLE 16 \n", " 16 m_nAccPreClose DOUBLE 16 \n", " 17 m_nBidPrice DOUBLE 16 \n", " 18 m_nBidVolume SHORT 3 \n", " 19 m_nAskPrice DOUBLE 16 \n", " 20 m_nAskVolume SHORT 3 \n", " 21 m_iABFlag CHAR 2 \n", " 22 m_nActBidVolume SHORT 3 \n", " 23 m_nAccActBidVolume SHORT 3 \n", " 24 m_nActBidTurnover DOUBLE 16 \n", " 25 m_nAccActBidTurnover DOUBLE 16 \n", " 26 m_nActAskVolume SHORT 3 \n", " 27 m_nAccActAskVolume SHORT 3 \n", " 28 m_nActAskTurnover DOUBLE 16 \n", " 29 m_nAccActAskTurnover DOUBLE 16 \n", " 30 m_nBidOrder DOUBLE 16 \n", " 31 m_nAccBidOrder DOUBLE 16 \n", " 32 m_nAskOrder DOUBLE 16 \n", " 33 m_nAccAskOrder DOUBLE 16 \n", " 34 m_nABOrderRate DOUBLE 16 \n", " 35 m_nAccABOrderRate DOUBLE 16 \n", " 36 m_nMItemsVolRate DOUBLE 16 \n", " 37 code_init STRING 18 ,\n", " 'chunkGranularity': 'TABLE',\n", " 'partitionTypeName': array(['VALUE', 'VALUE'], dtype=object),\n", " 'keepDuplicates': 'ALL',\n", " 'engineType': 'TSDB',\n", " 'partitionColumnName': array(['m_nDatetime', 'code_init'], dtype=object),\n", " 'partitionSchema': [array(['2050-11', '2050-10', '2050-09', '2050-08', '2050-07', '2050-06',\n", " '2050-05', '2050-04', '2050-03', '2050-02', '2050-01', '2049-12',\n", " '2049-11', '2049-10', '2049-09', '2049-08', '2049-07', '2049-06',\n", " '2049-05', '2049-04', '2049-03', '2049-02', '2049-01', '2048-12',\n", " '2048-11', '2048-10', '2048-09', '2048-08', '2048-07', '2048-06',\n", " '2048-05', '2048-04', '2048-03', '2048-02', '2048-01', '2047-12',\n", " '2047-11', '2047-10', '2047-09', '2047-08', '2047-07', '2047-06',\n", " '2047-05', '2047-04', '2047-03', '2047-02', '2047-01', '2046-12',\n", " '2046-11', '2046-10', '2046-09', '2046-08', '2046-07', '2046-06',\n", " '2046-05', '2046-04', '2046-03', '2046-02', '2046-01', '2045-12',\n", " '2045-11', '2045-10', '2045-09', '2045-08', '2045-07', '2045-06',\n", " '2045-05', '2045-04', '2045-03', '2045-02', '2045-01', '2044-12',\n", " '2044-11', '2044-10', '2044-09', '2044-08', '2044-07', '2044-06',\n", " '2044-05', '2044-04', '2044-03', '2044-02', '2044-01', '2043-12',\n", " '2043-11', '2043-10', '2043-09', '2043-08', '2043-07', '2043-06',\n", " '2043-05', '2043-04', '2043-03', '2043-02', '2043-01', '2042-12',\n", " '2042-11', '2042-10', '2042-09', '2042-08', '2042-07', '2042-06',\n", " '2042-05', '2042-04', '2042-03', '2042-02', '2042-01', '2041-12',\n", " '2041-11', '2041-10', '2041-09', '2041-08', '2041-07', '2041-06',\n", " '2041-05', '2041-04', '2041-03', '2041-02', '2041-01', '2040-12',\n", " '2040-11', '2040-10', '2040-09', '2040-08', '2040-07', '2040-06',\n", " '2040-05', '2040-04', '2040-03', '2040-02', '2040-01', '2039-12',\n", " '2039-11', '2039-10', '2039-09', '2039-08', '2039-07', '2039-06',\n", " '2039-05', '2039-04', '2039-03', '2039-02', '2039-01', '2038-12',\n", " '2038-11', '2038-10', '2038-09', '2038-08', '2038-07', '2038-06',\n", " '2038-05', '2038-04', '2038-03', '2038-02', '2038-01', '2037-12',\n", " '2037-11', '2037-10', '2037-09', '2037-08', '2037-07', '2037-06',\n", " '2037-05', '2037-04', '2037-03', '2037-01', '2036-12', '2036-11',\n", " '2036-10', '2036-09', '2036-08', '2036-07', '2036-06', '2036-05',\n", " '2036-04', '2036-03', '2036-02', '2036-01', '2035-12', '2035-11',\n", " '2035-10', '2035-09', '2035-08', '2035-07', '2035-06', '2035-05',\n", " '2035-04', '2035-03', '2035-02', '2035-01', '2034-12', '2034-11',\n", " '2034-10', '2034-09', '2034-08', '2034-07', '2034-06', '2034-05',\n", " '2034-04', '2034-03', '2034-02', '2034-01', '2033-12', '2033-11',\n", " '2033-10', '2033-09', '2033-08', '2033-07', '2033-06', '2033-05',\n", " '2033-04', '2033-03', '2033-02', '2033-01', '2032-12', '2032-11',\n", " '2032-10', '2032-09', '2032-08', '2032-07', '2032-06', '2032-05',\n", " '2032-04', '2032-03', '2032-02', '2032-01', '2031-12', '2031-11',\n", " '2031-10', '2031-09', '2031-08', '2031-07', '2031-06', '2031-05',\n", " '2031-04', '2031-03', '2031-02', '2031-01', '2030-12', '2030-11',\n", " '2030-10', '2030-09', '2030-08', '2030-07', '2030-06', '2030-05',\n", " '2030-04', '2030-03', '2030-02', '2030-01', '2029-12', '2029-11',\n", " '2029-10', '2029-09', '2029-08', '2029-07', '2029-06', '2029-05',\n", " '2029-04', '2029-03', '2029-02', '2029-01', '2028-12', '2028-11',\n", " '2028-10', '2028-09', '2028-08', '2028-07', '2028-06', '2028-05',\n", " '2028-04', '2028-03', '2028-02', '2028-01', '2027-12', '2027-11',\n", " '2027-10', '2027-09', '2027-08', '2027-07', '2027-06', '2027-05',\n", " '2027-04', '2027-03', '2027-02', '2027-01', '2026-12', '2026-11',\n", " '2026-10', '2026-09', '2026-08', '2026-07', '2026-06', '2026-05',\n", " '2026-04', '2026-03', '2026-02', '2026-01', '2025-12', '2025-11',\n", " '2025-10', '2025-09', '2025-08', '2025-07', '2025-06', '2025-05',\n", " '2025-04', '2025-03', '2025-02', '2025-01', '2024-12', '2024-11',\n", " '2024-10', '2024-09', '2024-08', '2024-07', '2024-06', '2024-05',\n", " '2024-04', '2024-03', '2024-02', '2024-01', '2023-12', '2023-11',\n", " '2023-10', '2023-09', '2023-08', '2023-07', '2023-06', '2023-05',\n", " '2023-04', '2023-03', '2023-02', '2023-01', '2022-12', '2022-11',\n", " '2022-10', '2022-09', '2022-08', '2022-07', '2022-06', '2022-05',\n", " '2022-04', '2022-03', '2022-02', '2022-01', '2021-12', '2021-11',\n", " '2021-10', '2021-09', '2021-08', '2021-07', '2021-06', '2021-05',\n", " '2021-04', '2021-03', '2021-02', '2021-01', '2020-12', '2020-11',\n", " '2020-10', '2020-09', '2020-08', '2020-07', '2020-06', '2020-05',\n", " '2020-04', '2020-03', '2020-02', '2020-01', '2019-12', '2019-11',\n", " '2019-10', '2019-09', '2019-08', '2019-07', '2019-06', '2019-05',\n", " '2019-04', '2019-03', '2019-02', '2019-01', '2018-12', '2018-11',\n", " '2018-10', '2018-09', '2018-08', '2018-07', '2018-06', '2018-05',\n", " '2018-04', '2018-03', '2018-02', '2018-01', '2017-12', '2017-11',\n", " '2017-10', '2017-09', '2017-08', '2017-07', '2017-06', '2017-05',\n", " '2017-04', '2017-03', '2017-02', '2017-01', '2016-12', '2016-11',\n", " '2016-10', '2016-09', '2016-08', '2016-07', '2016-06', '2016-05',\n", " '2016-04', '2016-03', '2016-02', '2016-01', '2015-12', '2015-11',\n", " '2015-10', '2015-09', '2015-08', '2015-07', '2015-06', '2015-05',\n", " '2015-04', '2015-03', '2015-02', '2015-01', '2014-12', '2014-11',\n", " '2014-10', '2014-09', '2014-08', '2014-07', '2014-06', '2014-05',\n", " '2014-04', '2014-03', '2014-02', '2014-01', '2013-12', '2013-11',\n", " '2013-10', '2013-09', '2013-08', '2013-07', '2013-06', '2013-05',\n", " '2013-04', '2013-03', '2013-02', '2013-01', '2012-12', '2012-11',\n", " '2012-10', '2012-09', '2012-08', '2012-07', '2012-06', '2012-05',\n", " '2012-04', '2012-03', '2012-02', '2012-01', '2011-12', '2011-11',\n", " '2011-10', '2011-09', '2011-08', '2011-07', '2011-06', '2011-05',\n", " '2011-04', '2011-03', '2011-02', '2011-01', '2010-12', '2010-11',\n", " '2010-10', '2010-09', '2010-08', '2010-07', '2010-06', '2010-05',\n", " '2010-04', '2010-03', '2010-02', '2010-01', '2009-12', '2009-11',\n", " '2009-10', '2009-09', '2009-08', '2009-07', '2009-06', '2009-05',\n", " '2009-04', '2009-03', '2009-02', '2009-01', '2008-12', '2008-11',\n", " '2008-10', '2008-09', '2008-08', '2008-07', '2008-06', '2008-05',\n", " '2008-04', '2008-03', '2008-02', '2008-01', '2007-12', '2007-11',\n", " '2007-10', '2007-09', '2007-08', '2007-07', '2007-06', '2007-05',\n", " '2007-04', '2007-03', '2007-02', '2007-01', '2006-12', '2006-11',\n", " '2006-10', '2006-09', '2006-08', '2006-07', '2006-06', '2006-05',\n", " '2006-04', '2006-03', '2006-02', '2006-01', '2005-12', '2005-11',\n", " '2005-10', '2005-09', '2005-08', '2005-07', '2005-06', '2005-05',\n", " '2005-04', '2005-03', '2005-02', '2005-01', '2004-12', '2004-11',\n", " '2004-10', '2004-09', '2004-08', '2004-07', '2004-06', '2004-05',\n", " '2004-04', '2004-03', '2004-02', '2004-01', '2003-12', '2003-11',\n", " '2003-10', '2003-09', '2003-08', '2003-07', '2003-06', '2003-05',\n", " '2003-04', '2003-03', '2003-02', '2003-01', '2002-12', '2002-11',\n", " '2002-10', '2002-09', '2002-08', '2002-07', '2002-06', '2002-05',\n", " '2002-04', '2002-03', '2002-02', '2002-01', '2001-12', '2001-11',\n", " '2001-10', '2001-09', '2001-08', '2001-07', '2001-06', '2001-05',\n", " '2001-04', '2001-03', '2001-02', '2001-01', '2000-12', '2000-11',\n", " '2000-10', '2000-09', '2000-08', '2000-07', '2000-06', '2000-05',\n", " '2000-04', '2000-03', '2000-02', '2000-01', '2037-02', '1899-12'],\n", " dtype='datetime64[M]'),\n", " array(['zn', 'y', 'wr', 'v', 'ss', 'sp', 'sn', 'sc', 'ru', 'rr', 'rb',\n", " 'pp', 'pg', 'pb', 'p', 'nr', 'ni', 'm', 'lu', 'lh', 'l', 'jm',\n", " 'jd', 'j', 'i', 'hc', 'fu', 'fb', 'eg', 'eb', 'cu', 'cs', 'c',\n", " 'bu', 'bc', 'bb', 'b', 'au', 'al', 'ag', 'a', 'ZC', 'WH', 'UR',\n", " 'TS', 'TF', 'TA', 'T', 'SR', 'SM', 'SF', 'SA', 'RS', 'RM', 'RI',\n", " 'PM', 'PK', 'PF', 'OI', 'MA', 'LR', 'JR', 'IM', 'IH', 'IF', 'IC',\n", " 'FG', 'CY', 'CJ', 'CF', 'AP'], dtype=object)],\n", " 'sortColumns': array(['code', 'm_nDatetime'], dtype=object),\n", " 'partitionSites': None}" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sess.run('loadTable(\"dfs://hft_futuremarket_ts\",\"TickPartitioned\").schema()')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 字段解释" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "```python\n", "# def get_trade_tick(self, start_date, end_date, code_list):\n", "\n", "# code_list_input = ', '.join(['\\'%s\\'' % code_id for code_id in code_list])\n", "scpt = \"\"\"\n", " stock_list := Array({code_list_input});\n", " r := select \n", " ['StockID'] as 'code',\n", " DateTimeToStr(['date']) as 'm_nDatetime',\n", "\n", " ['price'] as 'm_nPrice',\n", " ['open'] as 'm_nOpen',\n", " ['high'] as 'm_nHigh',\n", " ['low'] as 'm_nLow',\n", " ['close'] as 'm_nClose',\n", "\n", " ['sectional_high'] as 'm_nAccHigh',\n", " ['sectional_low'] as 'm_nAccLow',\n", "\n", " ['vol'] as 'm_iVolume', //成交量\n", " ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量\n", "\n", " ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量\n", " ['sectional_cjbs'] as 'm_nAccMatchItems',\n", "\n", " ['amount'] as 'm_iTurnover', //成交金额\n", " ['sectional_amount'] as 'm_iAccTurnover', \n", "\n", " ['yclose'] as 'm_nPreClose', //上一周期的收盘价\n", " ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘\n", " \n", " ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?\n", " ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量\n", " ['sale1'] as 'm_nAskPrice', \n", " ['sc1'] as 'm_nAskVolume', \n", "\n", " ['zmm'] as 'm_iABFlag', //买卖标识\n", "\n", " ['buy_vol'] as 'm_nActBidVolume', //主买量\n", " ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量\n", " ['buy_amount'] as 'm_nActBidTurnover', //主买金额\n", " ['sectional_buy_amount'] as 'm_nAccActBidTurnover', \n", "\n", " ['sale_vol'] as 'm_nActAskVolume', \n", " ['sectional_sale_vol'] as 'm_nAccActAskVolume', \n", " ['sale_amount'] as 'm_nActAskTurnover',\n", " ['sectional_sale_amount'] as 'm_nAccActAskTurnover', \n", "\n", " ['w_buy'] as 'm_nBidOrder', //委买\n", " ['sectional_w_buy'] as 'm_nAccBidOrder', \n", " ['w_sale'] as 'm_nAskOrder',\n", " ['sectional_w_sale'] as 'm_nAccAskOrder',\n", "\n", " ['wb'] as 'm_nABOrderRate', //委比\n", " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", " ['lb'] as 'm_nMItemsVolRate'//量比\n", "\n", " from tradetable\n", " datekey {start_date}T to {end_date}T+0.999 \n", " of stock_list\n", " end;\n", " \n", " return r; \n", "\"\"\".format(\n", " code_list_input=code_list_input,\n", " start_date=start_date,\n", " end_date=end_date\n", ")\n", "# logger.debug(scpt)\n", "# r = self.c.exec(scpt,timeout=self.timeout_default)\n", "# return r\n", "\n", "\"\"\"\n", "(source: 天软数据字典 ver 2022-05-25)\n", "vol: 周期内的成交量。如果周期为分钟，则表示这分钟的成 交量，并非从开盘累计.\n", " 对于期货，高频和日线的单位都是合约数量，如果要建立成交量和成交金额之间的关系，还需考虑合约乘数。\n", "amount: 周期内的成交金额。如果周期为分钟，则表示这分钟的成交金额，并非从开盘累计。\n", "cjbs: 对于期货/期权而言，是周期内的持仓的变动量，日线的话表示当日持仓量，单位为:手。\n", "wb: 公式:委买/委卖(w_buy/w_sale)\n", "lb: 由天软计算:\n", " 1、对于交易明细:\n", " 量比=现成交总手/[(过去 5 个交易日平均每分钟成交 量)×当日累计开市时间(分)]\n", " 2、对于秒线/1 分钟线: 量比取周期内最后一条交易明细的数据。\n", "\n", "zmm: 对 tradetable 公式:按下面的前后逻辑优先判定\n", " 没有成交 -> 0\n", " 当前成交价>上一笔买一价 ->主买 1 \n", " 当前成交价<上一笔卖一价 ->主卖 2 \n", " 否则->3(一般是集合竞价、涨停、跌停) \n", " 对于非交易明细(秒线或分钟线等):没有意义\n", "buy_vol: 主动性买盘成交量:周期内买卖标识为 1 的成交量+买 卖标识为 3 的成交量/2\n", "buy_amount: 主动性买盘成交金额:周期内买卖标识为 1 的成交金额 +买卖标识为 3 的成交金额/2\n", "sale_vol: 主动性卖盘成交量:周期内买卖标识为 2 的成交量之和 +买卖标识为 3 的成交量/2\n", "sale_amount: 主动性卖盘成交金额:周期内买卖标识为 2 的成交金额 +买卖标识为 3 的成交金额/2\n", "w_buy: 一个周期内的委买量\n", "sectional: 从开盘到当前时间的\n", "sectional_yclose: 上一交易日(周期是日)的收盘价，与‘上次价’不一 样。‘上次价’表示前一个周期的收盘价。\n", "sectional_vol: 从开盘到当前时间的成交量之和(主买+主卖)\n", "sectional_amount: 从开盘到当前时间的成交金额之和(主买+主卖)\n", "sectional_cjbs: 股指期货/期权:市场现有持仓量\n", "sectional_wb: 当日累计委买/当日累计委卖\n", "\n", "['sectional_open'] as 'm_nAccOpen', //时点当日开盘, 没必要存, 取上日2059的即可\n", "\n", "\"\"\"\n", "\n", "```" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.8.9 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.2" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" } } }, "nbformat": 4, "nbformat_minor": 2 }