Dump daily one-hot concept data to dolphindb.

main
Guofu Li 2 years ago
parent 32e2092b8f
commit 083c66abb1

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 3,
"id": "7246e0c8-61cd-4cbf-a978-aa0dc0172d6d", "id": "7246e0c8-61cd-4cbf-a978-aa0dc0172d6d",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -35,28 +35,17 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 60, "execution_count": null,
"id": "ed17fd0b-9b36-47e4-9ab6-11459a3621fb", "id": "ed17fd0b-9b36-47e4-9ab6-11459a3621fb",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"115548"
]
},
"execution_count": 60,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"sess = ddb.session('192.168.1.7', 8848)\n", "sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n", "sess.login('admin', '123456')\n",
"\n", "\n",
"# backup(backup_path, sql_obj, force, parallel)\n", "# backup(backup_path, sql_obj, force, parallel)\n",
"code = \"\"\"\n", "code = \"\"\"\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://hft_stock_ts\", \"TickPartitioned\")>, true, true)\n", " backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://hft_stock_ts\", \"TransePartitioned\")>, false, false)\n",
"\"\"\"\n", "\"\"\"\n",
"\n", "\n",
"sess.run(code)" "sess.run(code)"
@ -64,7 +53,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 32, "execution_count": 127,
"id": "c8d07fc8-d80c-490f-9220-0d3e8e4c72a4", "id": "c8d07fc8-d80c-490f-9220-0d3e8e4c72a4",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -74,7 +63,7 @@
"50" "50"
] ]
}, },
"execution_count": 32, "execution_count": 127,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -104,7 +93,137 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 33, "execution_count": 2,
"id": "45c4cbc3-08a1-4083-8117-44e8e3364375",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Failed to connect to host = localhost port = 8848 with error code 111\n"
]
},
{
"ename": "RuntimeError",
"evalue": "<Exception> in login: Couldn't send script/function to the remote host because the connection has been closed",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdolphindb\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mddb\u001b[39;00m\n\u001b[1;32m 2\u001b[0m sess \u001b[38;5;241m=\u001b[39m ddb\u001b[38;5;241m.\u001b[39msession(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlocalhost\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m8848\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogin\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43madmin\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m123456\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# backup(backup_path, sql_obj, force, parallel)\u001b[39;00m\n\u001b[1;32m 6\u001b[0m sess\u001b[38;5;241m.\u001b[39mrun(\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;124m backup(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/data/dolphindb/backup/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, <select * from loadTable(\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdfs://daily_stock_ts\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhft_daily_factor\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m)>, false, false);\u001b[39m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;124m backup(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/data/dolphindb/backup/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, <select * from loadTable(\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdfs://daily_stock_ts\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124midx_daily_concept\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m)>, false, false);\u001b[39m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;124m backup(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/data/dolphindb/backup/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, <select * from loadTable(\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdfs://daily_stock_ts\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdaily_kline\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m)>, false, false);\u001b[39m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m)\n",
"File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:142\u001b[0m, in \u001b[0;36msession.login\u001b[0;34m(self, userid, password, enableEncryption)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpassword \u001b[38;5;241m=\u001b[39m password\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menableEncryption \u001b[38;5;241m=\u001b[39m enableEncryption\n\u001b[0;32m--> 142\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogin\u001b[49m\u001b[43m(\u001b[49m\u001b[43muserid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpassword\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menableEncryption\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 143\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmutex\u001b[38;5;241m.\u001b[39mrelease()\n",
"\u001b[0;31mRuntimeError\u001b[0m: <Exception> in login: Couldn't send script/function to the remote host because the connection has been closed"
]
}
],
"source": [
"import dolphindb as ddb\n",
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"# backup(backup_path, sql_obj, force, parallel)\n",
"sess.run(\"\"\"\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://daily_stock_ts\", \"hft_daily_factor\")>, false, false);\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://daily_stock_ts\", \"idx_daily_concept\")>, false, false);\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://daily_stock_ts\", \"daily_kline\")>, false, false);\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "463a59a8-39cf-4406-9390-da5dfdc44fa7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"50"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"sess.run(\"\"\"\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://daily_stock_ts\", \"idx_daily_concept\")>, false, false);\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "84f99b12-e868-425e-bcd3-1733feea7524",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>dbName</th>\n",
" <th>tableName</th>\n",
" <th>success</th>\n",
" <th>errorMsg</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>dfs://daily_stock_ts</td>\n",
" <td>idx_daily_concept</td>\n",
" <td>True</td>\n",
" <td></td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" dbName tableName success errorMsg\n",
"0 dfs://daily_stock_ts idx_daily_concept True "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"sess.run(\"\"\"\n",
" dropTable(database(\"dfs://daily_stock_ts\"), \"idx_daily_concept\");\n",
" migrate('/data/dolphindb/backup/', \"dfs://daily_stock_ts\", \"idx_daily_concept\");\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "8b7dae3d-aef1-4c50-92b2-460d4fea0a96", "id": "8b7dae3d-aef1-4c50-92b2-460d4fea0a96",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -136,35 +255,274 @@
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>daily_kline</td>\n", " <td>idx_daily_concept</td>\n",
" <td>uoH</td>\n", " <td>1esb</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n", " <tr>\n",
" <th>1</th>\n", " <th>1</th>\n",
" <td>hft_daily_factor</td>\n", " <td>hft_daily_factor</td>\n",
" <td>u6J</td>\n", " <td>u6J</td>\n",
" </tr>\n", " </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>daily_kline</td>\n",
" <td>uoH</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" tableName physicalIndex\n", " tableName physicalIndex\n",
"0 daily_kline uoH\n", "0 idx_daily_concept 1esb\n",
"1 hft_daily_factor u6J" "1 hft_daily_factor u6J\n",
"2 daily_kline uoH"
] ]
}, },
"execution_count": 33, "execution_count": 13,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "source": [
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"sess.run(\"\"\"\n", "sess.run(\"\"\"\n",
" //getAllDBs()\n",
" //listTables(\"dfs://info_stock_ts\");\n",
" //listTables(\"dfs://hft_stock_ts\");\n",
" listTables(\"dfs://daily_stock_ts\");\n", " listTables(\"dfs://daily_stock_ts\");\n",
" //schema(loadTable(\"dfs://daily_stock_ts\", \"idx_daily_concept\"))\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "fb23b78d-c402-4c11-9504-c61793e5a2a3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'partitionType': 5,\n",
" 'partitionColumnType': 17,\n",
" 'partitionColumnIndex': 0,\n",
" 'chunkPath': None,\n",
" 'colDefs': name typeString typeInt comment\n",
" 0 code SYMBOL 17 \n",
" 1 m_nDate DATE 6 \n",
" 2 open DOUBLE 16 \n",
" 3 high DOUBLE 16 \n",
" 4 low DOUBLE 16 \n",
" 5 close DOUBLE 16 \n",
" 6 vol DOUBLE 16 \n",
" 7 amount DOUBLE 16 \n",
" 8 cjbs INT 4 \n",
" 9 yclose DOUBLE 16 \n",
" 10 PctChg DOUBLE 16 \n",
" 11 IsZt INT 4 \n",
" 12 IsDt INT 4 \n",
" 13 IsST INT 4 \n",
" 14 IsGoDelist INT 4 \n",
" 15 FloatShares DOUBLE 16 \n",
" 16 MarketValues DOUBLE 16 \n",
" 17 factor DOUBLE 16 ,\n",
" 'chunkGranularity': 'TABLE',\n",
" 'partitionTypeName': 'HASH',\n",
" 'keepDuplicates': 'ALL',\n",
" 'engineType': 'TSDB',\n",
" 'partitionColumnName': 'code',\n",
" 'partitionSchema': 50,\n",
" 'sortColumns': array(['code', 'm_nDate'], dtype=object),\n",
" 'partitionSites': None}"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" schema(loadTable(\"dfs://daily_stock_ts\", \"daily_kline\"))\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 84,
"id": "84e9e3df-aec7-4edf-96fc-2481ed6790b5",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>dbName</th>\n",
" <th>tableName</th>\n",
" <th>success</th>\n",
" <th>errorMsg</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>dfs://daily_stock_ts</td>\n",
" <td>idx_daily_concept</td>\n",
" <td>True</td>\n",
" <td></td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" dbName tableName success errorMsg\n",
"0 dfs://daily_stock_ts idx_daily_concept True "
]
},
"execution_count": 84,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n",
"sess.run(\"\"\"\n",
" migrate('/data/dolphindb/backup/', \"dfs://daily_stock_ts\", \"idx_daily_concept\")\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "58ac1e6f-51cd-4d35-a34f-8220fca52acb",
"metadata": {},
"outputs": [],
"source": [
"sess.run(\"\"\"\n",
" \n",
"\"\"\")" "\"\"\")"
] ]
}, },
{
"cell_type": "code",
"execution_count": 16,
"id": "3411a78a-4efc-4693-af3f-44abe6ba166e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>min_m_nDate</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2006-01-04</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" min_m_nDate\n",
"0 2006-01-04"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"select min(m_nDate) from loadTable('dfs://daily_stock_ts', 'daily_kline')\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a5ddd8f-0bb2-4427-b5af-b32d80d6b3cf",
"metadata": {},
"outputs": [],
"source": [
"sess.run(\"\"\"\n",
" tbl = loadTable(\"dfs://hft_stock_ts\", \"KLinePartitioned\");\n",
" select * from (\n",
" select code, m_nDate, count(*) from tbl where code='600009.SH' group by code, m_nDate map\n",
" );\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 131,
"id": "06efcf17-b210-452f-a5fb-d4f97b5eae6b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"10"
]
},
"execution_count": 131,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"# backup(backup_path, sql_obj, force, parallel)\n",
"code = \"\"\"\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://info_stock_ts\", \"info_list_delist\")>, true, false);\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://info_stock_ts\", \"info_industry\")>, true, false);\n",
"\"\"\"\n",
"\n",
"sess.run(code)"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 34, "execution_count": 34,
@ -1234,65 +1592,35 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 138,
"id": "d68ea326-82c3-4a7c-97cf-c04dd8aee56b", "id": "d68ea326-82c3-4a7c-97cf-c04dd8aee56b",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "ename": "RuntimeError",
"text/html": [ "evalue": "<Exception> in run: Server response: 'tbl = loadTable(\"dfs://hft_stock_ts\", \"OrderPartitioned\") => FileBlock[/hft_stock_ts/domain, e56bb91f-d4d8-48be-6c4c-df3c1eb96dc4, {}, v1, 9413, c1] returned from name node didn't contain any site' script: '\ntbl = loadTable(\"dfs://hft_stock_ts\", 'OrderPartitioned');\nselect sum(cnt) from (select count(*) as cnt from tbl map);\n'",
"<div>\n", "output_type": "error",
"<style scoped>\n", "traceback": [
" .dataframe tbody tr th:only-of-type {\n", "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
" vertical-align: middle;\n", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
" }\n", "Input \u001b[0;32mIn [138]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m sess \u001b[38;5;241m=\u001b[39m ddb\u001b[38;5;241m.\u001b[39msession(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m192.168.1.7\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m8848\u001b[39m)\n\u001b[1;32m 2\u001b[0m sess\u001b[38;5;241m.\u001b[39mlogin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madmin\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m123456\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124;43mtbl = loadTable(\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdfs://hft_stock_ts\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m, \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mOrderPartitioned\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m);\u001b[39;49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;124;43mselect sum(cnt) from (select count(*) as cnt from tbl map);\u001b[39;49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"\n", "File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:161\u001b[0m, in \u001b[0;36msession.run\u001b[0;34m(self, script, *args, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfetchSize\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m BlockReader(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcpp\u001b[38;5;241m.\u001b[39mrunBlock(script, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
" .dataframe tbody tr th {\n", "\u001b[0;31mRuntimeError\u001b[0m: <Exception> in run: Server response: 'tbl = loadTable(\"dfs://hft_stock_ts\", \"OrderPartitioned\") => FileBlock[/hft_stock_ts/domain, e56bb91f-d4d8-48be-6c4c-df3c1eb96dc4, {}, v1, 9413, c1] returned from name node didn't contain any site' script: '\ntbl = loadTable(\"dfs://hft_stock_ts\", 'OrderPartitioned');\nselect sum(cnt) from (select count(*) as cnt from tbl map);\n'"
" vertical-align: top;\n", ]
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sum_cnt</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1468390810</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sum_cnt\n",
"0 1468390810"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
} }
], ],
"source": [ "source": [
"sess = ddb.session('192.168.1.7', 8848)\n", "sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n", "sess.login('admin', '123456')\n",
"sess.run(\"\"\"\n", "sess.run(\"\"\"\n",
"tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\n", "tbl = loadTable(\"dfs://hft_stock_ts\", 'OrderPartitioned');\n",
"select sum(cnt) from (select count(*) as cnt from tbl map);\n", "select sum(cnt) from (select count(*) as cnt from tbl map);\n",
"\"\"\")" "\"\"\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 38, "execution_count": 20,
"id": "7cbc4906-7756-424a-9ce5-9d2b6d1bab4b", "id": "7cbc4906-7756-424a-9ce5-9d2b6d1bab4b",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -1323,18 +1651,18 @@
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>4991926704</td>\n", " <td>1483910</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" sum_cnt\n", " sum_cnt\n",
"0 4991926704" "0 1483910"
] ]
}, },
"execution_count": 38, "execution_count": 20,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@ -1343,17 +1671,73 @@
"sess = ddb.session('192.168.1.7', 8848)\n", "sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n", "sess.login('admin', '123456')\n",
"sess.run(\"\"\"\n", "sess.run(\"\"\"\n",
" tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\n", " tbl = loadTable(\"dfs://hft_stock_ts\", 'TickQueuePartitioned');\n",
" select sum(cnt) from (select count(*) as cnt from tbl map);\n", " select sum(cnt) from (select count(*) as cnt from tbl where code='689009.SH' map);\n",
"\"\"\")" "\"\"\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 39, "execution_count": 9,
"id": "e9ab5e57-dce5-4426-9bac-4238cd067197", "id": "e9ab5e57-dce5-4426-9bac-4238cd067197",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Will retrieve partition 0\n",
"Will retrieve partition 1\n",
"Will retrieve partition 2\n",
"Will retrieve partition 3\n",
"Will retrieve partition 4\n",
"Will retrieve partition 5\n",
"Will retrieve partition 6\n",
"Will retrieve partition 7\n",
"Will retrieve partition 8\n",
"Will retrieve partition 9\n",
"Will retrieve partition 10\n",
"Will retrieve partition 11\n",
"Will retrieve partition 12\n",
"Will retrieve partition 13\n",
"Will retrieve partition 14\n",
"Will retrieve partition 15\n",
"Will retrieve partition 16\n",
"Will retrieve partition 17\n",
"Will retrieve partition 18\n",
"Will retrieve partition 19\n",
"Will retrieve partition 20\n",
"Will retrieve partition 21\n",
"Will retrieve partition 22\n",
"Will retrieve partition 23\n",
"Will retrieve partition 24\n",
"Will retrieve partition 25\n",
"Will retrieve partition 26\n",
"Will retrieve partition 27\n",
"Will retrieve partition 28\n",
"Will retrieve partition 29\n",
"Will retrieve partition 30\n",
"Will retrieve partition 31\n",
"Will retrieve partition 32\n",
"Will retrieve partition 33\n",
"Will retrieve partition 34\n",
"Will retrieve partition 35\n",
"Will retrieve partition 36\n",
"Will retrieve partition 37\n",
"Will retrieve partition 38\n",
"Will retrieve partition 39\n",
"Will retrieve partition 40\n",
"Will retrieve partition 41\n",
"Will retrieve partition 42\n",
"Will retrieve partition 43\n",
"Will retrieve partition 44\n",
"Will retrieve partition 45\n",
"Will retrieve partition 46\n",
"Will retrieve partition 47\n",
"Will retrieve partition 48\n",
"Will retrieve partition 49\n"
]
},
{ {
"data": { "data": {
"text/html": [ "text/html": [
@ -1375,38 +1759,67 @@
" <thead>\n", " <thead>\n",
" <tr style=\"text-align: right;\">\n", " <tr style=\"text-align: right;\">\n",
" <th></th>\n", " <th></th>\n",
" <th>sum_cnt</th>\n", " <th>code</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>9388749</td>\n", " <td>000400.SZ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>000558.SZ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>000602.SZ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>000677.SZ</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>000731.SZ</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" sum_cnt\n", " code\n",
"0 9388749" "0 000400.SZ\n",
"1 000558.SZ\n",
"2 000602.SZ\n",
"3 000677.SZ\n",
"4 000731.SZ"
] ]
}, },
"execution_count": 39, "execution_count": 9,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "source": [
"sess.run(\"\"\"\n", "import pandas as pd\n",
" select sum(cnt) from (select count(*) as cnt from tbl where code='002182.SZ' map);\n", "\n",
"\"\"\")" "df_list = []\n",
"for i in range(50):\n",
" print(f\"Will retrieve partition {i}\")\n",
" df_list.append(\n",
" sess.run(f\"\"\"\n",
" select code from tbl where partition(code, {i}) group by code;\n",
" \"\"\")\n",
" )\n",
"df = pd.concat(df_list, axis=0)\n",
"df.head()"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 108, "execution_count": 26,
"id": "4ba45027-bbb5-4b27-99da-3452cc8d2f1c", "id": "10eb9af7-739f-4b77-aa92-9abfba875370",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -1430,39 +1843,135 @@
" <thead>\n", " <thead>\n",
" <tr style=\"text-align: right;\">\n", " <tr style=\"text-align: right;\">\n",
" <th></th>\n", " <th></th>\n",
" <th>count</th>\n", " <th>code</th>\n",
" <th>m_nDate</th>\n",
" <th>cnt</th>\n",
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n", " <tr>\n",
" <th>0</th>\n", " <th>0</th>\n",
" <td>2287</td>\n", " <td>689009.SH</td>\n",
" <td>2021-08-19</td>\n",
" <td>2728</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-02-16</td>\n",
" <td>2570</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-03-02</td>\n",
" <td>2428</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-03-31</td>\n",
" <td>2584</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-04-08</td>\n",
" <td>2872</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-04-12</td>\n",
" <td>2902</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-04-18</td>\n",
" <td>2766</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-04-20</td>\n",
" <td>2584</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>689009.SH</td>\n",
" <td>2022-04-22</td>\n",
" <td>2858</td>\n",
" </tr>\n", " </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" count\n", " code m_nDate cnt\n",
"0 2287" "0 689009.SH 2021-08-19 2728\n",
"1 689009.SH 2022-02-16 2570\n",
"2 689009.SH 2022-03-02 2428\n",
"3 689009.SH 2022-03-31 2584\n",
"4 689009.SH 2022-04-08 2872\n",
"5 689009.SH 2022-04-12 2902\n",
"6 689009.SH 2022-04-18 2766\n",
"7 689009.SH 2022-04-20 2584\n",
"8 689009.SH 2022-04-22 2858"
] ]
}, },
"execution_count": 108, "execution_count": 26,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [
"sess.run(\"\"\"\n",
" tbl = loadTable(\"dfs://hft_stock_ts\", 'TickQueuePartitioned');\n",
" select code, m_nDate, count(*) as cnt from tbl where code='689009.SH' and count(*) < 3000 group by code, m_nDate map;\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "09332a23-4c1b-4d9f-8e7b-c7ef74611700",
"metadata": {},
"outputs": [],
"source": [
"df.sort_values('code').to_csv('../assets/TickQueuePartitioned.log.csv')"
]
},
{
"cell_type": "code",
"execution_count": 136,
"id": "4ba45027-bbb5-4b27-99da-3452cc8d2f1c",
"metadata": {},
"outputs": [
{
"ename": "RuntimeError",
"evalue": "<Exception> in run: Couldn't send script/function to the remote host because the connection has been closed",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [136]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124;43m select count(*) from (\u001b[39;49m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;124;43m select code, m_nDate, count(*) as cnt from tbl where code=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m300065.SZ\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m group by code, m_nDate map\u001b[39;49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124;43m );\u001b[39;49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:161\u001b[0m, in \u001b[0;36msession.run\u001b[0;34m(self, script, *args, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfetchSize\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m BlockReader(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcpp\u001b[38;5;241m.\u001b[39mrunBlock(script, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mRuntimeError\u001b[0m: <Exception> in run: Couldn't send script/function to the remote host because the connection has been closed"
]
}
],
"source": [ "source": [
"sess.run(\"\"\"\n", "sess.run(\"\"\"\n",
" select count(*) from (\n", " select count(*) from (\n",
" select code, m_nDate, count(*) as cnt from tbl where code='002533.SZ' group by code, m_nDate map\n", " select code, m_nDate, count(*) as cnt from tbl where code='300065.SZ' group by code, m_nDate map\n",
" );\n", " );\n",
"\"\"\")" "\"\"\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 107, "execution_count": 126,
"id": "29ab8af5-e571-4064-b691-a186d9fb4d08", "id": "29ab8af5-e571-4064-b691-a186d9fb4d08",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -1495,81 +2004,24 @@
" </tr>\n", " </tr>\n",
" </thead>\n", " </thead>\n",
" <tbody>\n", " <tbody>\n",
" <tr>\n",
" <th>2013-01-04</th>\n",
" <td>2224</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2013-01-07</th>\n",
" <td>2093</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2013-01-08</th>\n",
" <td>3009</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2013-01-09</th>\n",
" <td>2890</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2013-01-10</th>\n",
" <td>2703</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2022-07-04</th>\n",
" <td>3906</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2022-07-05</th>\n",
" <td>4230</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2022-07-06</th>\n",
" <td>4030</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2022-07-07</th>\n",
" <td>4526</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2022-07-08</th>\n",
" <td>4639</td>\n",
" </tr>\n",
" </tbody>\n", " </tbody>\n",
"</table>\n", "</table>\n",
"<p>2287 rows × 1 columns</p>\n",
"</div>" "</div>"
], ],
"text/plain": [ "text/plain": [
" count\n", "Empty DataFrame\n",
"m_nDate \n", "Columns: [count]\n",
"2013-01-04 2224\n", "Index: []"
"2013-01-07 2093\n",
"2013-01-08 3009\n",
"2013-01-09 2890\n",
"2013-01-10 2703\n",
"... ...\n",
"2022-07-04 3906\n",
"2022-07-05 4230\n",
"2022-07-06 4030\n",
"2022-07-07 4526\n",
"2022-07-08 4639\n",
"\n",
"[2287 rows x 1 columns]"
] ]
}, },
"execution_count": 107, "execution_count": 126,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "source": [
"df = sess.run(\"\"\"\n", "df = sess.run(\"\"\"\n",
" select m_nDate, count(*) from tbl where code='002533.SZ' group by m_nDate map;\n", " select m_nDate, count(*) from tbl where code='300066.SZ' group by m_nDate map;\n",
"\"\"\").set_index('m_nDate')\n", "\"\"\").set_index('m_nDate')\n",
"df" "df"
] ]
@ -1665,7 +2117,7 @@
"\tdo {\n", "\tdo {\n",
"\t\t//print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date);\n", "\t\t//print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date);\n",
"\t\tprint('Deleting ' + code + ' of ' + del_date);\n", "\t\tprint('Deleting ' + code + ' of ' + del_date);\n",
"\t\tdelete from tbl where m_nDate=del_date, code=code;\n", "\t\tdelete from tbl where m_nDate=del_date and code=code;\n",
"\t\tdel_date = temporalAdd(del_date, 1, \"d\");\n", "\t\tdel_date = temporalAdd(del_date, 1, \"d\");\n",
"\t} while (del_date <= 2022.12.31)\n", "\t} while (del_date <= 2022.12.31)\n",
"}\n", "}\n",
@ -1886,7 +2338,25 @@
"id": "16dd860d-049f-420e-8d04-d1f8969e5ed1", "id": "16dd860d-049f-420e-8d04-d1f8969e5ed1",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": [
"sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"code = \"\"\"\n",
"tbl = loadTable(\"dfs://hft_stock_ts\", 'TickQueuePartitioned')\n",
"code = '689009.SH'\n",
"\n",
"del_date = 2013.01.01\n",
"do {\n",
" //print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date);\n",
" print('Deleting ' + code + ' of ' + del_date);\n",
" delete from tbl where m_nDate=del_date and code=code;\n",
" del_date = temporalAdd(del_date, 1, \"d\");\n",
"} while (del_date <= 2022.12.31)\n",
"\"\"\"\n",
"# 这段代码运行好赶紧注释掉,非常危险\n",
"# sess.run(code)"
]
} }
], ],
"metadata": { "metadata": {

@ -2,7 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 214, "execution_count": 3,
"id": "139fd1cb-aedf-4186-8408-4d630ba69599", "id": "139fd1cb-aedf-4186-8408-4d630ba69599",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@ -13,323 +13,6 @@
"sess.login('admin', '123456')" "sess.login('admin', '123456')"
] ]
}, },
{
"cell_type": "code",
"execution_count": 216,
"id": "98ef95cf-0a7f-4e9d-8c7c-bc1d2a6268bd",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>tableName</th>\n",
" <th>physicalIndex</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>divident</td>\n",
" <td>Gws</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>preliminary_earnings_estimate</td>\n",
" <td>FyP</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>is_common_ori</td>\n",
" <td>DvA</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>cfs_common_ori</td>\n",
" <td>AUH</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>bs_common_adj</td>\n",
" <td>x2R</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>earnings_preannouncement</td>\n",
" <td>EOu</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>cfs_common_adj</td>\n",
" <td>zCC</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>is_common_adj</td>\n",
" <td>Cdl</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>bs_common_ori</td>\n",
" <td>ykc</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" tableName physicalIndex\n",
"0 divident Gws\n",
"1 preliminary_earnings_estimate FyP\n",
"2 is_common_ori DvA\n",
"3 cfs_common_ori AUH\n",
"4 bs_common_adj x2R\n",
"5 earnings_preannouncement EOu\n",
"6 cfs_common_adj zCC\n",
"7 is_common_adj Cdl\n",
"8 bs_common_ori ykc"
]
},
"execution_count": 216,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"listTables('dfs://pit_stock_ts')\")"
]
},
{
"cell_type": "code",
"execution_count": 217,
"id": "12d5328e-ee57-4cf6-adeb-863649cbb19d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'partitionType': 5,\n",
" 'partitionColumnType': 17,\n",
" 'partitionColumnIndex': 0,\n",
" 'chunkPath': None,\n",
" 'colDefs': name typeString typeInt comment\n",
" 0 code SYMBOL 17 \n",
" 1 report_period DATE 6 \n",
" 2 appear_at_date DATE 6 \n",
" 3 S_PROFITNOTICE_STYLE STRING 18 \n",
" 4 S_PROFITNOTICE_CHANGEMIN DOUBLE 16 \n",
" 5 S_PROFITNOTICE_CHANGEMAX DOUBLE 16 \n",
" 6 S_PROFITNOTICE_NETPROFITMIN DOUBLE 16 \n",
" 7 S_PROFITNOTICE_NETPROFITMAX DOUBLE 16 \n",
" 8 S_PROFITNOTICE_REASON STRING 18 ,\n",
" 'chunkGranularity': 'TABLE',\n",
" 'partitionTypeName': 'HASH',\n",
" 'keepDuplicates': 'ALL',\n",
" 'engineType': 'TSDB',\n",
" 'partitionColumnName': 'code',\n",
" 'partitionSchema': 50,\n",
" 'sortColumns': array(['code', 'report_period', 'appear_at_date'], dtype=object),\n",
" 'partitionSites': None}"
]
},
"execution_count": 217,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" schema(loadTable(\"dfs://pit_stock_ts\", \"earnings_preannouncement\"))\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 218,
"id": "46547093-9ee6-46ed-a821-73f3237b3c5e",
"metadata": {},
"outputs": [],
"source": [
"import sqlalchemy as sa\n",
"engine = sa.create_engine(\n",
" 'mssql+pyodbc://sa:xn.123@192.168.1.91/tr_statement?driver=ODBC+Driver+18+for+SQL+Server',\n",
" connect_args = {\n",
" \"TrustServerCertificate\": \"yes\"\n",
" }, echo=False)"
]
},
{
"cell_type": "code",
"execution_count": 224,
"id": "26b5701e-17e5-439e-8921-e7523384eee7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('tr_statement', 'dbo', 'SRC_TS_DIVIDEND_ANNOUNCEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_PRELIMINARY_EARNING_EST', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_FINCOMP_CASHFLOWSTATEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_FINCOMP_INCOMESTATEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_FINCOMP_BALANCESHEET', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_CONSOLIDATED_CASHFLOWSTATEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_CONSOLIDATED_INCOMESTATEMENT', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_CONSOLIDATED_BALANCESHEET', 'BASE TABLE'),\n",
" ('tr_statement', 'dbo', 'CBS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CBS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CBS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CIS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CIS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CIS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CCFS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CCFS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'CCFS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FBS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FBS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FBS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FIS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FIS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FIS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FCFS_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FCFS_BEFORE_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'FCFS_AFTER_ADJ', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'PEE_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'PEE_WIND', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'EP_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'DIV_META', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'DIV_WIND', 'VIEW'),\n",
" ('tr_statement', 'dbo', 'SRC_TS_NON_RECURRING_PNL', 'BASE TABLE')]\n"
]
}
],
"source": [
"from pprint import pprint \n",
"\n",
"with engine.connect() as conn:\n",
" rs = conn.execute(\"\"\"\n",
"SELECT\n",
" *\n",
"FROM\n",
" tr_statement.INFORMATION_SCHEMA.TABLES;\n",
"\"\"\").fetchall()\n",
"\n",
"pprint(rs)"
]
},
{
"cell_type": "code",
"execution_count": 231,
"id": "2dcd793a-2442-4b4e-b261-b2cead090efc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'StockID', 1, 'char', 8, 8, None, None, 0, None, None, 1, None, 8, 1, 'NO', 47),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'StockName', 12, 'varchar', 20, 20, None, None, 1, None, None, 12, None, 20, 2, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'ReportPeriod', 4, 'int', 10, 4, 0, 10, 0, None, None, 4, None, None, 3, 'NO', 56),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'AppearAtDate', 4, 'int', 10, 4, 0, 10, 0, None, None, 4, None, None, 4, 'NO', 56),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_CONTENT', -1, 'text', 2147483647, 2147483647, None, None, 1, None, None, -1, None, 2147483647, 5, 'YES', 35),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_STYLE', 12, 'varchar', 20, 20, None, None, 1, None, None, 12, None, 20, 6, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_CHANG_SOURCE', 12, 'varchar', 20, 20, None, None, 1, None, None, 12, None, 20, 7, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_REASON', -1, 'text', 2147483647, 2147483647, None, None, 1, None, None, -1, None, 2147483647, 8, 'YES', 35),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'REMARK', -1, 'text', 2147483647, 2147483647, None, None, 1, None, None, -1, None, 2147483647, 9, 'YES', 35),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_NETPROFITMAX', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 10, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_NETPROFITUNIT', 12, 'varchar', 20, 20, None, None, 1, None, None, 12, None, 20, 11, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_NETPROFITMIN', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 12, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_CHANGEMIN', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 13, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'SRC_TS_EARNINGS_PREANNOUNCEMENT', 'S_PROFITNOTICE_CHANGEMAX', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 14, 'YES', 109)]\n"
]
}
],
"source": [
"with engine.connect() as conn:\n",
" rs = conn.execute(\"EXEC sp_columns SRC_TS_EARNINGS_PREANNOUNCEMENT\").fetchall()\n",
"\n",
"pprint(rs)"
]
},
{
"cell_type": "code",
"execution_count": 230,
"id": "e5904cbf-4405-4d2e-bc5b-88706273f5d0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('NE430047', '诺思兰德', 20211231, 20220225, '净利润亏损5174.6万元', '增亏', '天软计算', '本报告期内,公司经营业绩较去年亏损增加。主要原因为:1、报告期内公司研发费用、管理费用较上年度有所增加2、报告期内毛利率较高的技术转让收入减少导致本报告期亏损增加。', 'nan', -5174.6, '万元', -5174.6, -104.1182, -104.1182),\n",
" ('NE430489', '佳先股份', 20211231, 20220228, '净利润盈利6193.94万元比上年同期增长125.26%', '预增', '公告中公布', '本报告期归属于上市公司股东的净利润预计增长125.26%的主要原因如下:\\r\\n1、报告期内公司新厂区生产经营稳定产能逐步放大为公司满足旺盛\\r\\n的市场需求提供了重要保障公司通过不断地拓展销售渠道销售业务实现较好\\r\\n增长同时因疫情、海运费上涨以及原料价格上涨、供求关系紧张等影响 ... (254 characters truncated) ... 和盈利水平得到进一步提升。\\r\\n3、报告期内公司将控股子公司安徽沙丰新材料有限公司全年数据纳入合\\r\\n并报表而上年同期由于收购日为2020年12月8日合并报表中仅合并了安\\r\\n徽沙丰新材料有限公司12月份数据。上述合并范围的差异也导致了公司报告期\\r\\n营业收入、利润较去年同期增长。', 'nan', 6193.94, '万元', 6193.94, 125.26, 125.26),\n",
" ('NE430510', '丰光精密', 20211231, 20220121, '净利润盈利4800万元-5100万元比上年同期增长43.62%-52.6%', '预增', '公告中公布', '(1)报告期内公司下游行业需求旺盛,同时公司市场拓展取得较好效果,公司半导体装备制造类产品和工业自动化类产品营业收入较上年同期增加;(2)公司汽车类产品营业收入较上年同期增加;(3)公司前期开发的新客户、新项目逐步实现量产,相关产品营业收入相应增加。\\r\\n', 'nan', 5100.0, '万元', 4800.0, 43.62, 52.6),\n",
" ('NE430564', '天润科技', 20220331, 20220525, '净利润盈利250万元-320万元比上年同期增长74.14%-122.9%', '预增', '公告中公布', '2022年1月份受西安疫情的影响公司在陕西省内的部分项目完工及验收时间有所推迟公司2022年第一季度收入较上年同期有所下降但由于2021年第一季度确认收入占比84.87%的汕头市中心城区(金平、龙湖区)农村地籍调查服务采购项目毛利率较低导致2022年第一季度公司预计收入规模虽较去年同期有所下降但净利润却较同期有所增长。', 'nan', 320.0, '万元', 250.0, 74.14, 122.9),\n",
" ('NE831370', '新安洁 ', 20211231, 20220225, '净利润盈利4159万元比上年同期下降55.38%', '预减', '公告中公布', '1、公司本报告期增值税及社保支出较上年同期有增长\\r\\n2、公司因应收账款账龄和金额发生变化本报告期的信用减值损失增加\\r\\n3、本报告期公司因部分项目撤场等原因导致资产处置损失增加。\\r\\n', 'nan', 4159.0, '万元', 4159.0, -55.38, -55.38),\n",
" ('NE831689', '克莱特 ', 20211231, 20220321, '净利润盈利4577.19万元比上年同期增长80.24%', '预增', '公告中公布', '公司归属于母公司所有者净利润上升,主要系受行业政策等因素影响,风电\\r\\n新能源装备等下游行业景气度不断提升对公司产品需求增长较快使得公司订\\r\\n单不断增长公司盈利能力进一步提升。\\r\\n受益于国家产业政策的支持国内海上风力发电行业进入快速发展阶段。公\\r\\n司顺势而为积极拓展海上风力 ... (34 characters truncated) ... 合作关系,如空空冷却器、水冷系统等海上风力发电产品。\\r\\n受到上述因素的影响公司新能源装备领域的收入出现了大幅的增长带动公司\\r\\n收入从2020年的28,491.97万元上升至2021年的39,156.58万元,同时带动公司\\r\\n的净利润从2,539.48万元上升至4,577.19万元。', 'nan', 4577.19, '万元', 4577.19, 80.24, 80.24),\n",
" ('NE831689', '克莱特 ', 20220331, 20220302, '净利润盈利900万元-950万元比上年同期增长6.57%-12.49%', '预增', '公告中公布', 'nan', 'nan', 950.0, '万元', 900.0, 6.57, 12.49),\n",
" ('NE831832', '科达自控', 20211231, 20220124, '净利润盈利3600万元-4000万元比上年同期增长23.88%-37.64%', '预增', '公告中公布', '报告期内公司省内外市场拓展取得较好效果,营业收入与上年同期相比有所提升净利润相应增加。', 'nan', 4000.0, '万元', 3600.0, 23.88, 37.64),\n",
" ('NE832145', '恒合股份', 20211231, 20220228, '净利润盈利1670.1万元比上年同期下降45.6%', '预减', '公告中公布', '1、公司油气回收在线监测收入的下降是导致营业收入和净利润降低的主要因素。该业务受疫情等因素的影响下游客户暂停或延缓招标加之疫情的反复\\r\\n使得已经开始的项目有所放缓致使公司油气回收在线监测系统总体收入有所降\\r\\n低进而影响了公司的营收和净利润。\\r\\n2、同时液位量测系统已过政策大力推动期公司液位量测业务量有所下\\r\\n降导致2021年营业收入也随之降低。\\r\\n3、公司预计归属于上市公司股东的扣除非经常性损益后的净利润变动幅度\\r\\n较大主要系公司在2021年度收到计入非经常性损益的上市补助及重大贡献奖\\r\\n励所致。', 'nan', 1670.1, '万元', 1670.1, -45.6, -45.6),\n",
" ('NE832419', '路斯股份', 20211231, 20220223, '净利润盈利2900万元-3500万元比上年同期下降9.72%-25.19%', '预减', '公告中公布', 'nan', 'nan', 3500.0, '万元', 2900.0, -25.19, -9.72)]\n"
]
}
],
"source": [
"with engine.connect() as conn:\n",
" rs = conn.execute(\"select top 10 * from SRC_TS_EARNINGS_PREANNOUNCEMENT\").fetchall()\n",
"\n",
"pprint(rs)"
]
},
{
"cell_type": "code",
"execution_count": 229,
"id": "f6e2ef1d-2e68-49c1-8c57-a0b937037831",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('tr_statement', 'dbo', 'EP_WIND', 'WIND_CODE', 12, 'varchar', 13, 13, None, None, 1, None, None, 12, None, 13, 1, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'IntCode', 4, 'int', 10, 4, 0, 10, 1, None, None, 4, None, None, 2, 'YES', 38),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'ReportPeriod', 4, 'int', 10, 4, 0, 10, 0, None, None, 4, None, None, 3, 'NO', 56),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'AppearAtDate', 4, 'int', 10, 4, 0, 10, 0, None, None, 4, None, None, 4, 'NO', 56),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_STYLE', 12, 'varchar', 20, 20, None, None, 1, None, None, 12, None, 20, 5, 'YES', 39),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_CHANGEMIN', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 6, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_CHANGEMAX', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 7, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_NETPROFITMIN', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 8, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_NETPROFITMAX', 6, 'float', 15, 8, None, 10, 1, None, None, 6, None, None, 9, 'YES', 109),\n",
" ('tr_statement', 'dbo', 'EP_WIND', 'S_PROFITNOTICE_REASON', -1, 'text', 2147483647, 2147483647, None, None, 1, None, None, -1, None, 2147483647, 10, 'YES', 35)]\n"
]
}
],
"source": [
"with engine.connect() as conn:\n",
" rs = conn.execute(\"EXEC sp_columns EP_WIND\").fetchall()\n",
"\n",
"pprint(rs)"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 42, "execution_count": 42,

@ -28,7 +28,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 40, "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"with engine.connect() as conn:\n",
" stat = \"select [StockID], [date] from [StockDaily].[dbo].[DailyKLine] group by [StockID], [date]\"\n",
" rs = conn.execute(statd)\n",
" stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -37,16 +49,16 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 45, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"4843" "5015"
] ]
}, },
"execution_count": 45, "execution_count": 7,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }

@ -1,6 +1,6 @@
import dolphindb as ddb import dolphindb as ddb
from pprint import pprint
class DDBBase(object): class DDBBase(object):
@ -11,7 +11,10 @@ class DDBBase(object):
} }
def __init__(self): def __init__(self, host=None, **kwargs):
if host is not None:
self.ddb_config['host'] = host
self.ddb_sess = ddb.session(self.ddb_config['host'], 8848) self.ddb_sess = ddb.session(self.ddb_config['host'], 8848)
self.ddb_sess.login(self.ddb_config['username'], self.ddb_config['password']) self.ddb_sess.login(self.ddb_config['username'], self.ddb_config['password'])

@ -275,20 +275,6 @@ class DDBHFTLoader(DDBLoader):
print('Did create the persistent table with the memory table') print('Did create the persistent table with the memory table')
def make_calendar_df(self):
print('Will create calendar dataframe from SQL Server')
# 从KLine表查询主要是因为KLine表最小
with self.mssql_engine.connect() as conn:
stat = "select distinct S_INFO_WINDCODE, TRADE_DT from Level2BytesKline.dbo.KLine"
rs = conn.execute(stat)
stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]
df_calendar = pd.DataFrame(stock_date_list, columns=['code', 'm_nDate'])
df_calendar['m_nDate'] = self.make_date(df_calendar['m_nDate'])
print('Did make the DataFrame for calendar')
return df_calendar
def _make_table_skeleton(self, hft_type_name, table_capacity=default_table_capacity): def _make_table_skeleton(self, hft_type_name, table_capacity=default_table_capacity):
def _make_tbl_config(field_list): def _make_tbl_config(field_list):

@ -34,8 +34,8 @@ class DDBLoader(DDBBase):
'password' : 'passw0rd!' 'password' : 'passw0rd!'
} }
def __init__(self): def __init__(self, **kwargs):
super().__init__() super().__init__(**kwargs)
self.mssql_engine = sa.create_engine( self.mssql_engine = sa.create_engine(
"mssql+pyodbc://{username}:{password}@{host}/master?driver=ODBC+Driver+18+for+SQL+Server".format(**self.mssql_config), "mssql+pyodbc://{username}:{password}@{host}/master?driver=ODBC+Driver+18+for+SQL+Server".format(**self.mssql_config),
connect_args = { connect_args = {
@ -96,5 +96,16 @@ class DDBLoader(DDBBase):
return pd.to_timedelta(s_hr + s_min + s_sec + s_ms, unit='ms') return pd.to_timedelta(s_hr + s_min + s_sec + s_ms, unit='ms')
def make_calendar_df(self):
print('Will create calendar dataframe from SQL Server')
# 从KLine表查询主要是因为KLine表最小
with self.mssql_engine.connect() as conn:
stat = "select distinct S_INFO_WINDCODE, TRADE_DT from Level2BytesKline.dbo.KLine"
rs = conn.execute(stat)
stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]
df_calendar = pd.DataFrame(stock_date_list, columns=['code', 'm_nDate'])
df_calendar['m_nDate'] = self.make_date(df_calendar['m_nDate'])
print('Did make the DataFrame for calendar')
return df_calendar

@ -2,6 +2,15 @@
from loader.DDBPITLoader import DDBPITLoader from loader.DDBPITLoader import DDBPITLoader
from loader.DDBHFTLoader import DDBHFTLoader from loader.DDBHFTLoader import DDBHFTLoader
from loader.DDBBasicInfoLoader import DDBBasicInfoLoader from loader.DDBBasicInfoLoader import DDBBasicInfoLoader
from loader.DDBIndexLoader import DDBIndexLoader
def create_index_data():
# 板块指数数据
loader = DDBIndexLoader(host='192.168.1.7')
loader.load_ddb_database()
loader.create_ddb_partition_table()
loader.dump_idx_concept_to_ddb()
def create_hft_data(): def create_hft_data():
@ -45,8 +54,8 @@ def main():
# TODO: # TODO:
# 可以使用`Fire`库,对函数调用再做一次封装,就可以避免每次运行不同参数时候需要修改内部多处的代码。 # 可以使用`Fire`库,对函数调用再做一次封装,就可以避免每次运行不同参数时候需要修改内部多处的代码。
create_index_data()
create_hft_data() #create_hft_data()
#create_pit_data() #create_pit_data()

Loading…
Cancel
Save