guofu
/
dolphin-dev
1
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases 1 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '84e89f70aa'
${ noResults }
dolphin-dev/ipynb/ddb.ipynb

1268 lines
41 KiB
Raw Blame History Escape

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "7246e0c8-61cd-4cbf-a978-aa0dc0172d6d",
"metadata": {},
"outputs": [],
"source": [
"import dolphindb as ddb"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "5d0f471e-682e-43cc-abdb-7e52f3bbd707",
"metadata": {},
"outputs": [],
"source": [
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fca58bdc-2aa1-4610-9a94-67d55f97a6e1",
"metadata": {},
"outputs": [],
"source": [
"sess.run(\"\"\"\n",
" migrate('/data/dolphindb/backup/', \"dfs://hft_stock_ts\", \"OrderPartitioned\")\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 67,
"id": "ed17fd0b-9b36-47e4-9ab6-11459a3621fb",
"metadata": {},
"outputs": [],
"source": [
"# backup(backup_path, sql_obj, force, parallel)\n",
"code = \"\"\"\n",
" backup('/data/dolphindb/backup/', <select * from loadTable(\"dfs://hft_stock_ts\", \"OrderPartitioned\")>, true, true)\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 68,
"id": "8b7dae3d-aef1-4c50-92b2-460d4fea0a96",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"114350"
]
},
"execution_count": 68,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(code)"
]
},
{
"cell_type": "code",
"execution_count": 69,
"id": "bf3bc38e-74cb-4549-bdca-6a3d2a601488",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2287.0"
]
},
"execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"114350 / 50"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "11275975-af43-4123-973a-f75096eb4e43",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2311.0"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"115550 / 50"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "155735c7-bdad-4c64-a2a1-514fe54e088c",
"metadata": {},
"outputs": [],
"source": [
"import dolphindb as ddb\n",
"\n",
"sess = ddb.session('localhost', 8848)\n",
"sess.login('admin', '123456')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a2e95bc5-c1d5-47f5-9488-9b8702947afc",
"metadata": {},
"outputs": [],
"source": [
"# migrate(backupDir, [backupDBPath], [backupTableName], [newDBPath=backupDBPath], [newTableName=backupTableName])\n",
"code = \"\"\"\n",
" migrate('/data/dolphindb/backup/',\"dfs://hft_stock_ts\", \"KLinePartitioned\")\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8a6198ef-e7f1-4f58-ba24-32dc429c9803",
"metadata": {},
"outputs": [],
"source": [
"sess.run(code)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "60f71516-33fc-4147-9740-c493ccb6b6a8",
"metadata": {},
"outputs": [],
"source": [
"def make_kurto_code_partition(partition_num):\n",
" code = \"\"\"\n",
" tbl = loadTable(\"dfs://hft_stock_ts\", \"KLinePartitioned\");\n",
" ret = select \n",
" code, m_nDate, eachPre(\\, m_nClose)-1.0 as ret \n",
" from tbl \n",
" where partition(code, {partition_num}) \n",
" context by m_nDate;\n",
" kurto = select \n",
" code, m_nDate, sqrt(239) * sum(pow(ret, 3)) / pow(sum(pow(ret, 2)), 1.5) as kurto \n",
" from ret \n",
" group by code, m_nDate;\n",
" \"\"\".format(\n",
" partition_num = partition_num\n",
" )\n",
" sess.run(code)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "1397c1b7-f81a-465e-868a-894b3ef818fc",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>code</th>\n",
" <th>count</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>000400.SZ</td>\n",
" <td>2305</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>000558.SZ</td>\n",
" <td>2063</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>000602.SZ</td>\n",
" <td>47</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>000677.SZ</td>\n",
" <td>2009</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>000731.SZ</td>\n",
" <td>2311</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>94</th>\n",
" <td>688039.SH</td>\n",
" <td>624</td>\n",
" </tr>\n",
" <tr>\n",
" <th>95</th>\n",
" <td>688052.SH</td>\n",
" <td>52</td>\n",
" </tr>\n",
" <tr>\n",
" <th>96</th>\n",
" <td>688091.SH</td>\n",
" <td>198</td>\n",
" </tr>\n",
" <tr>\n",
" <th>97</th>\n",
" <td>688368.SH</td>\n",
" <td>656</td>\n",
" </tr>\n",
" <tr>\n",
" <th>98</th>\n",
" <td>688396.SH</td>\n",
" <td>575</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>99 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" code count\n",
"0 000400.SZ 2305\n",
"1 000558.SZ 2063\n",
"2 000602.SZ 47\n",
"3 000677.SZ 2009\n",
"4 000731.SZ 2311\n",
".. ... ...\n",
"94 688039.SH 624\n",
"95 688052.SH 52\n",
"96 688091.SH 198\n",
"97 688368.SH 656\n",
"98 688396.SH 575\n",
"\n",
"[99 rows x 2 columns]"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" select code, count(*) from kurto group by code\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "59758901-3728-4355-bbb8-15b3eb537cdf",
"metadata": {},
"outputs": [],
"source": [
"def create_daily_ddb_database():\n",
" code = \"\"\"\n",
" daily_stock_ts = database(\n",
" \"dfs://daily_stock_ts\", \n",
" HASH, [SYMBOL, 50], \n",
" engine = 'TSDB'\n",
" )\n",
" \"\"\"\n",
" sess.run(code)"
]
},
{
"cell_type": "code",
"execution_count": 38,
"id": "d8fa94ae-6ca4-41f3-8b13-47ab8876ba95",
"metadata": {},
"outputs": [],
"source": [
"def load_daily_ddb_database():\n",
" code = \"\"\"\n",
" daily_stock_ts = database(\"dfs://daily_stock_ts\")\n",
" \"\"\"\n",
" sess.run(code)\n",
"\n",
"load_daily_ddb_database()"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "84eb237e-959c-419f-bce1-6779acda86d4",
"metadata": {},
"outputs": [],
"source": [
"def create_kurto_partition_table():\n",
" # createPartitionedTable(dbHandle, table, tableName, [partitionColumns], [compressMethods], [sortColumns], [keepDuplicates=ALL], [sortKeyMappingFunction])\n",
" code = \"\"\"\n",
" dropTable(daily_stock_ts, \"kurto_partitioned\");\n",
" kurto_partitioned = createPartitionedTable(\n",
" daily_stock_ts,\n",
" kurto, \n",
" \"kurto_partitioned\",\n",
" partitionColumns = 'code',\n",
" compressMethods = {'m_nDate' : 'delta'},\n",
" sortColumns = `code`m_nDate\n",
" );\n",
" \"\"\"\n",
" sess.run(code)\n",
" \n",
"create_kurto_partition_table()"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "0d93b3d1-39b7-4e23-8abf-eda86dcd2a23",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>code</th>\n",
" <th>m_nDate</th>\n",
" <th>kurto</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-0.317991</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>000558.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-15.387820</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>000677.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-15.459625</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>000731.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>15.443339</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>000752.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>15.396262</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>000789.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-14.801720</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>000903.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-15.405085</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>000912.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>14.755241</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>000935.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>15.413214</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>000970.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>15.458413</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" code m_nDate kurto\n",
"0 000400.SZ 2013-01-04 -0.317991\n",
"1 000558.SZ 2013-01-04 -15.387820\n",
"2 000677.SZ 2013-01-04 -15.459625\n",
"3 000731.SZ 2013-01-04 15.443339\n",
"4 000752.SZ 2013-01-04 15.396262\n",
"5 000789.SZ 2013-01-04 -14.801720\n",
"6 000903.SZ 2013-01-04 -15.405085\n",
"7 000912.SZ 2013-01-04 14.755241\n",
"8 000935.SZ 2013-01-04 15.413214\n",
"9 000970.SZ 2013-01-04 15.458413"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" kurto_partitioned.tableInsert(kurto);\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "eea5cb0b-8452-41a9-a0ad-7cf52c856c25",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>count</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>154953</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" count\n",
"0 154953"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" select count(*) from kurto_partitioned;\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "9e7bc856-517b-4bf9-866d-4fe85b9705a3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'partitionType': 5,\n",
" 'partitionColumnType': 17,\n",
" 'partitionColumnIndex': 0,\n",
" 'chunkPath': None,\n",
" 'colDefs': name typeString typeInt comment\n",
" 0 code SYMBOL 17 \n",
" 1 m_nDate DATE 6 \n",
" 2 kurto DOUBLE 16 ,\n",
" 'chunkGranularity': 'TABLE',\n",
" 'partitionTypeName': 'HASH',\n",
" 'keepDuplicates': 'ALL',\n",
" 'engineType': 'TSDB',\n",
" 'partitionColumnName': 'code',\n",
" 'partitionSchema': 50,\n",
" 'sortColumns': array(['code', 'm_nDate'], dtype=object),\n",
" 'partitionSites': None}"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" schema(loadTable(\"dfs://daily_stock_ts\", \"kurto_partitioned\"));\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 46,
"id": "94ae00b5-ba42-48b9-b114-012cf306d871",
"metadata": {},
"outputs": [],
"source": [
"sess.run(\"m_table = table(kurto)\")"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "e30134f9-5110-421d-8f71-1bef25e7ef17",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>code</th>\n",
" <th>m_nDate</th>\n",
" <th>kurto</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-04</td>\n",
" <td>-0.317991</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-07</td>\n",
" <td>0.790872</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-08</td>\n",
" <td>0.655005</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-09</td>\n",
" <td>0.965054</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>000400.SZ</td>\n",
" <td>2013-01-10</td>\n",
" <td>-0.116488</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154948</th>\n",
" <td>688396.SH</td>\n",
" <td>2022-07-04</td>\n",
" <td>-15.420192</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154949</th>\n",
" <td>688396.SH</td>\n",
" <td>2022-07-05</td>\n",
" <td>-15.426922</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154950</th>\n",
" <td>688396.SH</td>\n",
" <td>2022-07-06</td>\n",
" <td>-15.389434</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154951</th>\n",
" <td>688396.SH</td>\n",
" <td>2022-07-07</td>\n",
" <td>-15.411602</td>\n",
" </tr>\n",
" <tr>\n",
" <th>154952</th>\n",
" <td>688396.SH</td>\n",
" <td>2022-07-08</td>\n",
" <td>-15.401453</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>154953 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" code m_nDate kurto\n",
"0 000400.SZ 2013-01-04 -0.317991\n",
"1 000400.SZ 2013-01-07 0.790872\n",
"2 000400.SZ 2013-01-08 0.655005\n",
"3 000400.SZ 2013-01-09 0.965054\n",
"4 000400.SZ 2013-01-10 -0.116488\n",
"... ... ... ...\n",
"154948 688396.SH 2022-07-04 -15.420192\n",
"154949 688396.SH 2022-07-05 -15.426922\n",
"154950 688396.SH 2022-07-06 -15.389434\n",
"154951 688396.SH 2022-07-07 -15.411602\n",
"154952 688396.SH 2022-07-08 -15.401453\n",
"\n",
"[154953 rows x 3 columns]"
]
},
"execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"m_table\")"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "947e3346-15f6-4fb5-b2d6-b97b013a3b36",
"metadata": {},
"outputs": [],
"source": [
"code = \"\"\"\n",
" // 保证创建新的分区表不会和已经存在表冲突\n",
" if (existsTable(\"{ddb_daily_path}\", \"{partition_tbl_name}\")) {{\n",
" dropTable({ddb_daily_dbname}, \"{partition_tbl_name}\");\n",
" }}\n",
"\n",
" {partition_tbl_name} = createPartitionedTable(\n",
" dbHandle = {ddb_daily_dbname},\n",
" table = {memory_tbl_name}, \n",
" \"{partition_tbl_name}\",\n",
" partitionColumns = 'code',\n",
" compressMethods = {{'m_nDate' : 'delta'}},\n",
" sortColumns = `code`m_nDate\n",
" );\n",
"\"\"\".format(\n",
" ddb_daily_path = \"A\",\n",
" ddb_daily_dbname = \"B\",\n",
" partition_tbl_name = \"C\",\n",
" memory_tbl_name = \"D\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "5d758c68-301a-47a7-9cd2-b7dd7edf1a55",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"('\\n'\n",
" ' // 保证创建新的分区表不会和已经存在表冲突\\n'\n",
" ' if (existsTable(\"A\", \"C\")) {\\n'\n",
" ' dropTable(B, \"C\");\\n'\n",
" ' }\\n'\n",
" '\\n'\n",
" ' C = createPartitionedTable(\\n'\n",
" ' dbHandle = B,\\n'\n",
" ' table = D, \\n'\n",
" ' \"C\",\\n'\n",
" \" partitionColumns = 'code',\\n\"\n",
" \" compressMethods = {'m_nDate' : 'delta'},\\n\"\n",
" ' sortColumns = `code`m_nDate\\n'\n",
" ' );\\n')\n"
]
}
],
"source": [
"from pprint import pprint\n",
"pprint(code)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "400424d9-9c58-4417-94e4-fe1569522b03",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'chunkGranularity': 'TABLE',\n",
" 'chunkPath': None,\n",
" 'colDefs': name typeString typeInt comment\n",
"0 code SYMBOL 17 \n",
"1 m_nDate DATE 6 \n",
"2 kurto DOUBLE 16 ,\n",
" 'engineType': 'TSDB',\n",
" 'keepDuplicates': 'ALL',\n",
" 'partitionColumnIndex': 0,\n",
" 'partitionColumnName': 'code',\n",
" 'partitionColumnType': 17,\n",
" 'partitionSchema': 50,\n",
" 'partitionSites': None,\n",
" 'partitionType': 5,\n",
" 'partitionTypeName': 'HASH',\n",
" 'sortColumns': array(['code', 'm_nDate'], dtype=object)}\n"
]
}
],
"source": [
"pprint(\n",
" sess.run(\"\"\"\n",
" schema(loadTable(\"dfs://daily_stock_ts\", \"hft_daily_factor\"));\n",
" \"\"\")\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 62,
"id": "68e74de8-359c-4c47-a361-2c006896211c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>count</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>309906</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" count\n",
"0 309906"
]
},
"execution_count": 62,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"\"\"\n",
" select count(*) from loadTable(\"dfs://daily_stock_ts\", \"hft_daily_factor\");\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "b1bd7605-c347-44f2-a90f-bc949cde0ef0",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>code</th>\n",
" <th>report_period</th>\n",
" <th>appear_in_period</th>\n",
" <th>appear_at_date</th>\n",
" <th>TOT_OPER_REV</th>\n",
" <th>OPER_REV</th>\n",
" <th>INT_INC</th>\n",
" <th>INSUR_PREM_UNEARNED</th>\n",
" <th>HANDLING_CHRG_COMM_INC</th>\n",
" <th>NET_INC_OTHER_OPS</th>\n",
" <th>...</th>\n",
" <th>OTHER_INCOME</th>\n",
" <th>MEMO</th>\n",
" <th>ASSET_DISPOSAL_INCOME</th>\n",
" <th>CONTINUED_NET_PROFIT</th>\n",
" <th>END_NET_PROFIT</th>\n",
" <th>CREDIT_IMPAIRMENT_LOSS</th>\n",
" <th>RD_EXPENSE</th>\n",
" <th>STMNOTE_FINEXP</th>\n",
" <th>FIN_EXP_INT_INC</th>\n",
" <th>TOT_OPER_COST2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>000677.SZ</td>\n",
" <td>2009-12-31</td>\n",
" <td>2009-12-31</td>\n",
" <td>2010-04-26</td>\n",
" <td>3.514495e+09</td>\n",
" <td>3.514495e+09</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>26392229.68</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>nan</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>3.474455e+09</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>000677.SZ</td>\n",
" <td>2009-12-31</td>\n",
" <td>2009-12-31</td>\n",
" <td>2011-12-10</td>\n",
" <td>3.514495e+09</td>\n",
" <td>3.514495e+09</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>26392229.68</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>nan</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>3.460335e+09</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>2 rows × 69 columns</p>\n",
"</div>"
],
"text/plain": [
" code report_period appear_in_period appear_at_date TOT_OPER_REV \\\n",
"0 000677.SZ 2009-12-31 2009-12-31 2010-04-26 3.514495e+09 \n",
"1 000677.SZ 2009-12-31 2009-12-31 2011-12-10 3.514495e+09 \n",
"\n",
" OPER_REV INT_INC INSUR_PREM_UNEARNED HANDLING_CHRG_COMM_INC \\\n",
"0 3.514495e+09 0.0 0.0 0.0 \n",
"1 3.514495e+09 0.0 0.0 0.0 \n",
"\n",
" NET_INC_OTHER_OPS ... OTHER_INCOME MEMO ASSET_DISPOSAL_INCOME \\\n",
"0 26392229.68 ... 0.0 nan 0.0 \n",
"1 26392229.68 ... 0.0 nan 0.0 \n",
"\n",
" CONTINUED_NET_PROFIT END_NET_PROFIT CREDIT_IMPAIRMENT_LOSS RD_EXPENSE \\\n",
"0 0.0 0.0 0.0 0.0 \n",
"1 0.0 0.0 0.0 0.0 \n",
"\n",
" STMNOTE_FINEXP FIN_EXP_INT_INC TOT_OPER_COST2 \n",
"0 0.0 0.0 3.474455e+09 \n",
"1 0.0 0.0 3.460335e+09 \n",
"\n",
"[2 rows x 69 columns]"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import dolphindb as ddb \n",
"\n",
"sess = ddb.session('192.168.1.167', 8848)\n",
"sess.login('admin', '123456')\n",
"\n",
"sess.run(\"\"\"\n",
" is_common_adj = loadTable(\"dfs://pit_stock_ts\", \"is_common_adj\")\n",
" is_common_ori = loadTable(\"dfs://pit_stock_ts\", \"is_common_ori\")\n",
" select * from is_common_ori where code='000677.SZ' and report_period=2009.12.31\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "f0b3928d-c06f-4976-b568-091d98293a6e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>code</th>\n",
" <th>report_period</th>\n",
" <th>appear_in_period</th>\n",
" <th>appear_at_date</th>\n",
" <th>TOT_OPER_REV</th>\n",
" <th>OPER_REV</th>\n",
" <th>INT_INC</th>\n",
" <th>INSUR_PREM_UNEARNED</th>\n",
" <th>HANDLING_CHRG_COMM_INC</th>\n",
" <th>NET_INC_OTHER_OPS</th>\n",
" <th>...</th>\n",
" <th>OTHER_INCOME</th>\n",
" <th>MEMO</th>\n",
" <th>ASSET_DISPOSAL_INCOME</th>\n",
" <th>CONTINUED_NET_PROFIT</th>\n",
" <th>END_NET_PROFIT</th>\n",
" <th>CREDIT_IMPAIRMENT_LOSS</th>\n",
" <th>RD_EXPENSE</th>\n",
" <th>STMNOTE_FINEXP</th>\n",
" <th>FIN_EXP_INT_INC</th>\n",
" <th>TOT_OPER_COST2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>000677.SZ</td>\n",
" <td>2009-12-31</td>\n",
" <td>2010-12-31</td>\n",
" <td>2011-04-29</td>\n",
" <td>3.514495e+09</td>\n",
" <td>3.514495e+09</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>26392229.68</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>nan</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>3.474455e+09</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1 rows × 69 columns</p>\n",
"</div>"
],
"text/plain": [
" code report_period appear_in_period appear_at_date TOT_OPER_REV \\\n",
"0 000677.SZ 2009-12-31 2010-12-31 2011-04-29 3.514495e+09 \n",
"\n",
" OPER_REV INT_INC INSUR_PREM_UNEARNED HANDLING_CHRG_COMM_INC \\\n",
"0 3.514495e+09 0.0 0.0 0.0 \n",
"\n",
" NET_INC_OTHER_OPS ... OTHER_INCOME MEMO ASSET_DISPOSAL_INCOME \\\n",
"0 26392229.68 ... 0.0 nan 0.0 \n",
"\n",
" CONTINUED_NET_PROFIT END_NET_PROFIT CREDIT_IMPAIRMENT_LOSS RD_EXPENSE \\\n",
"0 0.0 0.0 0.0 0.0 \n",
"\n",
" STMNOTE_FINEXP FIN_EXP_INT_INC TOT_OPER_COST2 \n",
"0 0.0 0.0 3.474455e+09 \n",
"\n",
"[1 rows x 69 columns]"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess.run(\"select * from is_common_adj where code='000677.SZ' and report_period=2009.12.31\")"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "d68ea326-82c3-4a7c-97cf-c04dd8aee56b",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sum_cnt</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sum_cnt\n",
"0 0"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sess = ddb.session('192.168.1.7', 8848)\n",
"sess.login('admin', '123456')\n",
"sess.run(\"\"\"\n",
"tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\n",
"select sum(cnt) from (select count(*) as cnt from tbl map);\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "7cbc4906-7756-424a-9ce5-9d2b6d1bab4b",
"metadata": {},
"outputs": [
{
"ename": "RuntimeError",
"evalue": "<Exception> in run: Server response: 'tbl = loadTable(\"dfs://hft_stock_ts\", \"TickPartitioned\") => getFileBlocksMeta on path '/hft_stock_ts/TickPartitioned.tbl' failed, reason: path does not exist' script: '\ntbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\nselect sum(cnt) from (select count(*) as cnt from tbl map);\n'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [28]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m sess \u001b[38;5;241m=\u001b[39m ddb\u001b[38;5;241m.\u001b[39msession(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m192.168.1.167\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m8848\u001b[39m)\n\u001b[1;32m 2\u001b[0m sess\u001b[38;5;241m.\u001b[39mlogin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madmin\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m123456\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124;43mtbl = loadTable(\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdfs://hft_stock_ts\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m, \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTickPartitioned\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m);\u001b[39;49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;124;43mselect sum(cnt) from (select count(*) as cnt from tbl map);\u001b[39;49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:161\u001b[0m, in \u001b[0;36msession.run\u001b[0;34m(self, script, *args, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfetchSize\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m BlockReader(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcpp\u001b[38;5;241m.\u001b[39mrunBlock(script, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mRuntimeError\u001b[0m: <Exception> in run: Server response: 'tbl = loadTable(\"dfs://hft_stock_ts\", \"TickPartitioned\") => getFileBlocksMeta on path '/hft_stock_ts/TickPartitioned.tbl' failed, reason: path does not exist' script: '\ntbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\nselect sum(cnt) from (select count(*) as cnt from tbl map);\n'"
]
}
],
"source": [
"sess = ddb.session('192.168.1.167', 8848)\n",
"sess.login('admin', '123456')\n",
"sess.run(\"\"\"\n",
"tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned');\n",
"select sum(cnt) from (select count(*) as cnt from tbl map);\n",
"\"\"\")"
]
},
{
"cell_type": "code",
"execution_count": 31,
"id": "0e6e949b-ff25-40eb-a70a-72a3a6639aec",
"metadata": {},
"outputs": [
{
"ename": "RuntimeError",
"evalue": "<Exception> in run: Failed to read response header from the socket with IO error type 13",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Input \u001b[0;32mIn [31]\u001b[0m, in \u001b[0;36m<cell line: 15>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m code \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124mtbl = loadTable(\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdfs://hft_stock_ts\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTickPartitioned\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\n\u001b[1;32m 3\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124m}\u001b[39m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[0;32m---> 15\u001b[0m \u001b[43msess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcode\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/.venv/tinysoft/lib/python3.8/site-packages/dolphindb/session.py:161\u001b[0m, in \u001b[0;36msession.run\u001b[0;34m(self, script, *args, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfetchSize\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m BlockReader(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcpp\u001b[38;5;241m.\u001b[39mrunBlock(script, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcpp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"\u001b[0;31mRuntimeError\u001b[0m: <Exception> in run: Failed to read response header from the socket with IO error type 13"
]
}
],
"source": [
"code = \"\"\"\n",
"tbl = loadTable(\"dfs://hft_stock_ts\", 'TickPartitioned')\n",
"\n",
"for (code in ['000666.SZ' , '000667.SZ', '000668.SZ', '000669.SZ']) {\n",
"\tdel_date = 2013.01.01\n",
"\t//end_date = 2013.01.31\n",
"\tdo {\n",
"\t\t//print('Deleting ' + code + ' from ' + start_date + ' to ' + end_date);\n",
"\t\tprint('Deleting ' + code + ' of ' + del_date);\n",
"\t\tdelete from tbl where m_nDate=del_date, code=code;\n",
"\t\tdel_date = temporalAdd(del_date, 1, \"d\");\n",
"\t} while (del_date <= 2022.12.31)\n",
"}\n",
"\"\"\"\n",
"sess.run(code)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ea0501e7-d416-45ce-add5-e443c55f158c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in new issue View Git Blame Copy Permalink