{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ " import sqlalchemy as sa\n", " engine = sa.create_engine(\n", " 'mssql+pyodbc://sa:passw0rd!@192.168.1.7/master?driver=ODBC+Driver+18+for+SQL+Server',\n", " connect_args = {\n", " \"TrustServerCertificate\": \"yes\"\n", " }, echo=False)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "with engine.connect() as conn:\n", " stat = \"select distinct S_INFO_WINDCODE, TRADE_DT from Level2BytesKLine.dbo.KLine\"\n", " rs = conn.execute(stat)\n", " stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [], "source": [ "stock_list, trade_list = zip(*stock_date_list)" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4843" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(set(stock_list))" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'blob' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m/home/guofu/Workspaces/dolphin-dev/mssql.ipynb Cell 3\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mgzip\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mProtoBuffEntitys\u001b[39;00m \u001b[39mimport\u001b[39;00m TranseMessage_pb2\n\u001b[0;32m----> 4\u001b[0m f_all \u001b[39m=\u001b[39m gzip\u001b[39m.\u001b[39mdecompress(blob)\n\u001b[1;32m 5\u001b[0m dataArray \u001b[39m=\u001b[39m TranseMessage_pb2\u001b[39m.\u001b[39mTranseArray()\n\u001b[1;32m 6\u001b[0m dataArray\u001b[39m.\u001b[39mParseFromString(f_all)\n", "\u001b[0;31mNameError\u001b[0m: name 'blob' is not defined" ] } ], "source": [ "import gzip\n", "from ProtoBuffEntitys import TranseMessage_pb2\n", "\n", "f_all = gzip.decompress(blob)\n", "dataArray = TranseMessage_pb2.TranseArray()\n", "dataArray.ParseFromString(f_all)\n", "\n", "print(dataArray.dataArray)" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "code 600843\n", "m_nDate 20130104\n", "m_nTime 92506510\n", "m_nIndex 0\n", "m_chFunctionCode 0\n", "m_chOrderKind 0\n", "m_chBSFlag 83\n", "m_nTradePrice 6.8\n", "m_nTradeVolume 200\n", "m_nAskOrder 0\n", "m_nBidOrder 0\n" ] } ], "source": [ "for item in dataArray.dataArray:\n", " fields = item.ListFields()\n", " for desc, val in fields:\n", " print(desc.name, val)\n", " break\n", " " ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "with engine.connect() as conn:\n", " stat = \"select top 1 * from Level2BytesTickQue.dbo.TickQue\"\n", " rs = conn.execute(stat)\n", "\n", " for row in rs.fetchall():\n", " print(type(row))\n", " blob = row[2]\n" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1499694" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import gzip\n", "from ProtoBuffEntitys import TickQueueMessage_pb2\n", "\n", "f_all = gzip.decompress(blob)\n", "dataArray = TickQueueMessage_pb2.TickQueueArray()\n", "dataArray.ParseFromString(f_all)" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", "\n", "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n", " 0 0 0 0 0 0 0 0 0 0 0 0 0]\n" ] } ], "source": [ "import numpy as np\n", "print(dataArray.dataArray[0].m_nABVolume)\n", "print(type(dataArray.dataArray[0].m_nABVolume))\n", "print(np.array(dataArray.dataArray[0].m_nABVolume))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import urllib\n", "\n", "server = 'serverName\\instanceName,port' # to specify an alternate port\n", "database = 'mydb' \n", "username = 'myusername' \n", "password = 'mypassword'\n", "\n", "params = urllib.parse.quote_plus(\"'DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';UID='+username+';PWD='+ password\")\n", "\n", "engine = sa.ceate_engine(\"mssql+pyodbc:///?odbc_connect=%s\" % params)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "from ProtoBuffEntitys import KLineMessage_pb2, OrderMessage_pb2, TickMessage_pb2, TickQueueMessage_pb2, TranseMessage_pb2" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "HFT_TYPE_LIST = [\n", " 'KLine', 'Order', 'Tick', 'TickQueue', 'Transe'\n", "]\n", "PROTOBUFF_NAME_LIST = [f\"{name}Message_pb2\" for name in PROTOBUFF_NAME_LIST]\n", "\n", "import importlib\n", "\n", "PROTOBUFF_MODULE_LIST = [importlib.import_module(f\".{name}\", package='ProtoBuffEntitys') for name in PROTOBUFF_MODULE_LIST]" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "ename": "SyntaxError", "evalue": "invalid syntax (99226286.py, line 1)", "output_type": "error", "traceback": [ "\u001b[0;36m Input \u001b[0;32mIn [35]\u001b[0;36m\u001b[0m\n\u001b[0;31m from ProtoBuffEntitys import KLineMessage_pb2.KLineArray.KLineData.DESCRIPTOR\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" ] } ], "source": [ "import ProtoBuffEntitys.KLineMessage_pb2.KLineArray.KLineData.DESCRIPTOR " ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "'code'\n", "'m_nDate'\n", "'m_nTime'\n", "'m_nOpen'\n", "'m_nHigh'\n", "'m_nLow'\n", "'m_nClose'\n", "'m_iVolume'\n", "'m_iTurover'\n", "'m_nMatchItems'\n" ] } ], "source": [ "from pprint import pprint\n", "\n", "for field in KLineMessage_pb2.KLineArray.KLineData.DESCRIPTOR.fields:\n", " pprint(field.name)\n" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'\\n db_ts_stock.createPartitionedTable(\\n table = t, \\n tableName = abd, \\n partitionColumns = `code`m_nDate, \\n sortColumns = `code`m_nDate`m_nTime,\\n compressMethods = {m_nDate:\"delta\", m_nTime:\"delta\"}\\n )\\n'" ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\"\"\"\n", " db_ts_stock.createPartitionedTable(\n", " table = t, \n", " tableName = {hft_type_name}, \n", " partitionColumns = `code`m_nDate, \n", " sortColumns = `code`m_nDate`m_nTime,\n", " compressMethods = {{m_nDate:\"delta\", m_nTime:\"delta\"}}\n", " )\n", "\"\"\".format(\n", " hft_type_name = \"abd\"\n", ")" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('600519.SH', '20210531', b'\\x1f\\x8b\\x08\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x8c\\xddy\\\\^Yz\\x1f\\xf8tuUw\\xed%$\\xb4K\\x15\\x12\\x97+\\xd7V}\\x9cr\\xb7\\xa7\\xd3\\xd3^\\xd2\\xe5`\\xbb\\xd3\\xe9\\xe9\\xd8\\ ... (1454851 characters truncated) ... 17\\xb2\\x87\\xe4\\xce\\x94T\\xe8\\xe2O\\xc8\\xe6\\x10\\x1c=\\r\\xd9+\\x02\\xcc\\x9c!\\xc56\\xdb\\xef\\xb3uf\\xe5,d\\x9f\\xc8\\xd1\\xbc$\\xe7~)\\xe4\\xff\\x075\\xea@$i\\x1c\\x1a\\x00')\n" ] } ], "source": [ "with engine.connect() as conn:\n", " stat = \"select * from Level2BytesOrder.dbo.[Order] where S_INFO_WINDCODE='600519.SH'\"\n", " rs = conn.execute(stat)\n", " for row in rs.fetchall():\n", " print(row)\n", " break" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(4708,)\n" ] } ], "source": [ "with engine.connect() as conn:\n", " stat = \"select count(*) from Level2BytesKLine.dbo.KLine where TRADE_DT='20220608'\"\n", " rs = conn.execute(stat)\n", " for row in rs.fetchall():\n", " print(row)" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('NonClusteredIndex-Order', 'nonclustered located on PRIMARY', 'S_INFO_WINDCODE, TRADE_DT')\n" ] } ], "source": [ "engine = sa.create_engine(\n", " 'mssql+pyodbc://sa:passw0rd!@192.168.1.7/Level2BytesOrder?driver=ODBC+Driver+18+for+SQL+Server',\n", " connect_args = {\n", " \"TrustServerCertificate\": \"yes\"\n", " }, echo=False)\n", " \n", "with engine.connect() as conn:\n", " stat = \"EXEC sp_helpindex 'Level2BytesOrder.dbo.[Order]'\"\n", " rs = conn.execute(stat)\n", " for row in rs.fetchall():\n", " print(row)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('master', 1, datetime.datetime(2003, 4, 8, 9, 13, 36, 390000))\n", "('tempdb', 2, datetime.datetime(2022, 8, 3, 4, 47, 56, 987000))\n", "('model', 3, datetime.datetime(2003, 4, 8, 9, 13, 36, 390000))\n", "('msdb', 4, datetime.datetime(2022, 5, 29, 16, 33, 42, 60000))\n", "('test', 5, datetime.datetime(2022, 8, 3, 4, 52, 46, 450000))\n", "('Level1BytesIndexFutureKLine', 6, datetime.datetime(2022, 8, 3, 5, 2, 17, 660000))\n", "('Level2BytesConvBondKLine', 7, datetime.datetime(2022, 8, 3, 5, 2, 30, 837000))\n", "('Level2BytesConvBondOrder', 8, datetime.datetime(2022, 8, 3, 5, 2, 39, 987000))\n", "('Level2BytesConvBondTick', 9, datetime.datetime(2022, 8, 3, 5, 2, 54, 587000))\n", "('Level2BytesConvBondTickQue', 10, datetime.datetime(2022, 8, 3, 5, 3, 58, 270000))\n", "('Level2BytesConvBondTranse', 11, datetime.datetime(2022, 8, 3, 5, 4, 14, 500000))\n", "('Level2BytesETFKLine', 12, datetime.datetime(2022, 8, 3, 5, 4, 27, 270000))\n", "('Level2BytesETFOrder', 13, datetime.datetime(2022, 8, 3, 5, 4, 43, 457000))\n", "('Level2BytesTick', 14, datetime.datetime(2022, 8, 3, 8, 51, 40, 633000))\n", "('Level2BytesTickQue', 15, datetime.datetime(2022, 8, 3, 8, 51, 58, 650000))\n", "('Level2BytesTranse', 16, datetime.datetime(2022, 8, 3, 8, 52, 14, 103000))\n", "('Level2BytesOrder', 17, datetime.datetime(2022, 8, 3, 8, 52, 27, 740000))\n", "('Level2BytesKLine', 18, datetime.datetime(2022, 8, 3, 8, 52, 44, 610000))\n", "('Level2BytesIndexTick', 19, datetime.datetime(2022, 8, 3, 9, 22, 36, 850000))\n", "('Level2BytesIndexKLine', 20, datetime.datetime(2022, 8, 3, 9, 22, 57, 527000))\n", "('Level2BytesETFTranse', 21, datetime.datetime(2022, 8, 3, 9, 23, 53, 713000))\n", "('Level2BytesETFTickQue', 22, datetime.datetime(2022, 8, 3, 9, 24, 9, 87000))\n", "('Level2BytesETFTick', 23, datetime.datetime(2022, 8, 3, 9, 24, 26, 267000))\n" ] } ], "source": [ "stat = \"\"\"SELECT name, database_id, create_date FROM sys.databases; \n", "\"\"\"\n", "\n", "with engine.connect() as conn:\n", " rs = conn.execute(stat)\n", " for row in rs.fetchall():\n", " print(row)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "a = [1,2,3]\n", "b = {x : (x + 1 if x != 3 else x + 2) for x in a}" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "\n", "with engine.connect() as conn:\n", " stat = \"select distinct S_INFO_WINDCODE, TRADE_DT from Level2BytesKline.dbo.KLine\"\n", " rs = conn.execute(stat)\n", " stock_date_list = [(stock_name, date) for stock_name, date in rs.fetchall()]\n", "\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
codem_nDate
0000001.SZ20130104
1000001.SZ20130107
2000001.SZ20130108
3000001.SZ20130109
4000001.SZ20130110
.........
7379201689009.SH20220704
7379202689009.SH20220705
7379203689009.SH20220706
7379204689009.SH20220707
7379205689009.SH20220708
\n", "

7379206 rows × 2 columns

\n", "
" ], "text/plain": [ " code m_nDate\n", "0 000001.SZ 20130104\n", "1 000001.SZ 20130107\n", "2 000001.SZ 20130108\n", "3 000001.SZ 20130109\n", "4 000001.SZ 20130110\n", "... ... ...\n", "7379201 689009.SH 20220704\n", "7379202 689009.SH 20220705\n", "7379203 689009.SH 20220706\n", "7379204 689009.SH 20220707\n", "7379205 689009.SH 20220708\n", "\n", "[7379206 rows x 2 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.DataFrame(stock_date_list, columns=['code', 'm_nDate'])" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "df = pd.read_csv('ddb_dump_journal.csv')" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
type_namestock_idstatus
\n", "
" ], "text/plain": [ "Empty DataFrame\n", "Columns: [type_name, stock_id, status]\n", "Index: []" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_259767/4140820348.py:1: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n", " df = df.append({'type_name':'KLine', 'stock_id':'000001.SZ', 'status':'OK'},ignore_index=True)\n" ] } ], "source": [ "df = df.append({'type_name':'KLine', 'stock_id':'000001.SZ', 'status':'OK'},ignore_index=True)" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
type_namestock_idstatus
KLine000001.SZOK
\n", "
" ], "text/plain": [ "Empty DataFrame\n", "Columns: []\n", "Index: [(KLine, 000001.SZ, OK)]" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.set_index(['type_name', 'stock_id', 'status'], inplace=True)\n", "df" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Series([], Name: (KLine, 000001.SZ, OK), dtype: float64)" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.loc[('KLine', '000001.SZ', 'OK')]" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "('KLine', '000001.SZ', 'OK') in df.index" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [], "source": [ "import pickle as pkl\n", "with open('tmp.pkl', 'rb') as fin:\n", " stock_list, date_list = pkl.load(fin)" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('20130104',\n", " '20130107',\n", " '20130108',\n", " '20130109',\n", " '20130110',\n", " '20130111',\n", " '20130114',\n", " '20130115',\n", " '20130116',\n", " '20130117')" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "date_list[:10]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" }, "vscode": { "interpreter": { "hash": "5a0c795ff324b912f12ad95b94c9d776ccc7a75bdf6a126a4f44b3067472979e" } } }, "nbformat": 4, "nbformat_minor": 4 }