You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
95 lines
2.6 KiB
95 lines
2.6 KiB
2 years ago
|
import importlib
|
||
|
|
import gzip
|
||
|
|
import pickle
|
||
|
|
from pprint import pprint
|
||
|
|
from tqdm import tqdm
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
|
||
|
|
import dolphindb as ddb
|
||
|
|
import dolphindb.settings as keys
|
||
|
|
|
||
|
|
import sqlalchemy as sa
|
||
|
|
|
||
|
|
import ProtoBuffEntitys
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
from ProtoBuffEntitys import TickQueueMessage_pb2
|
||
|
|
engine = sa.create_engine(
|
||
|
|
"mssql+pyodbc://{username}:{password}@{host}/master?driver=ODBC+Driver+18+for+SQL+Server".format(**DDBLoader.mssql_config),
|
||
|
|
connect_args = {
|
||
|
|
"TrustServerCertificate": "yes"
|
||
|
|
}, echo=False
|
||
|
|
)
|
||
|
|
|
||
|
|
s = ddb.session(DDBLoader.ddb_config['host'], 8848)
|
||
|
|
s.login(DDBLoader.ddb_config['username'], DDBLoader.ddb_config['password'])
|
||
|
|
|
||
|
|
with engine.connect() as conn:
|
||
|
|
stat = "select top 1 * from Level2BytesTickQue.dbo.TickQue"
|
||
|
|
rs = conn.execute(stat)
|
||
|
|
|
||
|
|
for row in rs.fetchall():
|
||
|
|
print(type(row))
|
||
|
|
blob = gzip.decompress(row[2])
|
||
|
|
dataArray = TickQueueMessage_pb2.TickQueueArray()
|
||
|
|
dataArray.ParseFromString(blob)
|
||
|
|
|
||
|
|
data_dict_list = [
|
||
|
|
{field.name : val for field, val in entry.ListFields()}
|
||
|
|
for entry in dataArray.dataArray
|
||
|
|
]
|
||
|
|
|
||
|
|
df = pd.DataFrame(data_dict_list)
|
||
|
|
df['code'] = make_symbol(df['code'])
|
||
|
|
df['m_nDate'] = make_date(df['m_nDate'])
|
||
|
|
df['m_nTime'] = df['m_nDate'] + make_time(df['m_nTime'])
|
||
|
|
df['m_nABVolume'] = make_nparray(df['m_nABVolume'])
|
||
|
|
|
||
|
|
pprint(df[['code', 'm_nDate', 'm_nTime']].head())
|
||
|
|
|
||
|
|
s.upload({'tb' : df})
|
||
|
|
res = s.run("typestr(tb)")
|
||
|
|
pprint(res)
|
||
|
|
|
||
|
|
stat = """
|
||
|
|
syms = symbol(exec code from tb)
|
||
|
|
replaceColumn!(tb, `code, syms)
|
||
|
|
"""
|
||
|
|
res = s.run(stat)
|
||
|
|
pprint(res)
|
||
|
|
pprint(s.run("schema(tb).colDefs"))
|
||
|
|
|
||
|
|
res = s.run("append!(gtt, tb)")
|
||
|
|
s.undef("tb", "VAR")
|
||
|
|
|
||
|
|
print(res)
|
||
|
|
break
|
||
|
|
|
||
|
|
|
||
|
|
def main2():
|
||
|
|
loader = DDBLoader()
|
||
|
|
#pprint(loader.mssql_name_dict)
|
||
|
|
|
||
|
|
stock_list, date_list = loader.get_stock_date_list(True)
|
||
|
|
print('Did get stock list and date list from mssql')
|
||
|
|
#pprint(date_list[:5])
|
||
|
|
#pprint(stock_list[:5])
|
||
|
|
|
||
|
|
loader.create_ddb_database(stock_list, date_list)
|
||
|
|
print('Did create ddb database')
|
||
|
|
|
||
|
|
for type_name in loader.hft_type_list:
|
||
|
|
loader.create_ddb_partition_table(type_name)
|
||
|
|
print(f"Did create ddb table for {type_name}")
|
||
|
|
|
||
|
|
loader.do_load_from_mssql(type_name, stock_list[0], date_list[0])
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main3()
|
||
|
|
|