From 3b0b17c04e23af30d6bb5b011a7fb180b316dd8f Mon Sep 17 00:00:00 2001 From: yzlocal Date: Wed, 9 Nov 2022 13:58:55 +0800 Subject: [PATCH] init --- .gitignore | 10 +++ README.md | 0 src/DDBfm.py | 86 ++++++++++++++++++ src/TSLfm.py | 217 +++++++++++++++++++++++++++++++++++++++++++++ src/data_loader.py | 0 5 files changed, 313 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 src/DDBfm.py create mode 100644 src/TSLfm.py create mode 100644 src/data_loader.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4514470 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.json +tmp* +*.csv +.ipynb* +.DS_Store +tmp* +data/ +output/ +logs/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/DDBfm.py b/src/DDBfm.py new file mode 100644 index 0000000..cad02f9 --- /dev/null +++ b/src/DDBfm.py @@ -0,0 +1,86 @@ +import dolphindb as ddb +import dolphindb.settings as keys +import numpy as np +import pandas as pd +# from loguru import logger + +class DDBfm(): + + ddb_config_servers = { + 'local': + {'host' : '192.168.64.3', + 'username' : 'admin', + 'password' : '123456'}, + 'dev': + {'host' : '192.168.1.88', + 'username' : 'admin', + 'password' : '123456'}, + 'prd': + {'host' : '192.168.1.7', + 'username' : 'admin', + 'password' : '123456'} + } + + # len: 71 + all_fm_init=['sc', 'v', 'TS', 'MA', 'AP', 'jm', 'bc', 'bb', 'fu', 'IM', 'IF', 'a', 'lu', 'FG', 'cu', 'al', 'IH', 'RS', 'pg', 'CF', 'SF', 'ni', 'hc', 'UR', 'm', 'SR', 'j', 'PF', 'RM', 'T', 'c', 'JR', 'l', 'p', 'sp', 'CY', 'pb', 'TF', 'b', 'eg', 'rb', 'PK', 'sn', 'nr', 'pp', 'CJ', 'eb', 'SA', 'y', 'RI', 'lh', 'jd', 'OI', 'WH', 'ss', 'ru', 'zn', 'fb', 'rr', 'PM', 'au', 'TA', 'ZC', 'IC', 'bu', 'SM', 'wr', 'cs', 'LR', 'ag', 'i'] + + + ddb_daily_path = "dfs://daily_futuremarket_ts" + ddb_daily_dbname = "db_daily_fm" + + ddb_hft_path="dfs://hft_futuremarket_ts" + + ddb_hft_tick_dbname="Tick" + ddf_hft_tick_tbname="TickPartitioned" + + ddb_hft_mink_dbname="MinKline" + ddf_hft_mink_tbname="MinKlinePartitioned" + + + def __init__(self, which_server='local', **kwargs): + + self.ddb_config = self.ddb_config_servers[which_server] + + self.sess = ddb.session(self.ddb_config['host'], 8848) + self.sess.login(self.ddb_config['username'], self.ddb_config['password']) + + def drop_dbpath(self,dbPath): + if self.sess.existsDatabase(dbPath): + self.sess.dropDatabase(dbPath) + + def create_ddb_database(self,dbPath,dbName): + db_init = self.sess.database(dbName='db_init', partitionType=keys.VALUE, partitions=self.all_fm_init, dbPath='') + + months=np.array(pd.date_range(start='2000-01', end='2050-12', freq="M"), dtype="datetime64[M]") + # print(months) + db_date = self.sess.database('db_date', partitionType=keys.VALUE, partitions=months, dbPath='') + + if self.sess.existsDatabase(dbPath): + self.sess.dropDatabase(dbPath) + db = self.sess.database(dbName=dbName, partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath, engine='OLAP') + # todo: engine=TSDB + + return db + + def add_new_hft_table(self, db, tbName, df): + t = self.sess.table(data=df,tableAliasName=tbName) + pt =db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDate', 'code_init']) + pt.append(t) + + + def append_hft_table(self, dbPath, tbName, df): + # load table to check? time&date&code + appender = ddb.tableAppender(tableName=tbName, ddbSession=self.sess,dbPath=dbPath) + appender.append(df) + + + + + +def main(): + + ddb = DDBfm() + ddb.create_ddb_database() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/src/TSLfm.py b/src/TSLfm.py new file mode 100644 index 0000000..68624df --- /dev/null +++ b/src/TSLfm.py @@ -0,0 +1,217 @@ +import itertools +from datetime import datetime + +import pandas as pd +from pprint import pprint + +import pyTSL + +TINYSOFT_HOSTNAME = 'tsl.tinysoft.com.cn' +TINYSOFT_PORT = 443 +TINYSOFT_USERNAME = 'xunuo2005' +TINYSOFT_PASSWORD = '20220613' + +class TSLfm: + + def __init__(self): + self.timeout_default=100000000 + + def __enter__(self): + print('entering') + self.c = pyTSL.Client(TINYSOFT_USERNAME, TINYSOFT_PASSWORD, TINYSOFT_HOSTNAME, 443) + self.c.login() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + print('logging out') + self.c.logout() + del(self.c) + + def get_code_list(self, bk_name='国债期货'): + + scpt = """return getbk('{bk_name}'); + """.format(bk_name=bk_name) + print(scpt) + r = self.c.exec(scpt,timeout=self.timeout_default) + return r.value() + + def get_trade_tick(self, start_date, end_date, code_list): + """ + (source: 天软数据字典 ver 2022-05-25) + vol: 周期内的成交量。如果周期为分钟，则表示这分钟的成 交量，并非从开盘累计. + 对于期货，高频和日线的单位都是合约数量，如果要建立成交量和成交金额之间的关系，还需考虑合约乘数。 + amount: 周期内的成交金额。如果周期为分钟，则表示这分钟的成交金额，并非从开盘累计。 + cjbs: 对于期货/期权而言，是周期内的持仓的变动量，日线的话表示当日持仓量，单位为:手。 + wb: 公式:委买/委卖(w_buy/w_sale) + lb: 由天软计算: + 1、对于交易明细: + 量比=现成交总手/[(过去 5 个交易日平均每分钟成交 量)×当日累计开市时间(分)] + 2、对于秒线/1 分钟线: 量比取周期内最后一条交易明细的数据。 + + zmm: 对 tradetable 公式:按下面的前后逻辑优先判定 + 没有成交 -> 0 + 当前成交价>上一笔买一价 ->主买 1 + 当前成交价<上一笔卖一价 ->主卖 2 + 否则->3(一般是集合竞价、涨停、跌停) + 对于非交易明细(秒线或分钟线等):没有意义 + buy_vol: 主动性买盘成交量:周期内买卖标识为 1 的成交量+买 卖标识为 3 的成交量/2 + buy_amount: 主动性买盘成交金额:周期内买卖标识为 1 的成交金额 +买卖标识为 3 的成交金额/2 + sale_vol: 主动性卖盘成交量:周期内买卖标识为 2 的成交量之和 +买卖标识为 3 的成交量/2 + sale_amount: 主动性卖盘成交金额:周期内买卖标识为 2 的成交金额 +买卖标识为 3 的成交金额/2 + w_buy: 一个周期内的委买量 + sectional: 从开盘到当前时间的 + sectional_yclose: 上一交易日(周期是日)的收盘价，与‘上次价’不一 样。‘上次价’表示前一个周期的收盘价。 + sectional_vol: 从开盘到当前时间的成交量之和(主买+主卖) + sectional_amount: 从开盘到当前时间的成交金额之和(主买+主卖) + sectional_cjbs: 股指期货/期权:市场现有持仓量 + sectional_wb: 当日累计委买/当日累计委卖 + + ['sectional_open'] as 'm_nAccOpen', //时点当日开盘, 没必要存, 取上日2059的即可 + + """ + + code_list_input = ', '.join(['\'%s\'' % code_id for code_id in code_list]) + scpt = """ + stock_list := Array({code_list_input}); + r := select + ['StockID'] as 'code', + DateTimeToStr(['date']) as 'datetime', + + ['price'] as 'm_nPrice', + ['open'] as 'm_nOpen', + ['high'] as 'm_nHigh', + ['low'] as 'm_nLow', + ['close'] as 'm_nClose', + + ['sectional_high'] as 'm_nAccHigh', + ['sectional_low'] as 'm_nAccLow', + + ['vol'] as 'm_iVolume', //成交量 + ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量 + + ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量 + ['sectional_cjbs'] as 'm_nAccMatchItems', + + ['amount'] as 'm_iTurnover', //成交金额 + ['sectional_amount'] as 'm_iAccTurnover', + + ['yclose'] as 'm_nPreClose', //上一周期的收盘价 + ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘 + + ['buy1'] as 'm_nBidPrice', //买一价？叫卖价? + ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量 + ['sale1'] as 'm_nAskPrice', + ['sc1'] as 'm_nAskVolume', + + ['zmm'] as 'm_iABFlag', //买卖标识 + + ['buy_vol'] as 'm_nActBidVolume', //主买量 + ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量 + ['buy_amount'] as 'm_nActBidTurnover', //主买金额 + ['sectional_buy_amount'] as 'm_nAccActBidTurnover', + + ['sale_vol'] as 'm_nActAskVolume', + ['sectional_sale_vol'] as 'm_nAccActAskVolume', + ['sale_amount'] as 'm_nActAskTurnover', + ['sectional_sale_amount'] as 'm_nAccActAskTurnover', + + ['w_buy'] as 'm_nBidOrder', //委买 + ['sectional_w_buy'] as 'm_nAccBidOrder', + ['w_sale'] as 'm_nAskOrder', + ['sectional_w_sale'] as 'm_nAccAskOrder', + + ['wb'] as 'm_nABOrderRate', //委比 + ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比 + ['lb'] as 'm_nMItemsVolRate'//量比 + + from tradetable + datekey {start_date}T to {end_date}T+0.999 + of stock_list + end; + + return r; + """.format( + code_list_input=code_list_input, + start_date=start_date, + end_date=end_date + ) + print(scpt) + r = self.c.exec(scpt,timeout=self.timeout_default) + return r + + def get_mkt_min_k(self, start_date, end_date, code_list): + + code_list_input = ', '.join(['\'%s\'' % code_id for code_id in code_list]) + scpt = """ + SetSysParam(pn_cycle(), cy_1m()); + stock_list := Array({code_list_input}); + r := select + ['StockID'] as 'code', + DateTimeToStr(['date']) as 'datetime', + + ['price'] as 'm_nPrice', + ['open'] as 'm_nOpen', + ['high'] as 'm_nHigh', + ['low'] as 'm_nLow', + ['close'] as 'm_nClose', + + ['sectional_high'] as 'm_nAccHigh', + ['sectional_low'] as 'm_nAccLow', + + ['vol'] as 'm_iVolume', //成交量 + ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量 + + ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量 + ['sectional_cjbs'] as 'm_nAccMatchItems', + + ['amount'] as 'm_iTurnover', //成交金额 + ['sectional_amount'] as 'm_iAccTurnover', + + ['yclose'] as 'm_nPreClose', //上一周期的收盘价 + ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘 + + ['buy1'] as 'm_nBidPrice', //买一价？叫卖价? + ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量 + ['sale1'] as 'm_nAskPrice', + ['sc1'] as 'm_nAskVolume', + + ['zmm'] as 'm_iABFlag', //买卖标识 + + ['buy_vol'] as 'm_nActBidVolume', //主买量 + ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量 + ['buy_amount'] as 'm_nActBidTurnover', //主买金额 + ['sectional_buy_amount'] as 'm_nAccActBidTurnover', + + ['sale_vol'] as 'm_nActAskVolume', + ['sectional_sale_vol'] as 'm_nAccActAskVolume', + ['sale_amount'] as 'm_nActAskTurnover', + ['sectional_sale_amount'] as 'm_nAccActAskTurnover', + + ['w_buy'] as 'm_nBidOrder', //委买 + ['sectional_w_buy'] as 'm_nAccBidOrder', + ['w_sale'] as 'm_nAskOrder', + ['sectional_w_sale'] as 'm_nAccAskOrder', + + ['wb'] as 'm_nABOrderRate', //委比 + ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比 + ['lb'] as 'm_nMItemsVolRate'//量比 + + + from markettable + datekey {start_date}T to {end_date}T+0.999 + of stock_list + end; + + return r; + """.format( + code_list_input=code_list_input, + start_date=start_date, + end_date=end_date + ) + print(scpt) + r = self.c.exec(scpt,timeout=self.timeout_default) + return r + + def process_result_data_type(self, r): + df = pd.DataFrame(r.value()) + print(df.shape) \ No newline at end of file diff --git a/src/data_loader.py b/src/data_loader.py new file mode 100644 index 0000000..e69de29