init

.gitignore

@@ -0,0 +1,10 @@
+*.json
+tmp*
+*.csv
+.ipynb*
+.DS_Store
+tmp*
+data/
+output/
+logs/
+*.pyc

src/DDBfm.py

@@ -0,0 +1,86 @@
+import dolphindb as ddb
+import dolphindb.settings as keys
+import numpy as np
+import pandas as pd
+# from loguru import logger
+
+class DDBfm():
+
+    ddb_config_servers = {
+        'local':
+            {'host' : '192.168.64.3',
+            'username' : 'admin',
+            'password' : '123456'},
+        'dev':
+            {'host' : '192.168.1.88',
+            'username' : 'admin',
+            'password' : '123456'},
+        'prd':
+            {'host' : '192.168.1.7',
+            'username' : 'admin',
+            'password' : '123456'}
+    }
+
+    # len: 71
+    all_fm_init=['sc', 'v', 'TS', 'MA', 'AP', 'jm', 'bc', 'bb', 'fu', 'IM', 'IF', 'a', 'lu', 'FG', 'cu', 'al', 'IH', 'RS', 'pg', 'CF', 'SF', 'ni', 'hc', 'UR', 'm', 'SR', 'j', 'PF', 'RM', 'T', 'c', 'JR', 'l', 'p', 'sp', 'CY', 'pb', 'TF', 'b', 'eg', 'rb', 'PK', 'sn', 'nr', 'pp', 'CJ', 'eb', 'SA', 'y', 'RI', 'lh', 'jd', 'OI', 'WH', 'ss', 'ru', 'zn', 'fb', 'rr', 'PM', 'au', 'TA', 'ZC', 'IC', 'bu', 'SM', 'wr', 'cs', 'LR', 'ag', 'i']
+    
+
+    ddb_daily_path = "dfs://daily_futuremarket_ts"
+    ddb_daily_dbname = "db_daily_fm"
+
+    ddb_hft_path="dfs://hft_futuremarket_ts"
+    
+    ddb_hft_tick_dbname="Tick"
+    ddf_hft_tick_tbname="TickPartitioned"
+
+    ddb_hft_mink_dbname="MinKline"
+    ddf_hft_mink_tbname="MinKlinePartitioned"
+
+
+    def __init__(self, which_server='local', **kwargs):
+
+        self.ddb_config = self.ddb_config_servers[which_server]
+
+        self.sess = ddb.session(self.ddb_config['host'], 8848)
+        self.sess.login(self.ddb_config['username'], self.ddb_config['password'])
+
+    def drop_dbpath(self,dbPath):
+        if self.sess.existsDatabase(dbPath): 
+            self.sess.dropDatabase(dbPath)
+
+    def create_ddb_database(self,dbPath,dbName):
+        db_init = self.sess.database(dbName='db_init', partitionType=keys.VALUE, partitions=self.all_fm_init, dbPath='')
+        
+        months=np.array(pd.date_range(start='2000-01', end='2050-12', freq="M"), dtype="datetime64[M]")
+        # print(months)
+        db_date = self.sess.database('db_date', partitionType=keys.VALUE, partitions=months, dbPath='')
+
+        if self.sess.existsDatabase(dbPath): 
+            self.sess.dropDatabase(dbPath)
+        db = self.sess.database(dbName=dbName, partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath, engine='OLAP')
+        # todo: engine=TSDB
+
+        return db
+    
+    def add_new_hft_table(self, db, tbName, df):
+        t = self.sess.table(data=df,tableAliasName=tbName)
+        pt =db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDate', 'code_init'])
+        pt.append(t)
+
+
+    def append_hft_table(self, dbPath, tbName, df):
+        # load table to check? time&date&code
+        appender = ddb.tableAppender(tableName=tbName, ddbSession=self.sess,dbPath=dbPath)
+        appender.append(df)
+            
+    
+
+
+
+def main():
+
+    ddb = DDBfm()
+    ddb.create_ddb_database()
+
+if __name__ == '__main__':
+    main()

src/TSLfm.py

@@ -0,0 +1,217 @@
+import itertools
+from datetime import datetime
+
+import pandas as pd
+from pprint import pprint
+
+import pyTSL
+
+TINYSOFT_HOSTNAME = 'tsl.tinysoft.com.cn'
+TINYSOFT_PORT = 443
+TINYSOFT_USERNAME = 'xunuo2005'
+TINYSOFT_PASSWORD = '20220613'
+
+class TSLfm:
+
+    def __init__(self):
+        self.timeout_default=100000000
+
+    def __enter__(self):
+        print('entering')
+        self.c = pyTSL.Client(TINYSOFT_USERNAME, TINYSOFT_PASSWORD, TINYSOFT_HOSTNAME, 443)
+        self.c.login()
+        return self
+        
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        print('logging out')
+        self.c.logout()
+        del(self.c)
+
+    def get_code_list(self, bk_name='国债期货'):
+        
+        scpt = """return getbk('{bk_name}');
+        """.format(bk_name=bk_name)
+        print(scpt)
+        r = self.c.exec(scpt,timeout=self.timeout_default)
+        return r.value()
+
+    def get_trade_tick(self, start_date, end_date, code_list):
+        """
+        (source: 天软数据字典 ver 2022-05-25)
+        vol: 周期内的成交量。如果周期为分钟，则表示这分钟的成 交量，并非从开盘累计.
+             对于期货，高频和日线的单位都是合约数量，如果要建立成交量和成交金额之间的关系，还需考虑合约乘数。
+        amount: 周期内的成交金额。如果周期为分钟，则表示这分钟的成交金额，并非从开盘累计。
+        cjbs: 对于期货/期权而言，是周期内的持仓的变动量，日线的话表示当日持仓量，单位为:手。
+        wb: 公式:委买/委卖(w_buy/w_sale)
+        lb: 由天软计算:
+            1、对于交易明细:
+            量比=现成交总手/[(过去 5 个交易日平均每分钟成交 量)×当日累计开市时间(分)]
+            2、对于秒线/1 分钟线: 量比取周期内最后一条交易明细的数据。
+
+        zmm: 对 tradetable 公式:按下面的前后逻辑优先判定
+            没有成交 -> 0
+            当前成交价>上一笔买一价 ->主买 1 
+            当前成交价<上一笔卖一价 ->主卖 2 
+            否则->3(一般是集合竞价、涨停、跌停) 
+            对于非交易明细(秒线或分钟线等):没有意义
+        buy_vol: 主动性买盘成交量:周期内买卖标识为 1 的成交量+买 卖标识为 3 的成交量/2
+        buy_amount: 主动性买盘成交金额:周期内买卖标识为 1 的成交金额 +买卖标识为 3 的成交金额/2
+        sale_vol: 主动性卖盘成交量:周期内买卖标识为 2 的成交量之和 +买卖标识为 3 的成交量/2
+        sale_amount: 主动性卖盘成交金额:周期内买卖标识为 2 的成交金额 +买卖标识为 3 的成交金额/2
+        w_buy: 一个周期内的委买量
+        sectional: 从开盘到当前时间的
+        sectional_yclose: 上一交易日(周期是日)的收盘价，与‘上次价’不一 样。‘上次价’表示前一个周期的收盘价。
+        sectional_vol: 从开盘到当前时间的成交量之和(主买+主卖)
+        sectional_amount: 从开盘到当前时间的成交金额之和(主买+主卖)
+        sectional_cjbs: 股指期货/期权:市场现有持仓量
+        sectional_wb: 当日累计委买/当日累计委卖
+
+        ['sectional_open'] as 'm_nAccOpen', //时点当日开盘, 没必要存, 取上日2059的即可
+
+        """
+
+        code_list_input = ', '.join(['\'%s\'' % code_id for code_id in code_list])
+        scpt = """
+            stock_list := Array({code_list_input});
+            r := select 
+                ['StockID'] as 'code',
+                DateTimeToStr(['date']) as 'datetime',
+
+                ['price'] as 'm_nPrice',
+                ['open'] as 'm_nOpen',
+                ['high'] as 'm_nHigh',
+                ['low'] as 'm_nLow',
+                ['close'] as 'm_nClose',
+
+                ['sectional_high'] as 'm_nAccHigh',
+                ['sectional_low'] as 'm_nAccLow',
+
+                ['vol'] as 'm_iVolume', //成交量
+                ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量
+
+                ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量
+                ['sectional_cjbs'] as 'm_nAccMatchItems',
+
+                ['amount'] as 'm_iTurnover', //成交金额
+                ['sectional_amount'] as 'm_iAccTurnover', 
+
+                ['yclose'] as 'm_nPreClose', //上一周期的收盘价
+                ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘
+                
+                ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?
+                ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量
+                ['sale1'] as 'm_nAskPrice', 
+                ['sc1'] as 'm_nAskVolume', 
+
+                ['zmm'] as 'm_iABFlag', //买卖标识
+
+                ['buy_vol'] as 'm_nActBidVolume', //主买量
+                ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量
+                ['buy_amount'] as 'm_nActBidTurnover', //主买金额
+                ['sectional_buy_amount'] as 'm_nAccActBidTurnover', 
+
+                ['sale_vol'] as 'm_nActAskVolume', 
+                ['sectional_sale_vol'] as 'm_nAccActAskVolume', 
+                ['sale_amount'] as 'm_nActAskTurnover',
+                ['sectional_sale_amount'] as 'm_nAccActAskTurnover', 
+
+                ['w_buy'] as 'm_nBidOrder', //委买
+                ['sectional_w_buy'] as 'm_nAccBidOrder', 
+                ['w_sale'] as 'm_nAskOrder',
+                ['sectional_w_sale'] as 'm_nAccAskOrder',
+
+                ['wb'] as 'm_nABOrderRate', //委比
+                ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比
+                ['lb'] as 'm_nMItemsVolRate'//量比
+
+                from tradetable
+                datekey {start_date}T to {end_date}T+0.999 
+                of stock_list
+            end;
+            
+            return r;        
+        """.format(
+            code_list_input=code_list_input,
+            start_date=start_date,
+            end_date=end_date
+        )
+        print(scpt)
+        r = self.c.exec(scpt,timeout=self.timeout_default)
+        return r
+    
+    def get_mkt_min_k(self, start_date, end_date, code_list):
+
+        code_list_input = ', '.join(['\'%s\'' % code_id for code_id in code_list])
+        scpt = """
+            SetSysParam(pn_cycle(), cy_1m());
+            stock_list := Array({code_list_input});
+            r := select 
+                ['StockID'] as 'code',
+                DateTimeToStr(['date']) as 'datetime',
+
+                ['price'] as 'm_nPrice',
+                ['open'] as 'm_nOpen',
+                ['high'] as 'm_nHigh',
+                ['low'] as 'm_nLow',
+                ['close'] as 'm_nClose',
+
+                ['sectional_high'] as 'm_nAccHigh',
+                ['sectional_low'] as 'm_nAccLow',
+
+                ['vol'] as 'm_iVolume', //成交量
+                ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量
+
+                ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量
+                ['sectional_cjbs'] as 'm_nAccMatchItems',
+
+                ['amount'] as 'm_iTurnover', //成交金额
+                ['sectional_amount'] as 'm_iAccTurnover', 
+
+                ['yclose'] as 'm_nPreClose', //上一周期的收盘价
+                ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘
+                
+                ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?
+                ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量
+                ['sale1'] as 'm_nAskPrice', 
+                ['sc1'] as 'm_nAskVolume', 
+
+                ['zmm'] as 'm_iABFlag', //买卖标识
+
+                ['buy_vol'] as 'm_nActBidVolume', //主买量
+                ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量
+                ['buy_amount'] as 'm_nActBidTurnover', //主买金额
+                ['sectional_buy_amount'] as 'm_nAccActBidTurnover', 
+
+                ['sale_vol'] as 'm_nActAskVolume', 
+                ['sectional_sale_vol'] as 'm_nAccActAskVolume', 
+                ['sale_amount'] as 'm_nActAskTurnover',
+                ['sectional_sale_amount'] as 'm_nAccActAskTurnover', 
+
+                ['w_buy'] as 'm_nBidOrder', //委买
+                ['sectional_w_buy'] as 'm_nAccBidOrder', 
+                ['w_sale'] as 'm_nAskOrder',
+                ['sectional_w_sale'] as 'm_nAccAskOrder',
+
+                ['wb'] as 'm_nABOrderRate', //委比
+                ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比
+                ['lb'] as 'm_nMItemsVolRate'//量比
+
+
+                from markettable
+                datekey {start_date}T to {end_date}T+0.999 
+                of stock_list
+            end;
+            
+            return r;        
+        """.format(
+            code_list_input=code_list_input,
+            start_date=start_date,
+            end_date=end_date
+        )
+        print(scpt)
+        r = self.c.exec(scpt,timeout=self.timeout_default)
+        return r
+    
+    def process_result_data_type(self, r):
+        df = pd.DataFrame(r.value())
+        print(df.shape)