From 82d3c28895c97eb370b9643b50a611293291e9f6 Mon Sep 17 00:00:00 2001 From: yzlocal Date: Wed, 9 Nov 2022 15:36:16 +0800 Subject: [PATCH] things work rn --- dolphindb-test-clean.ipynb | 1873 ++++++++++++++++++++++++++++++++++++ src/DDBfm.py | 37 +- src/TSLfm.py | 47 +- src/data_loader.py | 40 + test/test.dos | 29 + 5 files changed, 2000 insertions(+), 26 deletions(-) create mode 100644 dolphindb-test-clean.ipynb create mode 100644 test/test.dos diff --git a/dolphindb-test-clean.ipynb b/dolphindb-test-clean.ipynb new file mode 100644 index 0000000..909a8df --- /dev/null +++ b/dolphindb-test-clean.ipynb @@ -0,0 +1,1873 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import dolphindb as ddb\n", + "import dolphindb.settings as keys\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "\n", + "sess = ddb.session('192.168.64.3',8848)\n", + "sess.login('admin','123456')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'/hft_fm_combo': 'pt:2:0:pt_2; ',\n", + " '/db_compo_test': 'pt:2:0:pt_2; ',\n", + " '/compoDB': 'pt:2:0:pt_2; ',\n", + " '/daily_futuremarket_ts': 'db_daily_kline:2:0:db_daily_kline_2; '}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.run('getAllDBs()')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StockIDopenhighlowclosevolsectional_volcjbssectional_cjbspriceyclosebuy_volbuy_amountsale_volsale_amountzmmamountdatetime1
17AP22128387.08387.08387.08387.09.09.0-9.0195658387.08413.04.537741.54.537741.5375483.02022-09-1908:59:00
18AP22128361.08361.08361.08361.08.017.00.0195658361.08387.00.00.08.066892.0266892.02022-09-1909:00:00
19AP22128356.08356.08356.08356.06.023.00.0195658356.08361.00.00.06.050158.0250158.02022-09-1909:00:00
25AP22128356.08356.08356.08356.01.024.00.0195658356.08356.00.00.01.08347.028347.02022-09-1909:00:05
30AP22128397.08397.08397.08397.01.025.0-1.0195648397.08356.01.08395.00.00.018395.02022-09-1909:00:09
............................................................
2336544AP22128744.08744.08744.08744.01.02583.00.0137188744.08756.00.00.01.08722.028722.02022-11-0109:25:28
2336547AP22128746.08746.08746.08746.01.02584.00.0137188746.08744.00.00.01.08722.028722.02022-11-0109:25:34
2336549AP22128743.08743.08743.08743.01.02585.00.0137188743.08746.00.00.01.08722.028722.02022-11-0109:25:37
2336592AP22128747.08747.08747.08747.02.02587.00.0137188747.08743.00.00.02.017444.0217444.02022-11-0109:26:59
2336594AP22128747.08747.08747.08747.03.02590.03.0137218747.08747.03.026166.00.00.0126166.02022-11-0109:27:03
\n", + "

40456 rows × 19 columns

\n", + "
" + ], + "text/plain": [ + " StockID open high low close vol sectional_vol cjbs \\\n", + "17 AP2212 8387.0 8387.0 8387.0 8387.0 9.0 9.0 -9.0 \n", + "18 AP2212 8361.0 8361.0 8361.0 8361.0 8.0 17.0 0.0 \n", + "19 AP2212 8356.0 8356.0 8356.0 8356.0 6.0 23.0 0.0 \n", + "25 AP2212 8356.0 8356.0 8356.0 8356.0 1.0 24.0 0.0 \n", + "30 AP2212 8397.0 8397.0 8397.0 8397.0 1.0 25.0 -1.0 \n", + "... ... ... ... ... ... ... ... ... \n", + "2336544 AP2212 8744.0 8744.0 8744.0 8744.0 1.0 2583.0 0.0 \n", + "2336547 AP2212 8746.0 8746.0 8746.0 8746.0 1.0 2584.0 0.0 \n", + "2336549 AP2212 8743.0 8743.0 8743.0 8743.0 1.0 2585.0 0.0 \n", + "2336592 AP2212 8747.0 8747.0 8747.0 8747.0 2.0 2587.0 0.0 \n", + "2336594 AP2212 8747.0 8747.0 8747.0 8747.0 3.0 2590.0 3.0 \n", + "\n", + " sectional_cjbs price yclose buy_vol buy_amount sale_vol \\\n", + "17 19565 8387.0 8413.0 4.5 37741.5 4.5 \n", + "18 19565 8361.0 8387.0 0.0 0.0 8.0 \n", + "19 19565 8356.0 8361.0 0.0 0.0 6.0 \n", + "25 19565 8356.0 8356.0 0.0 0.0 1.0 \n", + "30 19564 8397.0 8356.0 1.0 8395.0 0.0 \n", + "... ... ... ... ... ... ... \n", + "2336544 13718 8744.0 8756.0 0.0 0.0 1.0 \n", + "2336547 13718 8746.0 8744.0 0.0 0.0 1.0 \n", + "2336549 13718 8743.0 8746.0 0.0 0.0 1.0 \n", + "2336592 13718 8747.0 8743.0 0.0 0.0 2.0 \n", + "2336594 13721 8747.0 8747.0 3.0 26166.0 0.0 \n", + "\n", + " sale_amount zmm amount date time1 \n", + "17 37741.5 3 75483.0 2022-09-19 08:59:00 \n", + "18 66892.0 2 66892.0 2022-09-19 09:00:00 \n", + "19 50158.0 2 50158.0 2022-09-19 09:00:00 \n", + "25 8347.0 2 8347.0 2022-09-19 09:00:05 \n", + "30 0.0 1 8395.0 2022-09-19 09:00:09 \n", + "... ... ... ... ... ... \n", + "2336544 8722.0 2 8722.0 2022-11-01 09:25:28 \n", + "2336547 8722.0 2 8722.0 2022-11-01 09:25:34 \n", + "2336549 8722.0 2 8722.0 2022-11-01 09:25:37 \n", + "2336592 17444.0 2 17444.0 2022-11-01 09:26:59 \n", + "2336594 0.0 1 26166.0 2022-11-01 09:27:03 \n", + "\n", + "[40456 rows x 19 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "df = pd.read_csv('data/trade_AP2211+AP2212+AP2301+AP2303+AP2304+AP2305+AP2310+CF2211+CF2301+CF2303+CF2305+CF2307+CF2309_20220919_20221101.csv',index_col=None)\n", + "df.drop(columns=['Unnamed: 0'],inplace=True)\n", + "df=df[df['time1']<'09:30:00']\n", + "df = df[df['vol']>0]\n", + "df['date']=df['date'].astype('datetime64[D]')\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "df['init']=df['init'].astype('str')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "df['init']=df['StockID'].apply(lambda x:x[:-4])\n", + "df['finishm']=df['StockID'].apply(lambda x:x[-4:])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['AP'], dtype=object)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['init'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "71" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "all_init={'sc', 'v', 'TS', 'MA', 'AP', 'jm', 'bc', 'bb', 'fu', 'IM', 'IF', 'a', 'lu', 'FG', 'cu', 'al', 'IH', 'RS', 'pg', 'CF', 'SF', 'ni', 'hc', 'UR', 'm', 'SR', 'j', 'PF', 'RM', 'T', 'c', 'JR', 'l', 'p', 'sp', 'CY', 'pb', 'TF', 'b', 'eg', 'rb', 'PK', 'sn', 'nr', 'pp', 'CJ', 'eb', 'SA', 'y', 'RI', 'lh', 'jd', 'OI', 'WH', 'ss', 'ru', 'zn', 'fb', 'rr', 'PM', 'au', 'TA', 'ZC', 'IC', 'bu', 'SM', 'wr', 'cs', 'LR', 'ag', 'i'}\n", + "len(all_init)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "db_init = sess.database(dbName='db_init', partitionType=keys.VALUE, partitions=list(all_init),dbPath='')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "db_code = sess.database(dbName='db_code', partitionType=keys.HASH, partitions=list(all_init),dbPath='')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DatetimeIndex(['1990-01-31', '1990-02-28', '1990-03-31', '1990-04-30',\n", + " '1990-05-31', '1990-06-30', '1990-07-31', '1990-08-31',\n", + " '1990-09-30', '1990-10-31',\n", + " ...\n", + " '2050-03-31', '2050-04-30', '2050-05-31', '2050-06-30',\n", + " '2050-07-31', '2050-08-31', '2050-09-30', '2050-10-31',\n", + " '2050-11-30', '2050-12-31'],\n", + " dtype='datetime64[ns]', length=732, freq='M')\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "dr = pd.date_range(start='1990-01-01', end='2050-12-31', freq=\"M\")\n", + "months=np.array(dr, dtype=\"datetime64[M]\")\n", + "print(dr)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['1990-01' '1990-02' '1990-03' '1990-04' '1990-05' '1990-06' '1990-07'\n", + " '1990-08' '1990-09' '1990-10' '1990-11' '1990-12' '1991-01' '1991-02'\n", + " '1991-03' '1991-04' '1991-05' '1991-06' '1991-07' '1991-08' '1991-09'\n", + " '1991-10' '1991-11' '1991-12' '1992-01' '1992-02' '1992-03' '1992-04'\n", + " '1992-05' '1992-06' '1992-07' '1992-08' '1992-09' '1992-10' '1992-11'\n", + " '1992-12' '1993-01' '1993-02' '1993-03' '1993-04' '1993-05' '1993-06'\n", + " '1993-07' '1993-08' '1993-09' '1993-10' '1993-11' '1993-12' '1994-01'\n", + " '1994-02' '1994-03' '1994-04' '1994-05' '1994-06' '1994-07' '1994-08'\n", + " '1994-09' '1994-10' '1994-11' '1994-12' '1995-01' '1995-02' '1995-03'\n", + " '1995-04' '1995-05' '1995-06' '1995-07' '1995-08' '1995-09' '1995-10'\n", + " '1995-11' '1995-12' '1996-01' '1996-02' '1996-03' '1996-04' '1996-05'\n", + " '1996-06' '1996-07' '1996-08' '1996-09' '1996-10' '1996-11' '1996-12'\n", + " '1997-01' '1997-02' '1997-03' '1997-04' '1997-05' '1997-06' '1997-07'\n", + " '1997-08' '1997-09' '1997-10' '1997-11' '1997-12' '1998-01' '1998-02'\n", + " '1998-03' '1998-04' '1998-05' '1998-06' '1998-07' '1998-08' '1998-09'\n", + " '1998-10' '1998-11' '1998-12' '1999-01' '1999-02' '1999-03' '1999-04'\n", + " '1999-05' '1999-06' '1999-07' '1999-08' '1999-09' '1999-10' '1999-11'\n", + " '1999-12' '2000-01' '2000-02' '2000-03' '2000-04' '2000-05' '2000-06'\n", + " '2000-07' '2000-08' '2000-09' '2000-10' '2000-11' '2000-12' '2001-01'\n", + " '2001-02' '2001-03' '2001-04' '2001-05' '2001-06' '2001-07' '2001-08'\n", + " '2001-09' '2001-10' '2001-11' '2001-12' '2002-01' '2002-02' '2002-03'\n", + " '2002-04' '2002-05' '2002-06' '2002-07' '2002-08' '2002-09' '2002-10'\n", + " '2002-11' '2002-12' '2003-01' '2003-02' '2003-03' '2003-04' '2003-05'\n", + " '2003-06' '2003-07' '2003-08' '2003-09' '2003-10' '2003-11' '2003-12'\n", + " '2004-01' '2004-02' '2004-03' '2004-04' '2004-05' '2004-06' '2004-07'\n", + " '2004-08' '2004-09' '2004-10' '2004-11' '2004-12' '2005-01' '2005-02'\n", + " '2005-03' '2005-04' '2005-05' '2005-06' '2005-07' '2005-08' '2005-09'\n", + " '2005-10' '2005-11' '2005-12' '2006-01' '2006-02' '2006-03' '2006-04'\n", + " '2006-05' '2006-06' '2006-07' '2006-08' '2006-09' '2006-10' '2006-11'\n", + " '2006-12' '2007-01' '2007-02' '2007-03' '2007-04' '2007-05' '2007-06'\n", + " '2007-07' '2007-08' '2007-09' '2007-10' '2007-11' '2007-12' '2008-01'\n", + " '2008-02' '2008-03' '2008-04' '2008-05' '2008-06' '2008-07' '2008-08'\n", + " '2008-09' '2008-10' '2008-11' '2008-12' '2009-01' '2009-02' '2009-03'\n", + " '2009-04' '2009-05' '2009-06' '2009-07' '2009-08' '2009-09' '2009-10'\n", + " '2009-11' '2009-12' '2010-01' '2010-02' '2010-03' '2010-04' '2010-05'\n", + " '2010-06' '2010-07' '2010-08' '2010-09' '2010-10' '2010-11' '2010-12'\n", + " '2011-01' '2011-02' '2011-03' '2011-04' '2011-05' '2011-06' '2011-07'\n", + " '2011-08' '2011-09' '2011-10' '2011-11' '2011-12' '2012-01' '2012-02'\n", + " '2012-03' '2012-04' '2012-05' '2012-06' '2012-07' '2012-08' '2012-09'\n", + " '2012-10' '2012-11' '2012-12' '2013-01' '2013-02' '2013-03' '2013-04'\n", + " '2013-05' '2013-06' '2013-07' '2013-08' '2013-09' '2013-10' '2013-11'\n", + " '2013-12' '2014-01' '2014-02' '2014-03' '2014-04' '2014-05' '2014-06'\n", + " '2014-07' '2014-08' '2014-09' '2014-10' '2014-11' '2014-12' '2015-01'\n", + " '2015-02' '2015-03' '2015-04' '2015-05' '2015-06' '2015-07' '2015-08'\n", + " '2015-09' '2015-10' '2015-11' '2015-12' '2016-01' '2016-02' '2016-03'\n", + " '2016-04' '2016-05' '2016-06' '2016-07' '2016-08' '2016-09' '2016-10'\n", + " '2016-11' '2016-12' '2017-01' '2017-02' '2017-03' '2017-04' '2017-05'\n", + " '2017-06' '2017-07' '2017-08' '2017-09' '2017-10' '2017-11' '2017-12'\n", + " '2018-01' '2018-02' '2018-03' '2018-04' '2018-05' '2018-06' '2018-07'\n", + " '2018-08' '2018-09' '2018-10' '2018-11' '2018-12' '2019-01' '2019-02'\n", + " '2019-03' '2019-04' '2019-05' '2019-06' '2019-07' '2019-08' '2019-09'\n", + " '2019-10' '2019-11' '2019-12' '2020-01' '2020-02' '2020-03' '2020-04'\n", + " '2020-05' '2020-06' '2020-07' '2020-08' '2020-09' '2020-10' '2020-11'\n", + " '2020-12' '2021-01' '2021-02' '2021-03' '2021-04' '2021-05' '2021-06'\n", + " '2021-07' '2021-08' '2021-09' '2021-10' '2021-11' '2021-12' '2022-01'\n", + " '2022-02' '2022-03' '2022-04' '2022-05' '2022-06' '2022-07' '2022-08'\n", + " '2022-09' '2022-10' '2022-11' '2022-12' '2023-01' '2023-02' '2023-03'\n", + " '2023-04' '2023-05' '2023-06' '2023-07' '2023-08' '2023-09' '2023-10'\n", + " '2023-11' '2023-12' '2024-01' '2024-02' '2024-03' '2024-04' '2024-05'\n", + " '2024-06' '2024-07' '2024-08' '2024-09' '2024-10' '2024-11' '2024-12'\n", + " '2025-01' '2025-02' '2025-03' '2025-04' '2025-05' '2025-06' '2025-07'\n", + " '2025-08' '2025-09' '2025-10' '2025-11' '2025-12' '2026-01' '2026-02'\n", + " '2026-03' '2026-04' '2026-05' '2026-06' '2026-07' '2026-08' '2026-09'\n", + " '2026-10' '2026-11' '2026-12' '2027-01' '2027-02' '2027-03' '2027-04'\n", + " '2027-05' '2027-06' '2027-07' '2027-08' '2027-09' '2027-10' '2027-11'\n", + " '2027-12' '2028-01' '2028-02' '2028-03' '2028-04' '2028-05' '2028-06'\n", + " '2028-07' '2028-08' '2028-09' '2028-10' '2028-11' '2028-12' '2029-01'\n", + " '2029-02' '2029-03' '2029-04' '2029-05' '2029-06' '2029-07' '2029-08'\n", + " '2029-09' '2029-10' '2029-11' '2029-12' '2030-01' '2030-02' '2030-03'\n", + " '2030-04' '2030-05' '2030-06' '2030-07' '2030-08' '2030-09' '2030-10'\n", + " '2030-11' '2030-12' '2031-01' '2031-02' '2031-03' '2031-04' '2031-05'\n", + " '2031-06' '2031-07' '2031-08' '2031-09' '2031-10' '2031-11' '2031-12'\n", + " '2032-01' '2032-02' '2032-03' '2032-04' '2032-05' '2032-06' '2032-07'\n", + " '2032-08' '2032-09' '2032-10' '2032-11' '2032-12' '2033-01' '2033-02'\n", + " '2033-03' '2033-04' '2033-05' '2033-06' '2033-07' '2033-08' '2033-09'\n", + " '2033-10' '2033-11' '2033-12' '2034-01' '2034-02' '2034-03' '2034-04'\n", + " '2034-05' '2034-06' '2034-07' '2034-08' '2034-09' '2034-10' '2034-11'\n", + " '2034-12' '2035-01' '2035-02' '2035-03' '2035-04' '2035-05' '2035-06'\n", + " '2035-07' '2035-08' '2035-09' '2035-10' '2035-11' '2035-12' '2036-01'\n", + " '2036-02' '2036-03' '2036-04' '2036-05' '2036-06' '2036-07' '2036-08'\n", + " '2036-09' '2036-10' '2036-11' '2036-12' '2037-01' '2037-02' '2037-03'\n", + " '2037-04' '2037-05' '2037-06' '2037-07' '2037-08' '2037-09' '2037-10'\n", + " '2037-11' '2037-12' '2038-01' '2038-02' '2038-03' '2038-04' '2038-05'\n", + " '2038-06' '2038-07' '2038-08' '2038-09' '2038-10' '2038-11' '2038-12'\n", + " '2039-01' '2039-02' '2039-03' '2039-04' '2039-05' '2039-06' '2039-07'\n", + " '2039-08' '2039-09' '2039-10' '2039-11' '2039-12' '2040-01' '2040-02'\n", + " '2040-03' '2040-04' '2040-05' '2040-06' '2040-07' '2040-08' '2040-09'\n", + " '2040-10' '2040-11' '2040-12' '2041-01' '2041-02' '2041-03' '2041-04'\n", + " '2041-05' '2041-06' '2041-07' '2041-08' '2041-09' '2041-10' '2041-11'\n", + " '2041-12' '2042-01' '2042-02' '2042-03' '2042-04' '2042-05' '2042-06'\n", + " '2042-07' '2042-08' '2042-09' '2042-10' '2042-11' '2042-12' '2043-01'\n", + " '2043-02' '2043-03' '2043-04' '2043-05' '2043-06' '2043-07' '2043-08'\n", + " '2043-09' '2043-10' '2043-11' '2043-12' '2044-01' '2044-02' '2044-03'\n", + " '2044-04' '2044-05' '2044-06' '2044-07' '2044-08' '2044-09' '2044-10'\n", + " '2044-11' '2044-12' '2045-01' '2045-02' '2045-03' '2045-04' '2045-05'\n", + " '2045-06' '2045-07' '2045-08' '2045-09' '2045-10' '2045-11' '2045-12'\n", + " '2046-01' '2046-02' '2046-03' '2046-04' '2046-05' '2046-06' '2046-07'\n", + " '2046-08' '2046-09' '2046-10' '2046-11' '2046-12' '2047-01' '2047-02'\n", + " '2047-03' '2047-04' '2047-05' '2047-06' '2047-07' '2047-08' '2047-09'\n", + " '2047-10' '2047-11' '2047-12' '2048-01' '2048-02' '2048-03' '2048-04'\n", + " '2048-05' '2048-06' '2048-07' '2048-08' '2048-09' '2048-10' '2048-11'\n", + " '2048-12' '2049-01' '2049-02' '2049-03' '2049-04' '2049-05' '2049-06'\n", + " '2049-07' '2049-08' '2049-09' '2049-10' '2049-11' '2049-12' '2050-01'\n", + " '2050-02' '2050-03' '2050-04' '2050-05' '2050-06' '2050-07' '2050-08'\n", + " '2050-09' '2050-10' '2050-11' '2050-12']\n" + ] + } + ], + "source": [ + "months=np.array(pd.date_range(start='2000-01-01', end='2050-12-31', freq=\"M\"), dtype=\"datetime64[M]\")\n", + "print(months)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "db_date = sess.database('db_date', partitionType=keys.VALUE, partitions=months, dbPath='')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "dbPath = 'dfs://hft_fm_combo'\n", + "if sess.existsDatabase(dbPath): \n", + " sess.dropDatabase(dbPath)\n", + "db = sess.database(dbName='db_hft_fm_3', partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# sess.run(\"\"\"\n", + "# {dbName} = database(\n", + "# directory = '{dbPath}', \n", + "# partitionType = COMPO, \n", + "# partitionScheme = [db_date, db_init])\n", + "# \"\"\".format(\n", + "# dbName = 'db_hft_fm_3',\n", + "# dbPath = 'hft_futuremarket_ts_combo'\n", + "# ))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "t = sess.table(data=df,tableAliasName='table')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sess.run(\"\"\"\n", + " {ddb_dbname}.createPartitionedTable(\n", + " table = {memory_table_name}, \n", + " tableName = `{partition_table_name}, \n", + " partitionColumns = `date`StockID, \n", + " sortColumns = `StockID`date,\n", + " compressMethods = {{date:\"delta\"}}\n", + " )\n", + " \"\"\".format(\n", + " ddb_dbname = 'db_hft_fm_3',\n", + " memory_table_name = 'testData',\n", + " partition_table_name = 'pt3'\n", + " ))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "pt =db.createPartitionedTable(table=t, tableName=\"pt\", partitionColumns=['date', 'init'])" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pt.append(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'/hft_fm_combo': 'pt:2:0:pt_2; ',\n", + " '/db_compo_test': 'pt:2:0:pt_2; ',\n", + " '/compoDB': 'pt:2:0:pt_2; ',\n", + " '/daily_futuremarket_ts': 'db_daily_kline:2:0:db_daily_kline_2; '}" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sess.run('getAllDBs()')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame({'date': np.array(['2012-01-01', '2012-02-01', '2012-05-01', '2012-06-01'], dtype=\"datetime64\"), 'val':[1,2,3,4],'code':['AP2313','FD1023','AP1023','ED2333']})\n", + "df\n", + "df['init']=df['code'].apply(lambda x: x[:2])\n", + "df\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "dbPath=\"dfs://db_compo_test\"\n", + "if sess.existsDatabase(dbPath):\n", + " sess.dropDatabase(dbPath)\n", + "\n", + "months=np.array(pd.date_range(start='2012-01', end='2012-10', freq=\"M\"), dtype=\"datetime64[M]\")\n", + "\n", + "db1 = sess.database('db1', partitionType=keys.VALUE,partitions=months, dbPath='')\n", + "\n", + "db2 = sess.database('db2', partitionType=keys.RANGE,partitions=[1, 6, 11], dbPath='')\n", + "# dbPath=\"dfs://db_compo_test\"\n", + "if sess.existsDatabase(dbPath):\n", + " sess.dropDatabase(dbPath)\n", + "db = sess.database(dbName='mydb', partitionType=keys.COMPO, partitions=[db1, db2], dbPath=dbPath)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = pd.DataFrame({'date':np.array(['2012-01-01', '2012-01-01', '2012-02-06', '2012-03-06'], dtype='datetime64'), 'val': [1, 6, 1, 6]})\n", + "t = sess.table(data=df2)\n", + "# df['date'].dtype()\n", + "df2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "db.createPartitionedTable(table=t, tableName='pt', partitionColumns=['date', 'val']).append(t)\n", + "re = sess.loadTable(tableName='pt', dbPath=dbPath).toDF()" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-11-09 14:48:51.524 | INFO | src.TSLfm:__enter__:18 - Logging in TSL.\n", + "2022-11-09 14:48:51.708 | DEBUG | src.TSLfm:get_mkt_min_k:209 - \n", + " SetSysParam(pn_cycle(), cy_1m());\n", + " stock_list := Array('CF2211', 'CF2301', 'CF2303', 'CF2305', 'CF2307', 'CF2309');\n", + " r := select \n", + " ['StockID'] as 'code',\n", + " DateTimeToStr(['date']) as 'm_nDatetime',\n", + "\n", + " ['price'] as 'm_nPrice',\n", + " ['open'] as 'm_nOpen',\n", + " ['high'] as 'm_nHigh',\n", + " ['low'] as 'm_nLow',\n", + " ['close'] as 'm_nClose',\n", + "\n", + " ['sectional_high'] as 'm_nAccHigh',\n", + " ['sectional_low'] as 'm_nAccLow',\n", + "\n", + " ['vol'] as 'm_iVolume', //成交量\n", + " ['sectional_vol'] as 'm_iAccVolume', //时点当日累计成交量\n", + "\n", + " ['cjbs'] as 'm_nMatchItems', // 成交笔数 周期内的持仓的变动量\n", + " ['sectional_cjbs'] as 'm_nAccMatchItems',\n", + "\n", + " ['amount'] as 'm_iTurnover', //成交金额\n", + " ['sectional_amount'] as 'm_iAccTurnover', \n", + "\n", + " ['yclose'] as 'm_nPreClose', //上一周期的收盘价\n", + " ['sectional_yclose'] as 'm_nAccPreClose', // 前日收盘\n", + " \n", + " ['buy1'] as 'm_nBidPrice', //买一价？叫卖价?\n", + " ['bc1'] as 'm_nBidVolume', //买一量 当前以买一价出价的委买量\n", + " ['sale1'] as 'm_nAskPrice', \n", + " ['sc1'] as 'm_nAskVolume', \n", + "\n", + " ['zmm'] as 'm_iABFlag', //买卖标识\n", + "\n", + " ['buy_vol'] as 'm_nActBidVolume', //主买量\n", + " ['sectional_buy_vol'] as 'm_nAccActBidVolume', //时点当日累计主买量\n", + " ['buy_amount'] as 'm_nActBidTurnover', //主买金额\n", + " ['sectional_buy_amount'] as 'm_nAccActBidTurnover', \n", + "\n", + " ['sale_vol'] as 'm_nActAskVolume', \n", + " ['sectional_sale_vol'] as 'm_nAccActAskVolume', \n", + " ['sale_amount'] as 'm_nActAskTurnover',\n", + " ['sectional_sale_amount'] as 'm_nAccActAskTurnover', \n", + "\n", + " ['w_buy'] as 'm_nBidOrder', //委买\n", + " ['sectional_w_buy'] as 'm_nAccBidOrder', \n", + " ['w_sale'] as 'm_nAskOrder',\n", + " ['sectional_w_sale'] as 'm_nAccAskOrder',\n", + "\n", + " ['wb'] as 'm_nABOrderRate', //委比\n", + " ['sectional_wb'] as 'm_nAccABOrderRate', //时点当日累计委比\n", + " ['lb'] as 'm_nMItemsVolRate'//量比\n", + "\n", + "\n", + " from markettable\n", + " datekey 20221001T to 20221101T+0.999 \n", + " of stock_list\n", + " end;\n", + " \n", + " return r; \n", + " \n", + "2022-11-09 14:49:05.395 | INFO | src.TSLfm:process_result_data_type:215 - Processing new df of shape (35910, 37), which looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", + "0 13445.0 13445.0 0.0 0.0 0.0 \n", + "1 13445.0 13445.0 0.0 0.0 0.0 \n", + "2 13445.0 13445.0 0.0 0.0 0.0 \n", + "3 13445.0 13445.0 0.0 0.0 0.0 \n", + "4 13445.0 13445.0 0.0 0.0 0.0 \n", + "\n", + " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", + "0 96132 0.0 0.0 13445.0 13445.0 \n", + "1 96132 0.0 0.0 13445.0 13445.0 \n", + "2 96132 0.0 0.0 13445.0 13445.0 \n", + "3 96132 0.0 0.0 13445.0 13445.0 \n", + "4 96132 0.0 0.0 13445.0 13445.0 \n", + "\n", + " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", + "0 0.0 0 0.0 0 0 \n", + "1 0.0 0 0.0 0 0 \n", + "2 0.0 0 0.0 0 0 \n", + "3 0.0 0 0.0 0 0 \n", + "4 0.0 0 0.0 0 0 \n", + "\n", + " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "\n", + " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "\n", + " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nMItemsVolRate \n", + "0 0.0 \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "2022-11-09 14:49:05.493 | INFO | src.TSLfm:process_result_data_type:229 - Processing done, new df looks like\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", + "0 13445.0 13445.0 0 0 0 \n", + "1 13445.0 13445.0 0 0 0 \n", + "2 13445.0 13445.0 0 0 0 \n", + "3 13445.0 13445.0 0 0 0 \n", + "4 13445.0 13445.0 0 0 0 \n", + "\n", + " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", + "0 30596 0.0 0.0 13445.0 13445.0 \n", + "1 30596 0.0 0.0 13445.0 13445.0 \n", + "2 30596 0.0 0.0 13445.0 13445.0 \n", + "3 30596 0.0 0.0 13445.0 13445.0 \n", + "4 30596 0.0 0.0 13445.0 13445.0 \n", + "\n", + " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", + "0 0.0 0 0.0 0 0 \n", + "1 0.0 0 0.0 0 0 \n", + "2 0.0 0 0.0 0 0 \n", + "3 0.0 0 0.0 0 0 \n", + "4 0.0 0 0.0 0 0 \n", + "\n", + " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", + "0 0 0 0.0 \n", + "1 0 0 0.0 \n", + "2 0 0 0.0 \n", + "3 0 0 0.0 \n", + "4 0 0 0.0 \n", + "\n", + " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", + "0 0.0 0 0 \n", + "1 0.0 0 0 \n", + "2 0.0 0 0 \n", + "3 0.0 0 0 \n", + "4 0.0 0 0 \n", + "\n", + " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nMItemsVolRate m_nDate code_init \n", + "0 0.0 2022-10-07 CF \n", + "1 0.0 2022-10-07 CF \n", + "2 0.0 2022-10-07 CF \n", + "3 0.0 2022-10-07 CF \n", + "4 0.0 2022-10-07 CF \n", + "2022-11-09 14:49:05.501 | INFO | src.TSLfm:__exit__:24 - Logging out TSL.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(35910, 39)\n", + " code m_nDatetime m_nPrice m_nOpen m_nHigh m_nLow m_nClose \\\n", + "0 CF2303 2022-10-07 21:01:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "1 CF2303 2022-10-07 21:02:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "2 CF2303 2022-10-07 21:03:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "3 CF2303 2022-10-07 21:04:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "4 CF2303 2022-10-07 21:05:00 13445.0 13445.0 13445.0 13445.0 13445.0 \n", + "\n", + " m_nAccHigh m_nAccLow m_iVolume m_iAccVolume m_nMatchItems \\\n", + "0 13445.0 13445.0 0 0 0 \n", + "1 13445.0 13445.0 0 0 0 \n", + "2 13445.0 13445.0 0 0 0 \n", + "3 13445.0 13445.0 0 0 0 \n", + "4 13445.0 13445.0 0 0 0 \n", + "\n", + " m_nAccMatchItems m_iTurnover m_iAccTurnover m_nPreClose m_nAccPreClose \\\n", + "0 30596 0.0 0.0 13445.0 13445.0 \n", + "1 30596 0.0 0.0 13445.0 13445.0 \n", + "2 30596 0.0 0.0 13445.0 13445.0 \n", + "3 30596 0.0 0.0 13445.0 13445.0 \n", + "4 30596 0.0 0.0 13445.0 13445.0 \n", + "\n", + " m_nBidPrice m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag \\\n", + "0 0.0 0 0.0 0 0 \n", + "1 0.0 0 0.0 0 0 \n", + "2 0.0 0 0.0 0 0 \n", + "3 0.0 0 0.0 0 0 \n", + "4 0.0 0 0.0 0 0 \n", + "\n", + " m_nActBidVolume m_nAccActBidVolume m_nActBidTurnover \\\n", + "0 0 0 0.0 \n", + "1 0 0 0.0 \n", + "2 0 0 0.0 \n", + "3 0 0 0.0 \n", + "4 0 0 0.0 \n", + "\n", + " m_nAccActBidTurnover m_nActAskVolume m_nAccActAskVolume \\\n", + "0 0.0 0 0 \n", + "1 0.0 0 0 \n", + "2 0.0 0 0 \n", + "3 0.0 0 0 \n", + "4 0.0 0 0 \n", + "\n", + " m_nActAskTurnover m_nAccActAskTurnover m_nBidOrder m_nAccBidOrder \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nAskOrder m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate \\\n", + "0 0.0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 0.0 \n", + "\n", + " m_nMItemsVolRate m_nDate code_init \n", + "0 0.0 2022-10-07 CF \n", + "1 0.0 2022-10-07 CF \n", + "2 0.0 2022-10-07 CF \n", + "3 0.0 2022-10-07 CF \n", + "4 0.0 2022-10-07 CF \n" + ] + } + ], + "source": [ + "\n", + "from src.TSLfm import TSLfm\n", + "import pandas as pd \n", + "import numpy as np\n", + "pd.set_option('display.max_columns', 100)\n", + "\n", + "with TSLfm() as tsl:\n", + " \n", + " # t_list= tsl.get_code_list()\n", + " t_list=['CF2211', 'CF2301', 'CF2303', 'CF2305', 'CF2307', 'CF2309']\n", + " # t_list=['AP2212']\n", + " # t_list=['CF2211']\n", + " df = tsl.process_result_data_type(tsl.get_mkt_min_k('20221001','20221101',t_list))\n", + " \n", + " print(df.shape)\n", + " print(df.head())\n", + "\n", + "df.to_csv('data/CF202210.csv')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
codem_nPricem_nOpenm_nHighm_nLowm_nClosem_nAccHighm_nAccLowm_iVolumem_iAccVolumem_nMatchItemsm_nAccMatchItemsm_iTuroverm_iAccTuroverm_nPreClosem_nAccPreClosem_nBidPricem_nBidVolumem_nAskPricem_nAskVolumem_iABFlagm_nActBidVolumem_nAccActBidVolumem_nActBidTuroverm_nAccActBidTuroverm_nActAskVolumem_nAccActAskVolumem_nActAskTuroverm_nAccActAskTuroverm_nBidOrderm_nAccBidOrderm_nAskOrderm_nAccAskOrderm_nABOrderRatem_nAccABOrderRatem_nMItemsVolRatem_nDatem_nTimecode_init
0CF221113400.013550.013550.013400.013400.013645.013400.028754-22.048543759101022047013550.013580.013400.0813550.020122.0348.0294580.04718285.06.0406.081330.05502185.0266.040331.0324.04755.00.4000008.4818090.02022-10-3109:01:00CF
1CF221113435.013400.013545.013400.013435.013645.013400.03757-1.04853406651026113513400.013580.013410.06413550.02002.0350.027110.04745395.01.0407.013555.05515740.0741.041072.0388.05143.03.2000007.9860000.02022-10-3109:02:00CF
2CF221113410.013435.013435.013410.013410.013645.013400.03760-3.04850406651030180013435.013580.013410.06213545.0100.0350.00.04745395.03.0410.040665.05556405.01335.042407.0428.05571.062.0000007.6120980.02022-10-3109:03:00CF
3CF221113545.013410.013545.013410.013545.013645.013400.03763-1.04849406651034246513410.013580.013545.0913550.03702.0352.027110.04772505.01.0411.013555.05569960.0580.042987.0278.05849.00.2432437.3494610.02022-10-3109:04:00CF
4CF221113520.013545.013545.013410.013520.013645.013400.023786-16.048333117651065423013545.013580.013425.0113515.0508.0360.0108440.04880945.015.0426.0203325.05773285.0759.043746.0160.06009.00.2000007.2800800.02022-10-3109:05:00CF
........................................................................................................................
685CF221113810.013810.013810.013810.013810.013810.013725.00110.01740015136013810.013600.013710.01013800.04000.011.00.0151360.00.00.00.00.050.015596.0200.013666.00.2500001.1412260.02022-11-0122:56:00CF
686CF221113810.013810.013810.013810.013810.013810.013725.00110.01740015136013810.013600.013725.01013800.04000.011.00.0151360.00.00.00.00.052.015648.0280.013946.00.2500001.1220420.02022-11-0122:57:00CF
687CF221113810.013810.013810.013810.013810.013810.013725.00110.01740015136013810.013600.013725.01013800.04000.011.00.0151360.00.00.00.00.020.015668.080.014026.00.2500001.1170680.02022-11-0122:58:00CF
688CF221113810.013810.013810.013810.013810.013810.013725.00110.01740015136013810.013600.013725.01013800.04000.011.00.0151360.00.00.00.00.020.015688.080.014106.00.2500001.1121510.02022-11-0122:59:00CF
689CF221113810.013810.013810.013810.013810.013810.013725.00110.01740015136013810.013600.013740.01013800.04000.011.00.0151360.00.00.00.00.084.015772.0360.014466.00.2500001.0902810.02022-11-0123:00:00CF
\n", + "

690 rows × 39 columns

\n", + "
" + ], + "text/plain": [ + " code m_nPrice m_nOpen m_nHigh m_nLow m_nClose m_nAccHigh \\\n", + "0 CF2211 13400.0 13550.0 13550.0 13400.0 13400.0 13645.0 \n", + "1 CF2211 13435.0 13400.0 13545.0 13400.0 13435.0 13645.0 \n", + "2 CF2211 13410.0 13435.0 13435.0 13410.0 13410.0 13645.0 \n", + "3 CF2211 13545.0 13410.0 13545.0 13410.0 13545.0 13645.0 \n", + "4 CF2211 13520.0 13545.0 13545.0 13410.0 13520.0 13645.0 \n", + ".. ... ... ... ... ... ... ... \n", + "685 CF2211 13810.0 13810.0 13810.0 13810.0 13810.0 13810.0 \n", + "686 CF2211 13810.0 13810.0 13810.0 13810.0 13810.0 13810.0 \n", + "687 CF2211 13810.0 13810.0 13810.0 13810.0 13810.0 13810.0 \n", + "688 CF2211 13810.0 13810.0 13810.0 13810.0 13810.0 13810.0 \n", + "689 CF2211 13810.0 13810.0 13810.0 13810.0 13810.0 13810.0 \n", + "\n", + " m_nAccLow m_iVolume m_iAccVolume m_nMatchItems m_nAccMatchItems \\\n", + "0 13400.0 28 754 -22.0 4854 \n", + "1 13400.0 3 757 -1.0 4853 \n", + "2 13400.0 3 760 -3.0 4850 \n", + "3 13400.0 3 763 -1.0 4849 \n", + "4 13400.0 23 786 -16.0 4833 \n", + ".. ... ... ... ... ... \n", + "685 13725.0 0 11 0.0 1740 \n", + "686 13725.0 0 11 0.0 1740 \n", + "687 13725.0 0 11 0.0 1740 \n", + "688 13725.0 0 11 0.0 1740 \n", + "689 13725.0 0 11 0.0 1740 \n", + "\n", + " m_iTurover m_iAccTurover m_nPreClose m_nAccPreClose m_nBidPrice \\\n", + "0 375910 10220470 13550.0 13580.0 13400.0 \n", + "1 40665 10261135 13400.0 13580.0 13410.0 \n", + "2 40665 10301800 13435.0 13580.0 13410.0 \n", + "3 40665 10342465 13410.0 13580.0 13545.0 \n", + "4 311765 10654230 13545.0 13580.0 13425.0 \n", + ".. ... ... ... ... ... \n", + "685 0 151360 13810.0 13600.0 13710.0 \n", + "686 0 151360 13810.0 13600.0 13725.0 \n", + "687 0 151360 13810.0 13600.0 13725.0 \n", + "688 0 151360 13810.0 13600.0 13725.0 \n", + "689 0 151360 13810.0 13600.0 13740.0 \n", + "\n", + " m_nBidVolume m_nAskPrice m_nAskVolume m_iABFlag m_nActBidVolume \\\n", + "0 8 13550.0 20 1 22.0 \n", + "1 64 13550.0 20 0 2.0 \n", + "2 62 13545.0 1 0 0.0 \n", + "3 9 13550.0 37 0 2.0 \n", + "4 1 13515.0 5 0 8.0 \n", + ".. ... ... ... ... ... \n", + "685 10 13800.0 40 0 0.0 \n", + "686 10 13800.0 40 0 0.0 \n", + "687 10 13800.0 40 0 0.0 \n", + "688 10 13800.0 40 0 0.0 \n", + "689 10 13800.0 40 0 0.0 \n", + "\n", + " m_nAccActBidVolume m_nActBidTurover m_nAccActBidTurover \\\n", + "0 348.0 294580.0 4718285.0 \n", + "1 350.0 27110.0 4745395.0 \n", + "2 350.0 0.0 4745395.0 \n", + "3 352.0 27110.0 4772505.0 \n", + "4 360.0 108440.0 4880945.0 \n", + ".. ... ... ... \n", + "685 11.0 0.0 151360.0 \n", + "686 11.0 0.0 151360.0 \n", + "687 11.0 0.0 151360.0 \n", + "688 11.0 0.0 151360.0 \n", + "689 11.0 0.0 151360.0 \n", + "\n", + " m_nActAskVolume m_nAccActAskVolume m_nActAskTurover \\\n", + "0 6.0 406.0 81330.0 \n", + "1 1.0 407.0 13555.0 \n", + "2 3.0 410.0 40665.0 \n", + "3 1.0 411.0 13555.0 \n", + "4 15.0 426.0 203325.0 \n", + ".. ... ... ... \n", + "685 0.0 0.0 0.0 \n", + "686 0.0 0.0 0.0 \n", + "687 0.0 0.0 0.0 \n", + "688 0.0 0.0 0.0 \n", + "689 0.0 0.0 0.0 \n", + "\n", + " m_nAccActAskTurover m_nBidOrder m_nAccBidOrder m_nAskOrder \\\n", + "0 5502185.0 266.0 40331.0 324.0 \n", + "1 5515740.0 741.0 41072.0 388.0 \n", + "2 5556405.0 1335.0 42407.0 428.0 \n", + "3 5569960.0 580.0 42987.0 278.0 \n", + "4 5773285.0 759.0 43746.0 160.0 \n", + ".. ... ... ... ... \n", + "685 0.0 50.0 15596.0 200.0 \n", + "686 0.0 52.0 15648.0 280.0 \n", + "687 0.0 20.0 15668.0 80.0 \n", + "688 0.0 20.0 15688.0 80.0 \n", + "689 0.0 84.0 15772.0 360.0 \n", + "\n", + " m_nAccAskOrder m_nABOrderRate m_nAccABOrderRate m_nMItemsVolRate \\\n", + "0 4755.0 0.400000 8.481809 0.0 \n", + "1 5143.0 3.200000 7.986000 0.0 \n", + "2 5571.0 62.000000 7.612098 0.0 \n", + "3 5849.0 0.243243 7.349461 0.0 \n", + "4 6009.0 0.200000 7.280080 0.0 \n", + ".. ... ... ... ... \n", + "685 13666.0 0.250000 1.141226 0.0 \n", + "686 13946.0 0.250000 1.122042 0.0 \n", + "687 14026.0 0.250000 1.117068 0.0 \n", + "688 14106.0 0.250000 1.112151 0.0 \n", + "689 14466.0 0.250000 1.090281 0.0 \n", + "\n", + " m_nDate m_nTime code_init \n", + "0 2022-10-31 09:01:00 CF \n", + "1 2022-10-31 09:02:00 CF \n", + "2 2022-10-31 09:03:00 CF \n", + "3 2022-10-31 09:04:00 CF \n", + "4 2022-10-31 09:05:00 CF \n", + ".. ... ... ... \n", + "685 2022-11-01 22:56:00 CF \n", + "686 2022-11-01 22:57:00 CF \n", + "687 2022-11-01 22:58:00 CF \n", + "688 2022-11-01 22:59:00 CF \n", + "689 2022-11-01 23:00:00 CF \n", + "\n", + "[690 rows x 39 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from usrc.DDBfm import DDBfm\n", + "ddb=DDBfm('dev')\n", + "\n", + "db = ddb.create_ddb_database(ddb.ddb_hft_path,ddb.ddb_hft_mink_dbname)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ddb.add_new_hft_table(db,ddb.ddf_hft_mink_tbname,df)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "ddb.append_hft_table(ddb.ddb_hft_path,ddb.ddf_hft_mink_tbname,df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ddb.ddb_sess.run(\"\"\"\n", + "# {table_name} = table({capacity}:0, {col_names}, [{col_types}]);\n", + "# \"\"\".format(\n", + "# table_name = ddb.ddf_hft_tick_tbname,\n", + "# capacity = 5000 * 1000,\n", + "# col_names = '`code`m_nDate',\n", + "# col_types = \"SYMBOL, DATE\"\n", + "# ))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.9 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/DDBfm.py b/src/DDBfm.py index cad02f9..8423491 100644 --- a/src/DDBfm.py +++ b/src/DDBfm.py @@ -2,7 +2,7 @@ import dolphindb as ddb import dolphindb.settings as keys import numpy as np import pandas as pd -# from loguru import logger +from loguru import logger class DDBfm(): @@ -28,17 +28,17 @@ class DDBfm(): ddb_daily_path = "dfs://daily_futuremarket_ts" ddb_daily_dbname = "db_daily_fm" - ddb_hft_path="dfs://hft_futuremarket_ts" - - ddb_hft_tick_dbname="Tick" + ddb_hft_dbPath="dfs://hft_futuremarket_ts" + ddb_hft_dbName="hft_futuremarket_ts" + ddf_hft_tick_tbname="TickPartitioned" - ddb_hft_mink_dbname="MinKline" ddf_hft_mink_tbname="MinKlinePartitioned" def __init__(self, which_server='local', **kwargs): + self.hft_db=None self.ddb_config = self.ddb_config_servers[which_server] self.sess = ddb.session(self.ddb_config['host'], 8848) @@ -48,26 +48,35 @@ class DDBfm(): if self.sess.existsDatabase(dbPath): self.sess.dropDatabase(dbPath) - def create_ddb_database(self,dbPath,dbName): + def create_hft_database(self): + dbPath=self.ddb_hft_dbPath + dbName=self.ddb_hft_dbName + + if self.sess.existsDatabase(dbPath): + self.sess.dropDatabase(dbPath) + db_init = self.sess.database(dbName='db_init', partitionType=keys.VALUE, partitions=self.all_fm_init, dbPath='') months=np.array(pd.date_range(start='2000-01', end='2050-12', freq="M"), dtype="datetime64[M]") - # print(months) + logger.debug(months) + db_date = self.sess.database('db_date', partitionType=keys.VALUE, partitions=months, dbPath='') if self.sess.existsDatabase(dbPath): self.sess.dropDatabase(dbPath) - db = self.sess.database(dbName=dbName, partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath, engine='OLAP') + self.hft_db = self.sess.database(dbName=dbName, partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath, engine='OLAP') + # todo: engine=TSDB - - return db + # db = self.sess.database(dbName=dbName, partitionType=keys.COMPO, partitions=[db_date, db_init], dbPath=dbPath, engine='TSDB') + logger.info(f'Created dbName:{dbName} at dbPath:{dbPath}') - def add_new_hft_table(self, db, tbName, df): - t = self.sess.table(data=df,tableAliasName=tbName) - pt =db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDate', 'code_init']) + def create_hft_table(self, tbName, df): + t = self.sess.table(data=df, tableAliasName=tbName) + pt = self.hft_db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDatetime', 'code_init']) + # pt = self.hft_db.createPartitionedTable(table=t, tableName=tbName, partitionColumns=['m_nDate', 'code_init'],sortColumns=["code","m_nDatetime"]) + pt.append(t) - def append_hft_table(self, dbPath, tbName, df): # load table to check? time&date&code appender = ddb.tableAppender(tableName=tbName, ddbSession=self.sess,dbPath=dbPath) diff --git a/src/TSLfm.py b/src/TSLfm.py index 68624df..4e40ac7 100644 --- a/src/TSLfm.py +++ b/src/TSLfm.py @@ -1,8 +1,6 @@ -import itertools -from datetime import datetime - import pandas as pd -from pprint import pprint +import numpy as np +from loguru import logger import pyTSL @@ -17,13 +15,13 @@ class TSLfm: self.timeout_default=100000000 def __enter__(self): - print('entering') + logger.info('Logging in TSL.') self.c = pyTSL.Client(TINYSOFT_USERNAME, TINYSOFT_PASSWORD, TINYSOFT_HOSTNAME, 443) self.c.login() return self def __exit__(self, exc_type, exc_val, exc_tb): - print('logging out') + logger.info('Logging out TSL.') self.c.logout() del(self.c) @@ -31,7 +29,7 @@ class TSLfm: scpt = """return getbk('{bk_name}'); """.format(bk_name=bk_name) - print(scpt) + logger.debug(scpt) r = self.c.exec(scpt,timeout=self.timeout_default) return r.value() @@ -75,7 +73,7 @@ class TSLfm: stock_list := Array({code_list_input}); r := select ['StockID'] as 'code', - DateTimeToStr(['date']) as 'datetime', + DateTimeToStr(['date']) as 'm_nDatetime', ['price'] as 'm_nPrice', ['open'] as 'm_nOpen', @@ -135,7 +133,7 @@ class TSLfm: start_date=start_date, end_date=end_date ) - print(scpt) + logger.debug(scpt) r = self.c.exec(scpt,timeout=self.timeout_default) return r @@ -147,7 +145,7 @@ class TSLfm: stock_list := Array({code_list_input}); r := select ['StockID'] as 'code', - DateTimeToStr(['date']) as 'datetime', + DateTimeToStr(['date']) as 'm_nDatetime', ['price'] as 'm_nPrice', ['open'] as 'm_nOpen', @@ -208,10 +206,35 @@ class TSLfm: start_date=start_date, end_date=end_date ) - print(scpt) + logger.debug(scpt) r = self.c.exec(scpt,timeout=self.timeout_default) return r def process_result_data_type(self, r): df = pd.DataFrame(r.value()) - print(df.shape) \ No newline at end of file + logger.info(f"Processing new df of shape {df.shape}, which looks like\n{df.head(5)}") + + # new = df["m_nDatetime"].str.split(" ", n = 1, expand = True) + df["m_nDatetime"]=df["m_nDatetime"].astype('datetime64[s]') + # df['m_nDate']=new[0].astype('datetime64[D]') + + df['code_init']=df['code'].apply(lambda x:x[:-4]) + + for col in ['m_iVolume','m_iAccVolume','m_nMatchItems','m_nAccMatchItems','m_nBidVolume','m_nAskVolume','m_nActBidVolume','m_nAccActBidVolume','m_nActAskVolume','m_nAccActAskVolume']: + df[col]=df[col].astype(np.int16) + + for col in ['m_iABFlag']: + df[col]=df[col].astype(np.int8) + + logger.info(f"Processing done, new df looks like\n{df.head(5)}") + + return df + +if __name__ == '__main__': + from loguru import logger + logger.add("../logs/{time:YYYYMMDD-HHmmss}_TSLfm.log", rotation="10 MB", compression="zip", level="INFO") + + with TSLfm() as tsl: + t_list=['CF2211'] + df = tsl.process_result_data_type(tsl.get_mkt_min_k('20221031','20221101',t_list)) + print(df) diff --git a/src/data_loader.py b/src/data_loader.py index e69de29..9551b47 100644 --- a/src/data_loader.py +++ b/src/data_loader.py @@ -0,0 +1,40 @@ +import sys +# import os + +from loguru import logger +from os.path import dirname, abspath, join +ROOT_DIR = abspath(join(dirname(abspath(__file__)), "..")) + +from loguru import logger +logger.remove() +logger.add(sys.stderr, level="INFO") +logger.add(ROOT_DIR+"/logs/{time:YYYYMMDD-HHmmss}.log", rotation="10 MB", compression="zip", level="INFO") + +from DDBfm import DDBfm +from TSLfm import TSLfm + + +def main(): + df=None + + with TSLfm() as tsl: + # code_list = tsl.get_code_list("国债期货") + # code_list += tsl.get_code_list("股指期货") + # code_list += tsl.get_code_list("上市期货") + # code_list=sorted(list(set(code_list))) + + begin_date='20221031' + end_date='20221101' + + + code_list=['CF2211'] + df = tsl.process_result_data_type(tsl.get_mkt_min_k(begin_date,end_date,code_list)) + print(df) + + ddb = DDBfm('dev') + ddb.create_hft_database() + ddb.create_hft_table(ddb.ddf_hft_mink_tbname,df) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/test/test.dos b/test/test.dos new file mode 100644 index 0000000..9579f1f --- /dev/null +++ b/test/test.dos @@ -0,0 +1,29 @@ +getAllDBs(); +getDFSDatabases(); +getScheduledJobs(); +loadTableBySQL(tableName='pt',dbPath="dfs://tsdb",sql="select * limit 10;"); +loadTable("dfs://tsdb","pt") +loadTable("dfs://hft_stock_ts","KLinePartitioned") +listTables("dfs:/hft_futuremarket_ts"); + +pt=loadTable("dfs://hft_futuremarket_ts","MinKlinePartitioned") +select top 40 * from pt + +select count(*) from loadTable("dfs://hft_futuremarket_ts", "MinKlinePartitioned") +schema(pt) + +n=1000000 +ID=rand(100, n) +dates=2017.08.07..2017.08.11 +date=rand(dates, n) +x=rand(10.0, n) +t=table(ID, date, x) + +dbDate = database(, VALUE, 2017.08.07..2017.08.11) +dbID=database(, RANGE, 0 50 100) +db = database("dfs://compoDB", COMPO, [dbDate, dbID]) +pt = db.createPartitionedTable(t, `pt, `date`ID) +pt.append!(t) + +pt=loadTable(db,`pt) +select count(x) from pt \ No newline at end of file