dolphin-dev/DDBFactor.py

from pprint import pprint
from tqdm import tqdm 
import functools

import dolphindb as ddb

from DDBLoader import DDBLoader



def load_ddb_table(hft_tbl_name):
    """
    这是一个用来简化载入分区表过程的语法糖，但似乎需要预先调用这个函数的场景并不多，简化效果不是很明显。
    """
    def decorator(func):
        @functools.wraps(func)
        def wrapper(self, *args, **kwargs):
            self.ddb_sess.run("""
                // 载入计算使用的原始数据：分钟线数据
                tbl = loadTable("{hft_ddb_path}", "{hft_tbl_name}");
            """.format(
                hft_ddb_path = DDBLoader.ddb_path,
                hft_tbl_name = hft_tbl_name,
            ))
            print('Did load', hft_tbl_name)

            return func(self)
        return wrapper
    return decorator


class DailyFactor(object):

    #ddb_hft_path = "dfs://hft_stock_ts"
    #ddb_hft_dbname = "db_hft_stock"
    ddb_daily_path = "dfs://daily_stock_ts"
    ddb_daily_dbname = "db_daily_stock"

    ddb_config = {
        'host' : '192.168.1.167',
        'username' : 'admin',
        'password' : '123456',
    }

    # 这里的partition数量未必需要和hft表的一致
    # 当读取hft表的时候，需要使用DDBLoader中的`num_code_partition`，而不是此字段
    num_code_partition = 50

    def __init__(self):
        self.ddb_sess = ddb.session(self.ddb_config['host'], 8848)
        self.ddb_sess.login(self.ddb_config['username'], self.ddb_config['password'])


    def create_ddb_database(self):
        """
        因为日频数据量较小，内部可以使用m_nDate作为sort_key，所以分区仅需要对stock_id做[HASH, 50]即可，因此不需要输入calendar数据
        """
        self.ddb_sess.run("""
            daily_stock_ts = database(
                "{ddb_hft_path}", 
                HASH, [SYMBOL, {num_code_parition}], 
                engine = 'TSDB'
            )
        """.format(
            ddb_hft_path = DDBLoader.ddb_path,
            num_code_partition = self.num_code_partition
        ))
        print('Did create database')
   

    def load_ddb_database(self):
        self.ddb_sess.run("""
            {dbName} = database(
                directory = '{dbPath}',
                partitionType = HASH,
                partitionScheme = [SYMBOL, {num_code_partition}],
                engine = 'TSDB'
            )
        """.format(
            dbName = self.ddb_daily_dbname,
            dbPath = self.ddb_daily_path,
            num_code_partition = self.num_code_partition
        ))
        print('Did load database.')


    def append_factor_columns(self, factor_name_list, memory_tbl_name, partition_tbl_name):
        code = """
            addColumn({partition_tbl_name}, {col_name_list}, {col_type_list});
        """.format(
            partition_tbl_name = partition_tbl_name,
            col_name_list = '`' + '`'.join(factor_name_list),
            col_type_list = '[' + ','.join(['DOUBLE']*len(factor_name_list)) + ']'
        )
        print('Will add columns via script:')
        print(code)
        self.ddb_sess.run(code)

        code = """
            {partition_tbl_name}.tableInsert({memory_tbl_name})
        """.format(
            partition_tbl_name = partition_tbl_name,
            memory_tbl_name = memory_tbl_name
        )
        print('Will append date via script:')
        print(code)
        self.ddb_sess.run(code)


    def append_to_partition_table(self, partition_tbl_name, memory_tbl_name):
        self.ddb_sess.run("""
            {partition_tbl_name}.tableInsert({memory_tbl_name})
        """.format(
            partition_tbl_name = partition_tbl_name,
            memory_tbl_name = memory_tbl_name
        ))


    def create_factor_partition_table(self, partition_tbl_name, memory_tbl_name):
        """
        把数据从内存表（`memory_tbl_name`）添加到分区表（`partition_tbl_name`）
        """
        # createPartitionedTable(
        #       dbHandle, table, tableName, 
        #       [partitionColumns], [compressMethods], 
        #       [sortColumns], [keepDuplicates=ALL], [sortKeyMappingFunction])
        code = """
            // 保证创建新的分区表不会和已经存在表冲突
            if (existsTable("{ddb_daily_path}", "{partition_tbl_name}")) {{
                dropTable({ddb_daily_dbname}, "{partition_tbl_name}");
            }}

            {partition_tbl_name} = createPartitionedTable(
                dbHandle = {ddb_daily_dbname},
                table = {memory_tbl_name}, 
                tableName = "{partition_tbl_name}",
                partitionColumns = 'code',
                compressMethods = {{'m_nDate' : 'delta'}},
                sortColumns = `code`m_nDate
            );
        """.format(
            ddb_daily_path = self.ddb_daily_path,
            ddb_daily_dbname = self.ddb_daily_dbname,
            partition_tbl_name = partition_tbl_name,
            memory_tbl_name = memory_tbl_name,
        )
        print('Will create partitioned factor table via script:')
        print(code)
        self.ddb_sess.run(code)


    @load_ddb_table("KLinePartitioned")
    def make_kurto_memory_table(self):

        memory_table_name = "kurto"
        code_tpl = """
            // 需要首先创建分钟线收益表
            // 使用`context by`使得计算结果仍然为一个序列
            // 使用`where partition()`来逐个加载分区
            ret_sql = select 
                code, m_nDate, eachPre(\, m_nClose)-1.0 as ret 
                from tbl 
                where partition(code, {partition_id}) 
                context by m_nDate;

            // 计算kurto指标，`ret`表中每日第一条记录为空，似乎并不造成影响
            kurto_sql = select 
                code, m_nDate, sqrt(239) * sum(pow(ret, 3)) / pow(sum(pow(ret, 2)), 1.5) as kurto 
                from ret_sql 
                group by code, m_nDate;
        """
        with tqdm(range(DDBLoader.num_code_partition)) as pbar:
        #with tqdm(range(1)) as pbar:
            for partition_id in pbar:
                self.ddb_sess.run(code_tpl.format(
                    partition_id = partition_id,
                ))

                # 因为原表有50个分区，需要逐个计算，因此先创建一个内存临时表
                # 否则一旦第一个分区插入分区表后，就无法再插入后续只包含部分字段的数据了
                if partition_id == 0:
                    self.ddb_sess.run("""
                        {memory_table_name} = table(kurto_sql)
                    """.format(
                        memory_table_name = memory_table_name
                    ))

                # 上面`table`语句仅仅是创建表结构，
                # 然后使用`tableInsert`把真实数据插入进去
                self.ddb_sess.run("""
                    {memory_table_name}.tableInsert(kurto_sql)
                """.format(
                    memory_table_name = memory_table_name
                ))
            print('Did finish all parititons for kurto.')
            pprint(self.ddb_sess.run(f"{memory_table_name}"))

        return memory_table_name


def main():
    factor = DailyFactor()
    factor.load_ddb_database()
    
    memory_table_name = factor.make_kurto_memory_table()
    factor.create_factor_partition_table(
        'hft_daily_factor', 
        memory_table_name
    )
    factor.append_to_partition_table(
        'hft_daily_factor', 
        memory_table_name
    )


if __name__ == '__main__':
    main()