QA script

Author: barretthugh

README.md

@@ -1 +1,22 @@
 # QA script
+
+Thanks @yutiansut's hard work  
+
+
+Custom scripts for QUANTAXIS  
+
+
+custom_update: read update type and freq from update_list.txt  
+
+QA_SU_save_list: for list update  
+QA_SU_save_short_freq: for minute freq update  
+QA_SU_save_long_freq: for day/week/month update  
+
+QA_data_fetch_adv for local data fetch, include stock/future/index/etf, day and min  
+support dynamic minute freq resample from 1min data  
+
+
+TODO:  
+option update  
+local option fetch  
+make sure everything is updated in update script

adv.py

@@ -0,0 +1,86 @@
+from QUANTAXIS.QAData import (
+    QA_DataStruct_Index_day,
+    QA_DataStruct_Index_min,
+    QA_DataStruct_Future_day,
+    QA_DataStruct_Future_min,
+    QA_DataStruct_Stock_block,
+    QA_DataStruct_Financial,
+    QA_DataStruct_Stock_day,
+    QA_DataStruct_Stock_min,
+    QA_DataStruct_Stock_transaction
+)
+
+DataStruct_d = {
+    'stock_day': QA_DataStruct_Stock_day,
+    'stock_min': QA_DataStruct_Stock_min,
+    'index_day': QA_DataStruct_Index_day,
+    'index_min': QA_DataStruct_Index_min,
+    'future_day': QA_DataStruct_Future_day,
+    'future_min': QA_DataStruct_Future_min
+}
+
+
+def QA_data_fetch_adv(
+        code='',
+        start='all',
+        end=None,
+        data_type='stock_day',
+        frequence='',
+        if_drop_index=True
+):
+    """'fetch data from database'
+    :param code: code list of stock, index, future, etf
+    :param start: date of start, can be str/int of 2019, 201901, '2019-01', '2019-01-01'
+    :param end: date of end, can be str/int of 2019, 201901, '2019-01', '2019-01-01'
+    :param data_type: stock_day, stock_min etc
+    :param frequence: minute freq of min data, can be standard min value of 15, 30min, or uncommon like 9, 17min
+    Returns:
+        [type] -- [description]
+
+        感谢@几何大佬的提示
+        https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/#return-the-specified-fields-and-the-id-field-only
+    """
+
+    data_type = data_type.lower()
+
+    if data_type in ['stock_list', 'index_list', 'future_list', 'etf_list']:
+        return QA_list_fetch(data_type)
+
+    # resample to given frequence
+    resample = False
+    if frequence:
+        if str(frequence).split('m')[0] in ['1', '5', '15', '30', '60']:
+            frequence = str(frequence).split('m')[0] + 'min'
+        else:
+            resample = str(frequence).split('m')[0] + 'min'
+            frequence = '1min'
+
+    res = QA_data_fetch(
+        code=code,
+        start=start,
+        end=end,
+        data_type=data_type,
+        frequence=frequence,
+        format='pd'
+    )
+    if res is None:
+        # 🛠 todo 报告是代码不合法，还是日期不合法
+        print(
+            "QA Error QA_data_fetch_adv parameter code=%s , start=%s, end=%s call QA_data_fetch return None"
+            % (code,
+               start,
+               end)
+        )
+        return None
+    else:
+        res_reset_index = res.set_index(
+            ['datetime',
+             'code'],
+            drop=if_drop_index
+        )
+        if resample:
+            ds = DataStruct_d[data_type](res_reset_index)
+            ds = ds.resample(resample)
+            return DataStruct_d[data_type](ds.assign(type=resample))
+
+        return DataStruct_d[data_type](res_reset_index)

docker-compose_dev.yaml

@@ -0,0 +1,46 @@
+@@ -1,45 +0,0 @@
+version: '2'
+services:
+    qa:
+        image: barretthugh/qa-jupyter
+        ports:
+            - "8888:8888"
+        environment:
+          - MONGODB=mgdb
+        volumes:
+            - qacode:/home
+            - .:/QUANTAXIS
+        restart: always
+
+    qacron:
+        image: barretthugh/qa-cron
+        environment:
+            - MONGODB=mgdb
+        restart: always
+
+    qaweb:
+        image: barretthugh/qa-web
+        ports:
+            - "8010:8010"
+        environment:
+            - MONGODB=mgdb
+        restart: always
+
+    mgdb:
+        image: mongo:4.1.3
+        ports:
+            - "27017:27017"
+        environment:
+            - TZ=Asia/Shanghai
+            - MONGO_INITDB_DATABASE=quantaxis
+        volumes:
+            - qamg:/data/db
+        restart: always
+
+volumes:
+    qamg:
+        external:
+            name: qamg
+    qacode:
+        external:
+            name: qacode

main.py

@@ -0,0 +1,48 @@
+from QUANTAXIS.QAFetch.QAQuery import QA_fetch_stock_list
+#from QUANTAXIS.QASU import crawl_eastmoney as crawl_eastmoney_file
+from QUANTAXIS.QASU import save_tdx as stdx
+from QUANTAXIS.QASU import save_tdx_file as tdx_file
+from QUANTAXIS.QASU import save_tushare as sts
+from QUANTAXIS.QASU import save_financialfiles
+from QUANTAXIS.QAUtil import DATABASE
+
+def QA_SU_save_list(engine, type_='', client=DATABASE):
+    """save data list like stock, index, etf, future
+
+    Arguments:
+        engine {[type]} -- [description]
+
+    Keyword Arguments:
+        client {[type]} -- [description] (default: {DATABASE})
+    """
+
+    engine = select_save_engine(engine)
+    engine.QA_SU_save_list(type_=type_, client=client)
+
+
+def QA_SU_save_short_freq(engine, type_='', min_list=[], client=DATABASE):
+    """save short freq of data: given list of minute freq
+
+    Arguments:
+        engine {[type]} -- [description]
+
+    Keyword Arguments:
+        client {[type]} -- [description] (default: {DATABASE})
+    """
+
+    engine = select_save_engine(engine)
+    engine.QA_SU_save_short_freq(type_=type_, min_list=min_list, client=client)
+
+
+def QA_SU_save_long_freq(engine, type_='', client=DATABASE):
+    """save long freq of data: like day, week etc
+
+    Arguments:
+        engine {[type]} -- [description]
+
+    Keyword Arguments:
+        client {[type]} -- [description] (default: {DATABASE})
+    """
+
+    engine = select_save_engine(engine)
+    engine.QA_SU_save_long_freq(type_=type_, client=client)