bestfitting团队完整代码.ipynb

{"cells":[{"metadata":{"id":"55EF1E088609443FA8CDDF08329684C3","mdEditEnable":false},"cell_type":"markdown","source":"# 1. 词向量训练语料的构建\n训练集后一亿 + 测试集A + 测试集B"},{"metadata":{"id":"183108B9ABEF47E4891AA1B90E278421","mdEditEnable":false},"cell_type":"markdown","source":"## 1.1 提取训练集后一亿语料\n将query和title混合并利用set去重"},{"metadata":{"id":"22F07C6F0A0549818E79437C8B46B139","collapsed":false,"scrolled":false},"cell_type":"code","outputs":[],"source":"import pandas as pd\nimport pickle\n# 读取后一亿数据用于训练\ntrain = pd.read_csv'/home/kesci/input/bytedance/test_final_part1.csv', header=None,\n     names=['query_id', 'query', 'query_title_id', 'title'], nrows=100000000, skiprows=900000000)\n     \ntrain.to_csv('/home/kesci/work/train_data/train_final.csv', header=False, index=False)\nb = []\nf = open(\"/home/kesci/work/train_data/train_final.csv\")\nlines = f.readlines(256*1024*1024)\ncount = 0\nwhile lines:\n    for line in lines:\n        count += 1\n        line = line.strip().split(\",\")\n        query = line[1]\n        title = line[3]\n        b.append(query)\n        b.append(title)\n        if count % 10000000 == 0:\n            print(\"count \", count)\n    lines = f.readlines(256*1024*1024)\nf.close()\nb = set(b)","execution_count":2},{"metadata":{"id":"99805B23B4174C318B6A37C08F04326C","mdEditEnable":false},"cell_type":"markdown","source":"## 1.2 提取测试集语料\n将query和title混合并利用set去重"},{"metadata":{"id":"27E62F8A7E334E89B89285F4059F5CE2"},"cell_type":"code","outputs":[],"source":"a = []\nf = open(\"/home/kesci/input/bytedance/bytedance_contest.final_2.csv\")\nlines = f.readlines(256*1024*1024)\ncount = 0\nwhile lines:\n    for line in lines:\n        count += 1\n        line = line.strip().split(\",\")\n        query = line[1]\n        title = line[3]\n        a.append(query)\n        a.append(title)\n        if count % 10000000 == 0:\n            print(\"count \", count)\n    lines = f.readlines(256*1024*1024)\nf.close()\na = set(a)\nc = []\nf = open(\"/home/kesci/input/bytedance/test_final_part1.csv\")\nlines = f.readlines(256*1024*1024)\ncount = 0\nwhile lines:\n    for line in lines:\n        count += 1\n        line = line.strip().split(\",\")\n        query = line[1]\n        title = line[3]\n        c.append(query)\n        c.append(title)\n        if count % 10000000 == 0:\n            print(\"count \", count)\n    lines = f.readlines(256*1024*1024)\nf.close()\nc = set(c)","execution_count":null},{"metadata":{"id":"B47B1E4F44E54027B82A031D9F8F6A7E","mdEditEnable":false},"cell_type":"markdown","source":"## 1.3 将训练集和测试集语料混合在一起并去重"},{"metadata":{"id":"970FBB3A8F12460599A1B13B8BBCFCB5"},"cell_type":"code","outputs":[],"source":"d = (set(c) | set(b) | set(a))\n# 将语料保存成gensim中LineSentence的格式\nwith open(\"silver_bullet/corpus\", mode=\"w\") as f:\n    for i in d:\n        f.write(i + '\\n')\nwith open(\"silver_bullet/final_set.pickle\", mode=\"wb\") as f:\n    pickle.dump(d, f)","execution_count":null},{"metadata":{"id":"69E1E5B1602B4C40A060D0275A5CA40A","mdEditEnable":false},"cell_type":"markdown","source":"# 2. 词向量的训练\n根据步骤1提取的语料分别训练word2vec和fasttext"},{"metadata":{"id":"A2C74F4C26D041B8A2D8B9114896870E"},"cell_type":"code","outputs":[],"source":"import gensim\nimport time\nimport pandas as pd\nimport numpy as np\nfrom gensim.models import word2vec,fasttext\nfrom gensim.models.fasttext import FastText\nfrom gensim.test.utils import get_tmpfile, datapath\nWV_MODEL_PATH = \"/home/kesci/work/helper_model/w2v_final2.model\"\nFAST_MODEL_PATH = \"/home/kesci/work/helper_model/fasttext_final.model\"","execution_count":null},{"metadata":{"id":"FF470B6E7E624B04944E352F10252B7E","mdEditEnable":false},"cell_type":"markdown","source":"## 2.1 训练word2vec\nword2vec每个epoch监控一下loss，在第8个epoch的loss异常的变成了0，多次训练都是类似的结果(可能跟数据量大小有关)最终取了第7个epoch个word2vec\n"},{"metadata":{"id":"12F37420C58E46BEB85460ABCB5D70B1"},"cell_type":"code","outputs":[],"source":"# 后面加载训练好的w2v模型时也需要有这个类的定义, 否则load会报找不到这个类的错误\nclass EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n    '''词向量训练的回调函数，用于保存模型，打印损失函数等等'''\n    def __init__(self, save_name):\n        self.save_path = save_name\n        self.epoch = 0\n        self.pre_loss = 0\n        self.best_loss = 999999999.9\n        self.since = time.time()\n\n    def on_epoch_end(self, model):\n        self.epoch += 1\n        cum_loss = model.get_latest_training_loss() # 返回的是从第一个epoch累计的\n        epoch_loss = cum_loss - self.pre_loss\n        time_taken = time.time() - self.since\n        print(\"Epoch %d, loss: %.2f, time: %dmin %ds\" %\n                    (self.epoch, epoch_loss, time_taken//60, time_taken%60))\n        # 避免loss等于0的情况\n        if self.best_loss > epoch_loss and epoch_loss > 0:\n            self.best_loss = epoch_loss\n            print(\"Better model. Best loss: %.2f\" % self.best_loss)\n            model.save(self.save_path)\n            print(\"Model %s save done!\" % self.save_path)\n\n        self.pre_loss = cum_loss\n        self.since = time.time()\n# 训练300维的word2vec\nmodel_word2vec = gensim.models.Word2Vec(min_count=2,window=5,size=300,workers=15,batch_words=500000)\nlinesen = datapath('/home/kesci/work/silver_bullet/corpus') # 语料文件\nmodel_word2vec.build_vocab(corpus_file=linesen)\nmodel_word2vec.train(corpus_file=linesen, total_words=model_word2vec.corpus_total_words, \n        total_examples=model_word2vec.corpus_count,epochs=30, compute_loss=True, report_delay=60,\ncallbacks=[EpochSaver(\"/home/kesci/work/helper_model/w2v_final2.model\")])","execution_count":null},{"metadata":{"id":"B1571C9F45E74D5984A92EEEA654B05D","mdEditEnable":false},"cell_type":"markdown","source":"## 2.2 训练fasttext\n由于时间关系，fasttext只训练了20个epoch"},{"metadata":{"id":"EB8C9773473140E59D286E1C6A747E28"},"cell_type":"code","outputs":[],"source":"class EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n     \"Callback to save model after every epoch\"\n     def __init__(self, path_prefix, epochs=0):\n         self.path_prefix = path_prefix\n         self.epoch = epochs\n\n     def on_epoch_end(self, model):\n         output_path = self.path_prefix\n         print(\"Save model to {}\".format(output_path))\n         model.save(output_path)\n         self.epoch += 1\n# 训练300维的fasttext词向量\nft = FastText(size=300,window=5,min_count=2,workers=15,batch_words=200000)\nlinesen = datapath('/home/kesci/work/silver_bullet/corpus')\nft.build_vocab(corpus_file=linesen)\nft.train(corpus_file=linesen, total_words=ft.corpus_total_words,\ntotal_examples=ft.corpus_count,epochs=30, compute_loss=True,\ncallbacks=[EpochSaver(\"/home/kesci/work/helper_model/fasttext_final.model\")])","execution_count":null},{"metadata":{"id":"38A90777937E42B293F7783F4881ECBD","collapsed":false,"scrolled":false,"mdEditEnable":false},"cell_type":"markdown","source":"# 3. tfidfvectorizer和countVectorizer模型\n后续将使用tfidf, cv计算相似度等特征"},{"metadata":{"id":"140B734B61BE4128867D5D3B485D6D7F"},"cell_type":"code","outputs":[],"source":"from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nimport pickle\nwith open(\"silver_bullet/final_set.pickle\", mode=\"rb\") as f:\n    d = pickle.load(f)\n# d:set 包含去重后的所有query, title\nTF_IDF_MODEL_PATH = \"helper_model/tf_idf.model\"\nCV_MODEL_PATH = \"helper_model/countVectorizer.model\"\n\n# tf_idf\ntf_idf = TfidfVectorizer(min_df=2,use_idf=True).fit(d)\nwith open(TF_IDF_MODEL_PATH, 'wb') as f:\n    pickle.dump(tf_idf, f)\n\n# cv\ncv = CountVectorizer(max_df=0.01, min_df=2).fit(d)\nwith open(CV_MODEL_PATH, 'wb') as f:\n    pickle.dump(cv, f)","execution_count":null},{"metadata":{"id":"5EE9C6CA645B4C7F9229C3B7FD948B71","mdEditEnable":false},"cell_type":"markdown","source":"# 4. 特征工程"},{"metadata":{"id":"2BD361658441469F82D9A25DBBCB6699"},"cell_type":"code","outputs":[],"source":"import os\nimport gc\nimport csv\nimport time\nimport math\nimport random\nimport pickle\nimport inspect\nfrom gensim.models import word2vec,fasttext\nimport gensim\nimport gzip\nimport multiprocessing\nfrom collections.abc import Iterable\nfrom scipy import spatial\nfrom sklearn.metrics.pairwise import *\nimport sklearn.preprocessing as pp\nimport copy\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport pickle\ntqdm.pandas()","execution_count":null},{"metadata":{"id":"4B09D82B19DA41DD89D95E2ACCA81595","collapsed":false,"scrolled":false,"mdEditEnable":false},"cell_type":"markdown","source":"## 4.1 nuniuqe全局特征"},{"metadata":{"id":"DB88C002B24243A6960335326BAD868F"},"cell_type":"code","outputs":[],"source":"# 分别读取训练集的最后一亿，测试集A，测试集B\ntrain = pd.read_csv('/home/kesci/work/train_data/train_final.csv', header=None, \n        nrows=int(1e8), skiprows=int(9e8))\ntest1 = pd.read_csv('/home/kesci/input/bytedance/test_final_part1.csv', header=None)\ntest2 = pd.read_csv('/home/kesci/input/bytedance/bytedance_contest.final_2.csv', header=None)\ndt = pd.concat([train, test1, test2])\ndt.columns = [0, 'query', 2, 'title', 4, 5, 6]\ndt = dt[['query', 'title']]\n\n# 计算query, title的unique的个数\ndt['query_nunique_title'] = dt.groupby('query').title.transform('nunique')\ndt['title_nunique_query'] = dt.groupby('title').query.transform('nunique')\n\ntrain_number = 100000000\ntest1_number = 20000000\ntest2_number = 100000000\n\ntrain_nuinque = dt[['query_nunique_title', 'title_nunique_query']][:train_number]\nwith open(\"/home/kesci/work/train_data/train_final_nunique.pickle\", mode=\"wb\") as f:\n    pickle.dump(train_nuinque, f)\ntest1_nuinque = dt[['query_nunique_title', 'title_nunique_query']][train_number:train_number+test1_number]\nwith open(\"/home/kesci/work/testdata1/test1_nunique.pickle\", mode=\"wb\") as f:\n    pickle.dump(test1_nuinque, f)    \ntest2_nuinque = dt[['query_nunique_title', 'title_nunique_query']][train_number+test1_number:]\nwith open(\"/home/kesci/work/testdata2/test2_nunique.pickle\", mode=\"wb\") as f:\n    pickle.dump(test2_nuinque, f)    ","execution_count":null},{"metadata":{"id":"2ADD7CA529D44EF09A66C37CC531FCC7","collapsed":false,"scrolled":false,"mdEditEnable":false},"cell_type":"markdown","source":"## 4.2 tfidf和cv相似度特征"},{"metadata":{"id":"7E0C21EBE0D7472F80BE40D56C6F592F"},"cell_type":"code","outputs":[],"source":"import pickle\nTF_IDF_MODEL_PATH = \"/home/kesci/work/helper_model/tf_idf.model\"\nCV_MODEL_PATH = \"/home/kesci/work/helper_model/countVectorizer.model\"\nwith open(TF_IDF_MODEL_PATH, mode=\"rb\") as f:\n    tfidf = pickle.load(f)\nwith open(CV_MODEL_PATH, mode=\"rb\") as f:\n    cv = pickle.load(f)","execution_count":null},{"metadata":{"id":"E4D55B4F943642C18B392E67A0D52F84"},"cell_type":"code","outputs":[],"source":"# 对query和title进行count vectorize向量化\ncv_query = cv.transform(dt['query'])\ncv_title = cv.transform(dt['title'])\nnumber = 10000\nepochs = int(cv_query.shape[0] / number)\nfor i in range(epochs):\n    cv_query1 = cv_query[i*number:(i+1)*number]\n    cv_title1 = cv_title[i*number:(i+1)*number]\n    # 计算query，title向量化后的余弦，欧几里得，曼哈顿距离\n    c = paired_cosine_distances(cv_query1, cv_title1)\n    e = paired_euclidean_distances(cv_query1, cv_title1)\n    m = paired_manhattan_distances(cv_query1, cv_title1)\n    if i == 0:\n        a1 = c\n        a2 = e\n        a3 = m\n    else:\n        a1 = np.concatenate((a1,c))\n        a2 = np.concatenate((a2,e))\n        a3 = np.concatenate((a3,m))\ncv_feature = np.array([a1,a2,a3]).T\n\ntrain_cv = cv_feature[:train_number]\nwith open('/home/kesci/work/train_data/train_final_cv', mode=\"wb\") as f:\n    pickle.dump(train_cv, f)\ntest1_cv = cv_feature[train_number:train_number+test1_number]\nwith open(\"/home/kesci/work/testdata1/test1_final_cv\", mode=\"wb\") as f:\n    pickle.dump(test1_cv, f)    \ntest2_cv = cv_feature[train_number+test1_number:]\nwith open(\"testdata2/test2_final_cv\", mode=\"wb\") as f:\n    pickle.dump(test2_cv, f)","execution_count":null},{"metadata":{"id":"F3A3241C9B4141BD863B05EF4B23B069"},"cell_type":"code","outputs":[],"source":"# 对query和title进行tfidf vectorize向量化\ntf_idf_query = tfidf.transform(dt['query'])\ntf_idf_title = tfidf.transform(dt['title'])\nnumber = 10000\nepochs = int(tf_idf_query.shape[0] / number)\nfor i in range(epochs):\n    tf_idf_query1 = cv_query[i*number:(i+1)*number]\n    tf_idf_title1 = cv_title[i*number:(i+1)*number]\n    # 计算query，title向量化后的余弦，欧几里得，曼哈顿距离\n    c = paired_cosine_distances(tf_idf_query1, tf_idf_title1\n    e = paired_euclidean_distances(tf_idf_query1, tf_idf_title1)\n    m = paired_manhattan_distances(tf_idf_query1, tf_idf_title1)\n    if i == 0:\n        a1 = c\n        a2 = e\n        a3 = m\n    else:\n        a1 = np.concatenate((a1,c))\n        a2 = np.concatenate((a2,e))\n        a3 = np.concatenate((a3,m))\ntf_idf_feature = np.array([a1,a2,a3]).T\n\ntrain_tfidf = tf_idf_feature[:train_number]\nwith open('/home/kesci/work/train_data/train_final_tfidf', mode=\"wb\") as f:\n    pickle.dump(train_tfidf, f)\ntest1_tfidf = tf_idf_feature[train_number:train_number+test1_number]\nwith open('/home/kesci/work/testdata1/test1_final_tfidf', mode=\"wb\") as f:\n    pickle.dump(test1_tfidf, f)    \ntest2_tfidf = tf_idf_featuree[train_number+test1_number:]\nwith open('/home/kesci/work/testdata2/test2_final_tfidf', mode=\"wb\") as f:\n    pickle.dump(test2_tfidf, f)","execution_count":null},{"metadata":{"id":"161AB6B3C69148418ECB4BE4B8BD18DB","mdEditEnable":false},"cell_type":"markdown","source":"## 4.3 其余特征\n诸如长度特征、是否存在、编辑距离、jarrard距离、词向量相似度等特征都写在Feature类下面（具体解释在文档中），方便多线程加速。"},{"metadata":{"id":"012998AA15624391A6AA597CBE2E69D2"},"cell_type":"code","outputs":[],"source":"tfidf_dic = dict(zip(tfidf.get_feature_names(), tfidf.idf_))\ntfidf_set = tfidf.get_feature_names()\n\nclass EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n    '''用于保存模型, 打印损失函数等等'''\n    def __init__(self, save_name):\n        self.save_path = save_name\n        self.epoch = 0\n        self.pre_loss = 0\n        self.best_loss = 999999999.9\n        self.since = time.time()\n\n    def on_epoch_end(self, model):\n        self.epoch += 1\n        cum_loss = model.get_latest_training_loss() # 返回的是从第一个epoch累计的\n        epoch_loss = cum_loss - self.pre_loss\n        time_taken = time.time() - self.since\n        print(\"Epoch %d, loss: %.2f, time: %dmin %ds\" %\n                    (self.epoch, epoch_loss, time_taken//60, time_taken%60))\n        if self.best_loss > epoch_loss:\n            self.best_loss = epoch_loss\n            print(\"Better model. Best loss: %.2f\" % self.best_loss)\n            model.save(self.save_path)\n            print(\"Model %s save done!\" % self.save_path)\n\n        self.pre_loss = cum_loss\n        self.since = time.time()\n\n# 载入词向量模型\nw2v = word2vec.Word2Vec.load(WV_MODEL_PATH)\nft = fasttext.FastText.load(FAST_MODEL_PATH)","execution_count":null},{"metadata":{"id":"2383F9C3E388488E9236F8320EDDEEEF"},"cell_type":"code","outputs":[],"source":"class Feature:\n    '''\n    逐行进行特征工程，静态方法用于输出特征。\n    '''\n\n    def __init__(self, skip_rows=0):\n        # 以后做全局特征的时候可能要用\n        self.idx = skip_rows\n        # 包括了所有的静态方法的名字\n        self._static = [i for i in dir(self)\n                        if isinstance(inspect.getattr_static(self, i), staticmethod)]\n\n    def output(self, s1, s2):\n        '''输出所有静态方法下的特征'''\n        self.idx += 1\n        # 逐一调用每个静态方法\n        # output = [getattr(self, method)(s1.split(\" \"), s2.split(\" \")) for method in self._static]\n        output = []\n        s1, s2 = s1.split(), s2.split()\n        # 逐一导入各个特征\n        output.append(self.length_feature(s1, s2))\n        output.append(self.isin(s1, s2))\n        output.append(self.lev(s1, s2))\n        output.append(self.Jaccrad(s1, s2))\n        output.append(self.longestCommonSubsequence(s1, s2))\n        output.append(self.lcsubstr_lens(s1, s2))\n        output.append(self.prefix_suffix_features(s1, s2))\n        return list(Feature.flatten(output))\n\n    @classmethod\n    def flatten(cls, l: Iterable):\n        '''列表铺平'''\n        for i in l:\n            if isinstance(i, Iterable):\n                yield from i\n            else:\n                yield i\n\n    # ----- 长度特征 ------\n    @staticmethod\n    def length_feature(q1_tokens, q2_tokens):\n        q1_unique_tokens = set(q1_tokens)\n        q2_unique_tokens = set(q2_tokens)\n        shared_tokens = q1_unique_tokens.intersection(q2_unique_tokens)\n        q1_token_count = len(q1_tokens)\n        q2_token_count = len(q2_tokens)\n        q1_unique_token_count = len(q1_unique_tokens)\n        q2_unique_token_count = len(q2_unique_tokens)\n        q1_unique_ratio = q1_unique_token_count / q1_token_count\n        q2_unique_ratio = q2_unique_token_count / q2_token_count\n        min_token_count = min(q1_token_count, q2_token_count)\n        max_token_count = max(q1_token_count, q2_token_count)\n        min_unique_token_count = min(q1_unique_token_count, q2_unique_token_count)\n        max_unique_token_count = max(q1_unique_token_count, q2_unique_token_count)\n        min_unique_ratio = min(q1_unique_ratio, q2_unique_ratio)\n        max_unique_ratio = max(q1_unique_ratio, q2_unique_ratio)\n        count_add = q1_token_count + q2_token_count\n        count_sub = abs(q1_token_count - q2_token_count)\n        count_mul = q1_token_count * q2_token_count\n        unique_count_add = q1_unique_token_count + q2_unique_token_count\n        unique_count_sub = abs(q1_unique_token_count - q2_unique_token_count)\n        unique_count_mul = q1_unique_token_count * q2_unique_token_count\n        unique_ratio_add = q1_unique_ratio + q2_unique_ratio\n        unique_ratio_sub = abs(q1_unique_ratio - q2_unique_ratio)\n        unique_ratio_mul = q1_unique_ratio * q2_unique_ratio\n        shared_ratio_min = len(shared_tokens) / max(q1_unique_token_count, q2_unique_token_count)\n        shared_ratio_max = len(shared_tokens) / min(q1_unique_token_count, q2_unique_token_count)\n        return [min_token_count, max_token_count,\n          min_unique_token_count, max_unique_token_count,\n          min_unique_ratio, max_unique_ratio,\n         count_add, count_sub,\n          unique_count_add, unique_count_sub, unique_count_mul,\n          unique_ratio_add, unique_ratio_sub, unique_ratio_mul,\n          shared_ratio_min, shared_ratio_max]\n\n\n    # ------是否存在------\n    @staticmethod\n    def isin(s1, s2):\n        return [int(' '.join(s1) in ' '.join(s2)), np.mean([int(w in s2) for w in s1])]\n\n\n    # ------ 编辑距离 ------\n    @staticmethod\n    def lev(s1, s2):\n        n1, n2 = len(s1), len(s2)\n        lev_matrix = [[0 for i1 in range(n1 + 1)] for i2 in range(n2 + 1)]\n        for i1 in range(1, n1 + 1):\n            lev_matrix[0][i1] = i1\n        for i2 in range(1, n2 + 1):\n            lev_matrix[i2][0] = i2\n        for i2 in range(1, n2 + 1):\n            for i1 in range(1, n1 + 1):\n                cost = 0 if s1[i1 - 1] == s2[i2 - 1] else 1\n                elem = min(lev_matrix[i2 - 1][i1] + 1,\n                           lev_matrix[i2][i1 - 1] + 1,\n                           lev_matrix[i2 - 1][i1 - 1] + cost)\n                lev_matrix[i2][i1] = elem\n        total_length = s1.__len__() + s2.__len__()\n        return [lev_matrix[-1][-1], (total_length - lev_matrix[-1][-1]) / total_length]\n    \n    # ------ jaccrad相似度 ------  \n    @staticmethod\n    def Jaccrad(s1, s2):\n        grams_reference = set(s1)\n        grams_model = set(s2)\n        temp = 0\n        for i in grams_reference:\n            if i in grams_model:\n                temp = temp + 1\n        fenmu = len(grams_model) + len(grams_reference) - temp  # 并集\n        jaccard_coefficient = float(temp / fenmu)  # 交集\n        del temp\n        del fenmu\n        del grams_model\n        del grams_reference\n        return jaccard_coefficient\n\n\n    # ------ 最长公共子序列 ------ \n    @staticmethod\n    def longestCommonSubsequence(s1, s2):\n        # 生成字符串长度加1的0矩阵，m用来保存对应位置匹配的结果\n        m = [[0 for x in range(len(s2) + 1)] for y in range(len(s1) + 1)]\n        # d用来记录转移方向\n        d = [[None for x in range(len(s2) + 1)] for y in range(len(s1) + 1)]\n        for p1 in range(len(s1)):\n            for p2 in range(len(s2)):\n                if s1[p1] == s2[p2]:  # 字符匹配成功，则该位置的值为左上方的值加1\n                    m[p1 + 1][p2 + 1] = m[p1][p2] + 1\n                    d[p1 + 1][p2 + 1] = 'ok'\n                elif m[p1 + 1][p2] > m[p1][p2 + 1]:  # 左值大于上值，则该位置的值为左值，并标记回溯时的方向\n                    m[p1 + 1][p2 + 1] = m[p1 + 1][p2]\n                    d[p1 + 1][p2 + 1] = 'left'\n                else:  # 上值大于左值，则该位置的值为上值，并标记方向up\n                    m[p1 + 1][p2 + 1] = m[p1][p2 + 1]\n                    d[p1 + 1][p2 + 1] = 'up'\n        (p1, p2) = (len(s1), len(s2))\n        s = []\n        while m[p1][p2]:  # 不为None时\n            c = d[p1][p2]\n            if c == 'ok':  # 匹配成功，插入该字符，并向左上角找下一个\n                s.append(s1[p1 - 1])\n                p1 -= 1\n                p2 -= 1\n            if c == 'left':  # 根据标记，向左找下一个\n                p2 -= 1\n            if c == 'up':  # 根据标记，向上找下一个\n                p1 -= 1\n        del m\n        del d\n        return [len(s), len(s)/max(len(s1), len(s2)), len(s)/min(len(s1), len(s2))]\n    \n    # ------ 最长匹配的长度 ------\n    @staticmethod\n    def lcsubstr_lens(s1, s2):\n        m=[[0 for i in range(len(s2)+1)]  for j in range(len(s1)+1)]  #生成0矩阵，为方便后续计算，比字符串长度多了一列\n        mmax=0   #最长匹配的长度\n        p=0  #最长匹配对应在s1中的最后一位\n        for i in range(len(s1)):\n            for j in range(len(s2)):\n                if s1[i]==s2[j]:\n                    m[i+1][j+1]=m[i][j]+1\n                    if m[i+1][j+1]>mmax:\n                        mmax=m[i+1][j+1]\n                        p=i+1\n        del m\n        return [mmax, mmax/max(len(s1), len(s2)), mmax/min(len(s1), len(s2))]\n    \n    # ------ 前缀后缀特征 ------\n    @staticmethod\n    def prefix_suffix_features(q1_tokens, q2_tokens):\n        q1_tokens_reversed, q2_tokens_reversed = q1_tokens[::-1], q2_tokens[::-1]\n        max_score = max(len(q1_tokens), len(q2_tokens))\n        max_recip_score = sum([1 / pos for pos in range(1, max_score + 1)])\n\n        prefix_score, recip_prefix_score, idf_prefix_score = 0.0, 0.0, 0.0\n        for i, (tok1, tok2) in enumerate(zip(q1_tokens, q2_tokens)):\n            if tok1 == tok2:\n                prefix_score += 1.0\n                recip_prefix_score += 1 / (i + 1)\n                # idf_prefix_score += token2idf[tok1]\n            else:\n                prefix_score /= max_score\n                recip_prefix_score /= max_recip_score\n                # idf_prefix_score /= max_idf_score\n\n        suffix_score, recip_suffix_score, idf_suffix_score = 0.0, 0.0, 0.0\n        for i, (tok1, tok2) in enumerate(zip(q1_tokens_reversed, q2_tokens_reversed)):\n            if tok1 == tok2:\n                suffix_score += 1.0\n                recip_suffix_score += 1 / (i + 1)\n            else:\n                suffix_score /= max_score\n                recip_suffix_score /= max_recip_score\n                \n        return [prefix_score, recip_prefix_score,\n                suffix_score, recip_suffix_score]\n                \n    @classmethod\n    def fasttext_transform(cls, sentence):\n        '''将句子的每个词用fasttext embedding转化成向量，计算这些向量的平均值'''\n        size = ft.trainables.layer1_size\n        length = len(sentence)\n        vec = np.zeros(shape=(1, size), dtype=np.float32)\n        for word in sentence:\n            try:\n                vec += ft.wv[word] * tfidf_dic[word]\n            # 如果这个词不在vocabulary中\n            except:\n                length -= 1\n        return vec / length if length > 0 else vec\n    \n    @classmethod\n    def w2v_transform(cls, sentence):\n        '''将句子的每个词用word2vec embedding转化成向量，计算这些向量的平均值'''\n        size = w2v.trainables.layer1_size\n        length = len(sentence)\n        vec = np.zeros(shape=(1, size), dtype=np.float32)\n        for word in sentence:\n            try: \n                vec += w2v.wv[word] * tfidf_dic[word]\n            # 如果这个词不在vocabulary中\n            except:\n                length -= 1\n        return vec / length if length > 0 else vec\n    \n    # ------ 词向量相似度特征 ------\n    @staticmethod\n    def vecotor_compute_cosine(s1, s2):\n        fv1 = Feature.fasttext_transform(s1)\n        fv2 = Feature.fasttext_transform(s2)\n        fv3 = Feature.w2v_transform(s1)\n        fv4 = Feature.w2v_transform(s2)\n        a = [\n                spatial.distance.braycurtis(fv1, fv2),spatial.distance.braycurtis(fv3, fv4),\n                spatial.distance.canberra(fv1, fv2),spatial.distance.canberra(fv3, fv4),\n                spatial.distance.chebyshev(fv1, fv2),spatial.distance.chebyshev(fv3, fv4),\n                spatial.distance.correlation(fv1, fv2),spatial.distance.correlation(fv3, fv4),\n                spatial.distance.cosine(fv1, fv2),spatial.distance.cosine(fv3, fv4),\n                spatial.distance.euclidean(fv1, fv2),spatial.distance.euclidean(fv3, fv4),\n                ]\n        return a","execution_count":null},{"metadata":{"id":"B73A05D1C5554BD082C9B1FC017C5893"},"cell_type":"code","outputs":[],"source":"feature = Feature()\ndef func(df):\n    return df.progress_apply(lambda x: feature.output(x['query'], x['title']), axis = 1)\nn_processes = 15\n# 将dataframe分成几部分，用multiprocessing模块下的进程池加速apply的过程\ndata_split = np.array_split(df, n_processes)\npool = multiprocessing.Pool(n_processes)\nc = pd.concat(pool.map(func, data_split)) # 聚合多个进程的apply结果\ndiscrete_features = np.asarray(c.tolist())\nfor i in range(discrete_features.shape[1]):\n    discrete_features[discrete_features[:, i] == None, i] = 0\ndiscrete_features = pd.DataFrame(discrete_features)\ndiscrete_features = discrete_features.fillna(0)\ndiscrete_features = np.array(discrete_features)","execution_count":null},{"metadata":{"id":"1C606B55D4074E45826860B0CA7D02F9"},"cell_type":"code","outputs":[],"source":"import h5py\n# 定义加载和保存hdf5文件的方法\ndef load_hdf5(infile):\n    with h5py.File(infile,\"r\") as f:\n         return f[\"image\"][()]\ndef write_hdf5(arr,outfile):\n  with h5py.File(outfile,\"w\") as f:\n    f.create_dataset(\"image\", data=arr, dtype=arr.dtype)","execution_count":null},{"metadata":{"id":"9E5D8D30AFE84C70891743C59745E954"},"cell_type":"code","outputs":[],"source":"# 用hdf5格式保存得到的特征\nwrite_hdf5(discrete_features[:train_number], '/home/kesci/work/train_data/train_feature')\nwrite_hdf5(discrete_features[train_n\\umber:train_number+test1_number], '/home/kesci/work/testdata1/test1_feature')\nwrite_hdf5(discrete_features[train_number+test1_number:], '/home/kesci/work/testdata2/test2_feature')","execution_count":null},{"metadata":{"id":"02E47A4C01224FC0B0C98F775DE42030","mdEditEnable":false},"cell_type":"markdown","source":"## 4.4 pad_sequence，用于神经网络训练"},{"metadata":{"id":"ADC678044B874418AE08D4A2262B87A9"},"cell_type":"code","outputs":[],"source":"# 分别用长度15，30对query和title进行前向padding和向后trucating\n# 保存数据\nx1 = df['query'].str.split().values\nx1 = keras.preprocessing.sequence.pad_sequences(x1, maxlen=15)\nwrite_hdf5(x1[:train_number], \"train_data/train_final_query\")\nwrite_hdf5(x1[train_n\\umber:train_number+test1_number], \"testdata1/testdata1_query\")\nwrite_hdf5(x1[train_number+test1_number:], \"testdata2/test2_final_query\")\n\nx2 = df['title'].str.split().values\nx2 = keras.preprocessing.sequence.pad_sequences(x2, maxlen=30)\nwrite_hdf5(x2[:train_number], \"train_data/train_final_title\")\nwrite_hdf5(x2[train_n\\umber:train_number+test1_number], \"testdata1/testdata1_title\")\nwrite_hdf5(x2[train_number+test1_number:], \"testdata2/test2_final_title\")","execution_count":null},{"metadata":{"id":"7190AC5D16FC44B39DAF2E5E360ADED6","mdEditEnable":false},"cell_type":"markdown","source":"# 5. lightgbm模型\n使用了3000W训练集，使用了步骤4中除了4.4之外的全部特征，进行5折训练"},{"metadata":{"id":"A6ABA86F0684482480BB50222596B375","mdEditEnable":false},"cell_type":"markdown","source":"## 5.1 载入数据"},{"metadata":{"id":"805C8CD433914A65B801C23796E503E0"},"cell_type":"code","outputs":[],"source":"from scipy import sparse\nimport pandas as pd\nimport h5py\nfrom sklearn.model_selection import train_test_split\nimport pickle\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nimport gensim\nimport gzip\ndef load_hdf5(infile):\n    with h5py.File(infile,\"r\") as f:\n         return f[\"image\"][()]\n\n# 载入数据\nTRAIN_TF_FEATURE_PATH = '/home/kesci/work/train_data/train_final_tfidf'\nTRAIN_CV_FEATURE_PATH = '/home/kesci/work/train_data/train_final_cv'\nTRAIN_QUERY_TITLE_PATH = '/home/kesci/work/train_data/train_final_nunique.pickle'\nTRAIN_Y_PATH = '/home/kesci/work/train_data/train_final_y'\n\nTEST_QUERY_TITLE_PATH = '/home/kesci/work/testdata2/test2_nunique.pickle'\nTEST_TF_FEATURE_PATH = '/home/kesci/work/testdata2/test2_final_tfidf'\nTEST_CV_FEATURE_PATH = '/home/kesci/work/testdata2/test2_final_cv'\n\nwith open(TRAIN_Y_PATH, mode=\"rb\") as f:\n    train_y = pickle.load(f)\nwith open(TRAIN_TF_FEATURE_PATH, mode=\"rb\") as f:\n    tf_idf_feature = pickle.load(f)\nwith open(TRAIN_CV_FEATURE_PATH, mode=\"rb\") as f:\n    tf_cv_feature = pickle.load(f)\nwith open(TRAIN_QUERY_TITLE_PATH, mode=\"rb\") as f:\n    query_title_feature = pickle.load(f)\ntrain_y = [int(i) for i in train_y]\ntrain_y = np.array(train_y)\nfeature = load_hdf5(\"/home/kesci/work/train_data/train_feature\")\ntrain_data = np.hstack((feature, tf_idf_feature,tf_cv_feature,query_title_feature))","execution_count":null},{"metadata":{"id":"75553DCE307445FC9C1C57F9B5DAF0C8","mdEditEnable":false},"cell_type":"markdown","source":"## 5.2 五折训练"},{"metadata":{"id":"6DBAD0D1B8034D9799D0340A9D2DA70D"},"cell_type":"code","outputs":[],"source":"NFOLDS = 5 # 设定5折\nkfold = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=2019)\nkf = kfold.split(train_data, train_y)\n# 超参数设置\nparams = {\n    'boosting_type':'gbdt',\n    'num_leaves':128,\n    'reg_alpha':5,\n    'reg_lambda':5,\n    'n_estimators':4053,\n    'objective':'binary',\n    'subsample':0.7,\n    'colsample_bytree':0.7,\n    'subsample_freq':1,\n    'learning_rate':0.05,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'random_state':8012,\n    'n_jobs':15,\n    'metric':['auc','binary_logloss'],\n    'verbosity':-1,\n}\n# 执行训练\nfor i, (train_fold, validate) in enumerate(kf):\n    if i == 0:\n        continue\n    if i == 1:\n        continue\n    print(i)\n    X_train, X_validate, label_train, label_validate = train_data[train_fold, :], train_data[validate, :], train_y[train_fold], train_y[validate]\n\n    dtrain = lgb.Dataset(X_train, label_train)\n    dvalid = lgb.Dataset(X_validate, label_validate, reference=dtrain)\n\n    bst = lgb.train(params, dtrain, num_boost_round=10000, valid_sets=dvalid, \n    verbose_eval=10, early_stopping_rounds=100)\n    # 保存训练好的模型\n    with open(\"/home/kesci/work/5_lgb_cv\" + str(i), mode=\"wb\") as f:\n        pickle.dump(bst, f)","execution_count":null},{"metadata":{"id":"0A8327700CA74FF6865C0D75A8C381BC","mdEditEnable":false},"cell_type":"markdown","source":"## 5.3 lgb模型训练集预测\n将预测所得的lgb5折特征也喂入神经网络"},{"metadata":{"id":"8404E4E5359645B1892EFABC19C183D9"},"cell_type":"code","outputs":[],"source":"b = np.zeros((len(train_data), 5))\nfor i in range(5):\n    with open(\"/home/kesci/work/5_lgb_cv\" + str(i), mode=\"rb\") as f:\n        gbm = pickle.load(f)\n    b[:,i] = gbm.predict(train_data)\n\n# 将预测的结果保存下来，作为神经网络的输入之一（模型stacking）\nwith open(\"/home/kesci/work/train_data/train_final_lgb_feature\", mode=\"wb\") as f:\n    pickle.dump(b, f)","execution_count":null},{"metadata":{"id":"91E9972F6F594CB6B603F5C68FA47622","mdEditEnable":false},"cell_type":"markdown","source":"## 5.4 lgb模型测试集B预测"},{"metadata":{"id":"9AE20E5B8E3D40F5B16E03CDF571E4F5"},"cell_type":"code","outputs":[],"source":"with open(TEST_TF_FEATURE_PATH, mode=\"rb\") as f:\n    tf_idf_feature = pickle.load(f)\nwith open(TEST_CV_FEATURE_PATH, mode=\"rb\") as f:\n    tf_cv_feature = pickle.load(f)\nwith open(TEST_QUERY_TITLE_PATH, mode=\"rb\") as f:\n    query_title_feature = pickle.load(f)\nfeature = load_hdf5('/home/kesci/work/testdata2/test2_feature')\ntest_data = np.hstack((feature, tf_idf_feature,tf_cv_feature,query_title_feature))\nb = np.zeros((len(test_data), 5))\nfor i in range(5):\n    with open(\"/home/kesci/work/5_lgb_cv\" + str(i), mode=\"rb\") as f:\n        gbm = pickle.load(f)\n    b[:,i] = gbm.predict(test_data)\n    \n# 将预测的结果保存下来，作为神经网络的输入之一（模型stacking）\nwith open(\"/home/kesci/work/testdata2/test2_lgb_feature\", mode=\"wb\") as f:\n    pickle.dump(b, f)","execution_count":null},{"metadata":{"id":"3ED157E94AE543E580E8B444861ED618","mdEditEnable":false},"cell_type":"markdown","source":"# 6. esim模型\n最终提交的是两个esim模型，两个模型的主要区别是embedding词向量不同，一个是word2vec+fasttext加权平均，一个仅仅是word2vec"},{"metadata":{"id":"EF9B8259531343D090E386D21BC01DF9","mdEditEnable":false},"cell_type":"markdown","source":"## 6.1 word2vec和fasttext提取embedding矩阵"},{"metadata":{"id":"9BF85A3617C846E89E22D9677E091F7B"},"cell_type":"code","outputs":[],"source":"import numpy as np\nimport pickle\n\nWV_MODEL_PATH = \"/home/kesci/work/helper_model/w2v_final2.model\"\nFAST_MODEL_PATH = \"/home/kesci/work/helper_model/fasttext_final.model\"\nWV_EMBEDDING_PATH1 = \"features/word2vec_embedding1.pickle\"\nWV_EMBEDDING_PATH2 = \"features/word2vec_embedding2.pickle\"\nFT_EMBEDDING_PATH1 = \"features/fasttext_embedding1.pickle\"\nFT_EMBEDDING_PATH2 = \"features/fasttext_embedding2.pickle\"\n\nclass EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n    \"Callback to save model after every epoch\"\n\n    def __init__(self, path_prefix, epochs=0):\n        self.path_prefix = path_prefix\n        self.epoch = epochs\n\n    def on_epoch_end(self, model):\n        output_path = self.path_prefix\n        print(\"Save model to {}\".format(output_path))\n        model.save(output_path)\n        self.epoch += 1\n\nfastvec_model = fasttext.FastText.load(FAST_MODEL_PATH)\nwordvec_model = word2vec.Word2Vec.load(WV_MODEL_PATH)\n\nn = 2000000\nchunk_size = 1000000\n'''生成w2v, ft两个词嵌入矩阵并保存'''\n# ----- word2vec ------\nw2_embeddings_matrix = np.zeros((n, wordvec_model.vector_size))\nkeys = wordvec_model.wv.vocab.keys()\nfor word in range(n):\n    if str(word) in keys:\n        w2_embeddings_matrix[word] = wordvec_model[str(word)]\nwith open(WV_EMBEDDING_PATH1, mode=\"wb\") as f:\n    pickle.dump(w2_embeddings_matrix[:chunk_size], f)\nwith open(WV_EMBEDDING_PATH2, mode=\"wb\") as f:\n    pickle.dump(w2_embeddings_matrix[chunk_size:], f)\n\n\n# ----- fasttext ------\nft_embeddings_matrix = np.zeros((n, fastvec_model.vector_size))\nkeys = fastvec_model.wv.vocab.keys()\nfor word in range(n):\n    if str(word) in keys:\n        ft_embeddings_matrix[word] = fastvec_model[str(word)]\nwith open(FT_EMBEDDING_PATH1, mode=\"wb\") as f:\n    pickle.dump(ft_embeddings_matrix[:chunk_size], f)\nwith open(FT_EMBEDDING_PATH2, mode=\"wb\") as f:\n    pickle.dump(ft_embeddings_matrix[chunk_size:], f)","execution_count":null},{"metadata":{"id":"8D5C4EA19603495AB4B1E71C09ECD783","mdEditEnable":false},"cell_type":"markdown","source":"## 6.2 神经网络训练数据准备\n两个esim的不同只是embedding_matrix的不同，以此来保证模型的多样性，提高融合收益"},{"metadata":{"id":"224E851597284EA190360A7338433981"},"cell_type":"code","outputs":[],"source":"import os\nimport gc\nimport csv\nimport time\nimport math\nimport random\nimport pickle\nimport inspect\nimport multiprocessing\nfrom collections.abc import Iterable\nfrom scipy import spatial\n\nfrom scipy import sparse\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport pickle\nimport numpy as np\nimport lightgbm as lgb\nfrom scipy import sparse\nfrom gensim.models import word2vec,fasttext\nimport gensim\nimport gzip\nfrom keras.callbacks import *\nimport numpy as np\nfrom numpy import random\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom gensim.models import Word2Vec, FastText, word2vec\nimport tensorflow as tf\nimport keras\nimport keras.backend as K\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.layers.embeddings import Embedding\nfrom keras.initializers import glorot_uniform\nfrom keras.layers.noise import GaussianNoise\nfrom keras import optimizers\nfrom keras import regularizers\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.activations import softmax\nfrom keras.optimizers import Nadam, Adam\nfrom keras.models import load_model\nimport pickle\nfrom tqdm import tqdm\ntqdm.pandas()\nimport pickle\nfrom typing import List\nimport h5py\ndef load_hdf5(infile):\n    with h5py.File(infile,\"r\") as f:\n         return f[\"image\"][()]\ndef write_hdf5(arr,outfile):\n  with h5py.File(outfile,\"w\") as f:\n    f.create_dataset(\"image\", data=arr, dtype=arr.dtype)","execution_count":null},{"metadata":{"id":"F1878AB290E04C98A27C4841F8419215"},"cell_type":"code","outputs":[],"source":"WV_EMBEDDING_PATH1 = \"/home/kesci/work/embedding/word2vec_embedding1.pickle\"\nWV_EMBEDDING_PATH2 = \"/home/kesci/work/embedding/word2vec_embedding2.pickle\"\nFT_EMBEDDING_PATH1 = \"/home/kesci/work/embedding/fasttext_embedding1.pickle\"\nFT_EMBEDDING_PATH2 = \"/home/kesci/work/embedding/fasttext_embedding2.pickle\"\n\nwith open(WV_EMBEDDING_PATH1, mode=\"rb\") as f:\n    embed_matrix11 = pickle.load(f)\nwith open(WV_EMBEDDING_PATH2, mode=\"rb\") as f:\n    embed_matrix12 = pickle.load(f)\nwith open(FT_EMBEDDING_PATH1, mode=\"rb\") as f:\n    embed_matrix21 = pickle.load(f)\nwith open(FT_EMBEDDING_PATH2, mode=\"rb\") as f:\n    embed_matrix22 = pickle.load(f)\n    \nembed_matrix1 = np.vstack([embed_matrix11, embed_matrix12]).astype('float32')\nembed_matrix2 = np.vstack([embed_matrix21, embed_matrix22]).astype('float32')\n\n# 模型一的embedding_matrix\nembed_matrix = embed_matrix2\n\n# 模型二的embedding_matrix\nembed_matrix = 0.5 * embed_matrix1 + 0.5 * embed_matrix2","execution_count":null},{"metadata":{"id":"73701933DBBE4E8B8CEF947C605E99CF"},"cell_type":"code","outputs":[],"source":"query = load_hdf5(\"train_data/train_final_query\")\ntitle = load_hdf5(\"train_data/train_final_title\")\n# 加载之前提取的特征\nwith open(\"/home/kesci/work/train_data/train_final_lgb_feature\", mode=\"rb\") as f:\n    lgb_feature = pickle.load(f)\nTRAIN_TF_FEATURE_PATH = '/home/kesci/work/train_data/train_final_tfidf'\nTRAIN_CV_FEATURE_PATH = '/home/kesci/work/train_data/train_final_cv'\nTRAIN_QUERY_TITLE_PATH = '/home/kesci/work/train_data/train_final_nunique.pickle'\nTRAIN_Y_PATH = '/home/kesci/work/train_data/train_final_y'\nwith open(TRAIN_Y_PATH, mode=\"rb\") as f:\n    train_y = pickle.load(f)\nwith open(TRAIN_TF_FEATURE_PATH, mode=\"rb\") as f:\n    tf_idf_feature = pickle.load(f)\nwith open(TRAIN_CV_FEATURE_PATH, mode=\"rb\") as f:\n    tf_cv_feature = pickle.load(f)\nwith open(TRAIN_QUERY_TITLE_PATH, mode=\"rb\") as f:\n    query_title_feature = pickle.load(f)\ntrain_y = [int(i) for i in train_y]\ntrain_y = np.array(train_y)\ntrain_data = np.hstack((tf_idf_feature,tf_cv_feature,query_title_feature,lgb_feature))","execution_count":null},{"metadata":{"id":"9A4AAD6DEF47488FA0081ABC863FB052"},"cell_type":"code","outputs":[],"source":"X = np.array(range(len(train_data)))\ny = np.zeros(len(train_data))\n# 划分测试集和验证集\ntrain_index, test_index, _, _ = train_test_split(X, y, test_size=0.01, random_state=42)\ntrain_features = train_data[train_index, :]\nval_features = train_data[test_index, :]\ntrain_query = query[train_index,:]\nval_query = query[test_index, :]\ntrain_title = title[train_index,:]\nval_title = title[test_index, :]\ny_train = train_y[train_index]\ny_val = train_y[test_index]","execution_count":null},{"metadata":{"id":"06441B8C85874AF2A14CBCA80F3D7DA1","mdEditEnable":false},"cell_type":"markdown","source":"## 6.3 模型训练"},{"metadata":{"id":"1F395B195C2343C8A229857308060710"},"cell_type":"code","outputs":[],"source":"def auc(y_true, y_pred):\n    '''计算auc的metric'''\n    auc = tf.metrics.auc(y_true, y_pred)[1]\n    K.get_session().run(tf.local_variables_initializer())\n    return auc\n\n# 以下是一些自定义的计算函数，用于神经网络的构建\ndef unchanged_shape(input_shape):\n    return input_shape\n\ndef substract(input_1, input_2):\n    neg_input_2 = Lambda(lambda x: -x, output_shape=unchanged_shape)(input_2)\n    out_ = Add()([input_1, neg_input_2])\n    return out_\n\n\ndef submult(input_1, input_2):\n    \"Get multiplication and subtraction then concatenate results\"\n    mult = Multiply()([input_1, input_2])\n    sub = substract(input_1, input_2)\n    out_ = Concatenate()([sub, mult])\n    return out_\n\n\ndef apply_multiple(input_, layers):\n    \"Apply layers to input then concatenate result\"\n    if not len(layers) > 1:\n        raise ValueError('Layers list should contain more than 1 layer')\n    else:\n        agg_ = []\n        for layer in layers:\n            agg_.append(layer(input_))\n        out_ = Concatenate()(agg_)\n    return out_\n\n\ndef time_distributed(input_, layers):\n    out_ = []\n    node_ = input_\n    for layer_ in layers:\n        node_ = TimeDistributed(layer_)(node_)\n    out_ = node_\n    return out_\n\n\ndef soft_attention_alignment(input_1, input_2):\n    attention = Dot(axes=-1)([input_1, input_2])\n\n    w_att_1 = Lambda(lambda x: softmax(x, axis=1),\n                     output_shape=unchanged_shape)(attention)\n    # 这里相当于矩阵转置了\n    w_att_2 = Permute((2, 1))(Lambda(lambda x: softmax(x, axis=2),\n                                     output_shape=unchanged_shape)(attention))\n\n    in1_aligned = Dot(axes=1)([w_att_1, input_1])\n    in2_aligned = Dot(axes=1)([w_att_2, input_2])\n    return in1_aligned, in2_aligned\n\n\nclass AttentivePoolingLayer(Layer):\n    def __init__(self, **kwargs):\n        super(AttentivePoolingLayer, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        # 定义需要学习的参数\n        if isinstance(input_shape, list):\n            input_shape = input_shape[0]\n        feature_dim = input_shape[-1]\n        self.U = self.add_weight(shape=(feature_dim, feature_dim), name='U', initializer='uniform', trainable=True)\n        self.built = True\n        super(AttentivePoolingLayer, self).build(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        feature_dim = input_shape[0][-1]\n        return [(input_shape[0][0], feature_dim),(input_shape[1][0], feature_dim)]\n        # return input_shape\n\n    def call(self, inputs, **kwargs):\n        q = inputs[0]\n        a = inputs[1]\n\n        time_step = inputs[0].shape[1].value\n        feature_dim = inputs[0].shape[2].value\n\n        G = tf.tensordot(q, self.U, axes=1)\n        G = tf.reshape(G, shape=(-1, time_step, feature_dim), )\n        G = tf.matmul(G, a, transpose_b=True)\n        G = tf.nn.tanh(G)\n\n        g_q = tf.reduce_max(G, axis=-1)\n        g_a = tf.reduce_max(G, axis=1)\n\n        g_q = tf.expand_dims(g_q, axis=-1)\n        g_a = tf.expand_dims(g_a, axis=-1)\n\n        g_q = tf.nn.softmax(g_q, dim=1)\n        g_a = tf.nn.softmax(g_a, dim=1)\n\n        # q = q * g_q\n        # a = a * g_a\n\n        q = tf.matmul(q, g_q, transpose_a=True)\n        q = tf.squeeze(q, [-1])\n        a = tf.matmul(a, g_a, transpose_a=True)\n        a = tf.squeeze(a, [-1])\n\n        return [q, a]","execution_count":null},{"metadata":{"id":"DF47E32A623A4A138623473D3AEF72C8","mdEditEnable":false},"cell_type":"markdown","source":"**模型的一些超参数**"},{"metadata":{"id":"8558FC30D8D04DAE9C84FD08D4DBA85D"},"cell_type":"code","outputs":[],"source":"dense_dim=200\nlstm_num = 30\nlstm_drop = 0.3\nBATCH_SIZE = 2000\nMAX_SEQUENCE_LENGTH1 = 15\nMAX_SEQUENCE_LENGTH2 = 30\nEMBEDDING_DIM = 300\nEPOCHS = 10\ndense_dropout = 0.3\nnum_heads=3\nunits=20\nspatial_dropout=0.3","execution_count":null},{"metadata":{"id":"4A88B7DD52B24A90A9EA9655BAE9E734","mdEditEnable":false},"cell_type":"markdown","source":"**模型构建**"},{"metadata":{"id":"7759AE9D15A143908793D1EA28E84D1C"},"cell_type":"code","outputs":[],"source":"q1 = Input(name='q1', shape=(MAX_SEQUENCE_LENGTH1,))\nq2 = Input(name='q2', shape=(MAX_SEQUENCE_LENGTH2,))\n\n# Embedding\neb_layer = Embedding(input_dim=embed_matrix.shape[0], output_dim=EMBEDDING_DIM,\n        weights=[embed_matrix], trainable=False)\nbn = BatchNormalization(axis=2)\nq1_embed = bn(eb_layer(q1))\nq2_embed = bn(eb_layer(q2))\n\n# Encode\nencode = Bidirectional(CuDNNLSTM(30, return_sequences=True))\nq1_encoded = encode(q1_embed)\nq2_encoded = encode(q2_embed)\n\n# Attention\nq1_aligned, q2_aligned = soft_attention_alignment(q1_encoded, q2_encoded)\n\n# Compose\nq1_combined = Concatenate()([q1_encoded, q2_aligned, submult(q1_encoded, q2_aligned)])\nq2_combined = Concatenate()([q2_encoded, q1_aligned, submult(q2_encoded, q1_aligned)])\n\ncompose = Bidirectional(CuDNNLSTM(30, return_sequences=True))\nq1_compare = compose(q1_combined)\nq2_compare = compose(q2_combined)\n\n# Aggregate\nq1_rep = apply_multiple(q1_compare, [GlobalAvgPool1D(), GlobalMaxPool1D()])\nq2_rep = apply_multiple(q2_compare, [GlobalAvgPool1D(), GlobalMaxPool1D()])\n\n# magic featurues\nmagic_input = Input(shape=(13,))\nmagic_dense = BatchNormalization()(magic_input)\nmagic_dense = Dense(64, activation='relu')(magic_dense)\nmagic_dense = Dropout(0.3)(magic_dense)\n\n# Classifier\nmerged = Concatenate()([q1_rep, q2_rep, magic_dense])\nx = merged\ndense = BatchNormalization()(merged)\ndense = Dense(dense_dim, activation='elu')(dense)\ndense = BatchNormalization()(dense)\ndense = Dropout(dense_dropout)(dense)\ndense = Dense(dense_dim, activation='elu')(dense)\ndense = BatchNormalization()(dense)\ndense = Dropout(dense_dropout)(dense)\nout_ = Dense(1, activation='sigmoid')(dense)\n\nmodel = Model(inputs=[q1, q2, magic_input], outputs=out_)\nmodel.compile(optimizer=Adam(lr=1e-3), loss='binary_crossentropy', metrics=[auc])","execution_count":null},{"metadata":{"id":"9D98D7AB16EC440C945C6CCBF92C8586","mdEditEnable":false},"cell_type":"markdown","source":"**回调函数**"},{"metadata":{"id":"B3C7F9CA97BA42C588F6B564ECF0F4A5"},"cell_type":"code","outputs":[],"source":"class roc_callback(keras.callbacks.Callback):\n    '''指数衰减学习率的回调函数'''\n    def on_epoch_end(self, epoch, logs={}): \n        # 改变学习率\n        old_lr = float(K.get_value(self.model.optimizer.lr))\n        new_lr = old_lr * 0.9\n        K.set_value(self.model.optimizer.lr, new_lr)\n        print(f'学习率由{old_lr:.6f}调整为{new_lr:.6f}')\n        return\n    \nroc_cb = roc_callback()\n# 用验证集上的loss判断是否需要提前结束\ncb= EarlyStopping(monitor=\"val_loss\", patience=4, verbose=1, mode='min')\n# 及时保存模型\ncheckpoint = ModelCheckpoint(filepath='models/esim1.h5', monitor='val_loss',verbose=1,save_best_only='True',mode='min',period=1)","execution_count":null},{"metadata":{"id":"3024B8C1A41A4B0883D16273ECC7F812","mdEditEnable":false},"cell_type":"markdown","source":"**fit**"},{"metadata":{"id":"76CA946DF4FF45AC97E4DC09A8BD4A98"},"cell_type":"code","outputs":[],"source":"hist = model.fit([train_query, train_title, train_features], y_train, batch_size=4000, \nepochs=30 , verbose=1, shuffle = True, \nvalidation_data = ([val_query, val_title, val_features], y_val),\ncallbacks=[cb,checkpoint, roc_cb])","execution_count":null},{"metadata":{"id":"205EE79D2B504C278CC3A89CCF8750B4","collapsed":false,"scrolled":false,"mdEditEnable":false},"cell_type":"markdown","source":"## 6.4 模型预测"},{"metadata":{"id":"D5502E828BC741CA8E75EAEA066BB3FF"},"cell_type":"code","outputs":[],"source":"# 加载模型和数据\nmodel1 = load_model(\"models/esim1.h5\", custom_objects={'auc':auc, \"softmax\":softmax})\nmodel2 = load_model(\"models/esim2.h5\", custom_objects={'auc':auc, \"softmax\":softmax})\nwith open(\"/home/kesci/work/testdata2/test2_lgb_feature\", mode=\"rb\") as f:\n    lgb_feature = pickle.load(f)\nTEST_TF_FEATURE_PATH = '/home/kesci/work/testdata2/test2_final_tfidf'\nTEST_CV_FEATURE_PATH = '/home/kesci/work/testdata2/test2_final_cv'\nTEST_QUERY_TITLE_PATH = '/home/kesci/work/testdata2/test2_nunique.pickle'\nwith open(TEST_TF_FEATURE_PATH, mode=\"rb\") as f:\n    tf_idf_feature = pickle.load(f)\nwith open(TEST_CV_FEATURE_PATH, mode=\"rb\") as f:\n    tf_cv_feature = pickle.load(f)\nwith open(TEST_QUERY_TITLE_PATH, mode=\"rb\") as f:\n    query_title_feature = pickle.load(f)\ntest_data = np.hstack((tf_idf_feature,tf_cv_feature,query_title_feature,lgb_feature))\n\nquery = load_hdf5(\"testdata2/test2_final_query\")\ntitle = load_hdf5(\"testdata2/test2_final_title\")\n\n# 进行预测并保存结果\nnn_pred1 = model1.predict([query, title, test_data], verbose=1)\nnn_pred2 = model2.predict([query, title, test_data], verbose=1)\n\nwith open(\"testdata2/nn_pred1.pickle\", mode=\"wb\") as f:\n    pickle.dump(nn_pred1, f)\nwith open(\"testdata2/nn_pred2.pickle\", mode=\"wb\") as f:\n    pickle.dump(nn_pred2, f)","execution_count":null},{"metadata":{"id":"35CF53443701410A80AFBA396DD98D6F","mdEditEnable":false},"cell_type":"markdown","source":"# 7. 模型融合与提交"},{"metadata":{"id":"DD0DF314929D4F0E841005BB2EF6C83B"},"cell_type":"code","outputs":[],"source":"test = pd.read_csv('/home/kesci/input/bytedance/bytedance_contest.final_2.csv', header=None,\n    names=['query_id', 'query', 'query_title_id', 'title'], \n    usecols=['query_id', 'query_title_id'])\nwith open(\"testdata2/nn_pred1.pickle\", mode=\"rb\") as f:\n    nn_pred1 = pickle.load(f) # 第一个esim的预测结果\ntest['pred'] = nn_pred1\ntest.to_csv('submit_nn1', header=False, index=False)\nwith open(\"testdata2/nn_pred2.pickle\", mode=\"rb\") as f:\n    nn_pred2 = pickle.load(f) # 第二个esim的预测结果\ntest['pred'] = nn_pred2\ntest.to_csv('submit_nn2', header=False, index=False)\nsub1 = pd.read_csv('submit_nn1', header=None)\nsub2 = pd.read_csv('submit_nn2', header=None)\n\nsub1[2] = 0.53 * sub1[2] + 0.47 * sub2[2] # 两个esim模型的预测结果加权平均，这也是我们在测试集B上的最终提交\nsub1[2] = sub1[2].astype('float32')\nsub1.to_csv('submit_ronghe1.csv', header=False, index=False)\n!https_proxy=\"http://klab-external-proxy\" ./kesci_submit -file submit_ronghe1.csv -token 99ff95d5193957a0 -mode archive","execution_count":null}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":0}