get_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import operator\n",
    "import pandas as pd\n",
    "import jieba\n",
    "from language.langconv import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105575\n",
      "['在 这里 了', '哦 了 哦 了 哦 了 , 咱聊点 别的 吧', '不要 这样 说 嘛 ！ 很 不 文明 哦', '无法 理解 您 的话 ， 获取 帮助 请 发送   help', '今天 这 天气 很 适合 聊天 的 说', '好话 不 分 轻重 ！', '夜曲 小 三', '如果 你 来 勾引 我 ， 我 一定 假装 上当 ！', '乱 什么 呢 ?', '最 喜欢 毛 鸭子 了 ， 又 乖 又 听话 就 象 我 一样 ！']\n"
     ]
    }
   ],
   "source": [
    "def Traditional2Simplified(sentence):\n",
    "    sentence = Converter('zh-hans').convert(sentence)\n",
    "    return sentence\n",
    "def is_all_chinese(strs):\n",
    "    for chart in strs:\n",
    "        if chart < u'\\u4e00' or chart > u'\\u9fff':\n",
    "            return False\n",
    "    return True\n",
    "with open('qingyun.tsv', 'r', encoding='utf-8') as f:\n",
    "    lines = f.read().split('\\n')\n",
    "    lines = lines[:-2]\n",
    "question = []\n",
    "answer = []\n",
    "for pos, line in enumerate(lines):\n",
    "    if '\\t' not in line:\n",
    "        print(line)\n",
    "    line = line.split('\\t')\n",
    "    q = line[0].strip()\n",
    "    a = line[1].strip()\n",
    "    question.append(' '.join(jieba.lcut(Traditional2Simplified(q).strip(), cut_all=False)))\n",
    "    answer.append(' '.join(jieba.lcut(Traditional2Simplified(a).strip(), cut_all=False)))\n",
    "\n",
    "print(len(question))\n",
    "print(answer[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stop_words:  395\n"
     ]
    }
   ],
   "source": [
    "def is_all_chinese(strs):\n",
    "    for chart in strs:\n",
    "        if chart < u'\\u4e00' or chart > u'\\u9fff':\n",
    "            return False\n",
    "    return True\n",
    "character = set()\n",
    "for seq in question + answer:\n",
    "    word_list = seq.split(' ')\n",
    "    for word in word_list:\n",
    "        if not is_all_chinese(word):\n",
    "            character.add(word)\n",
    "def is_pure_english(keyword):  \n",
    "    return all(ord(c) < 128 for c in keyword)\n",
    "character=list(character)\n",
    "stop_words = set()\n",
    "for pos, word in enumerate(character):\n",
    "    if not is_pure_english(word):\n",
    "        stop_words.add(word)\n",
    "print('stop_words: ', len(stop_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['南京 在 哪里', '咋 死 ? ? ? 红烧 还是 爆炒', '你 个 小 骚货 哥哥 的 巴操 你 爽 不爽', '额 麻麻 怎么 会 有 那 玩意儿', '拿尿冲 一冲', '那 重点 是 什么', '章子怡 新宠', '求 勾引', '说 不 粗来', '喜欢 毛线']\n",
      "['BOS 在 这里 了 EOS', 'BOS 哦 了 哦 了 哦 了 , 咱聊点 别的 吧 EOS', 'BOS 不要 这样 说 嘛 很 不 文明 哦 EOS', 'BOS 无法 理解 您 的话 获取 帮助 请 发送   help EOS', 'BOS 今天 这 天气 很 适合 聊天 的 说 EOS', 'BOS 好话 不 分 轻重 EOS', 'BOS 夜曲 小 三 EOS', 'BOS 如果 你 来 勾引 我 我 一定 假装 上当 EOS', 'BOS 乱 什么 呢 ? EOS', 'BOS 最 喜欢 毛 鸭子 了 又 乖 又 听话 就 象 我 一样 EOS']\n",
      "['在 这里 了 EOS', '哦 了 哦 了 哦 了 , 咱聊点 别的 吧 EOS', '不要 这样 说 嘛 很 不 文明 哦 EOS', '无法 理解 您 的话 获取 帮助 请 发送   help EOS', '今天 这 天气 很 适合 聊天 的 说 EOS', '好话 不 分 轻重 EOS', '夜曲 小 三 EOS', '如果 你 来 勾引 我 我 一定 假装 上当 EOS', '乱 什么 呢 ? EOS', '最 喜欢 毛 鸭子 了 又 乖 又 听话 就 象 我 一样 EOS']\n"
     ]
    }
   ],
   "source": [
    "maxLen=18\n",
    "for pos, seq in enumerate(question):\n",
    "    seq_list = seq.split(' ')\n",
    "    for epoch in range(3):\n",
    "        for pos_, word in enumerate(seq_list):\n",
    "            if word in stop_words:\n",
    "                seq_list.pop(pos_)\n",
    "    if len(seq_list) > maxLen:\n",
    "        seq_list = seq_list[:maxLen]\n",
    "    question[pos] = ' '.join(seq_list)\n",
    "for pos, seq in enumerate(answer):\n",
    "    seq_list = seq.split(' ')\n",
    "    for epoch in range(3):\n",
    "        for pos_, word in enumerate(seq_list):\n",
    "            if word in stop_words:\n",
    "                seq_list.pop(pos_)\n",
    "    if len(seq_list) > maxLen:\n",
    "        seq_list = seq_list[:maxLen]\n",
    "    answer[pos] = ' '.join(seq_list)\n",
    "    \n",
    "answer_a = ['BOS ' + i + ' EOS' for i in answer]\n",
    "answer_b = [i + ' EOS' for i in answer]\n",
    "print(question[:10])\n",
    "print(answer_a[:10])\n",
    "print(answer_b[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_bag:  43349\n"
     ]
    }
   ],
   "source": [
    "import  pickle\n",
    "counts = {}\n",
    "BE = ['BOS', 'EOS']\n",
    "for word_list in question + answer + BE:\n",
    "    for word in word_list.split(' '):\n",
    "        counts[word] = counts.get(word, 0) + 1 \n",
    "word_to_index = {}\n",
    "for pos, i in enumerate(counts.keys()):\n",
    "    word_to_index[i] = pos\n",
    "    \n",
    "index_to_word = {}\n",
    "for pos, i in enumerate(counts.keys()):\n",
    "    index_to_word[pos] = i\n",
    "    \n",
    "vocab_bag =list(word_to_index.keys())\n",
    "with open('word_to_index.pkl', 'wb') as f:\n",
    "    pickle.dump(word_to_index, f, pickle.HIGHEST_PROTOCOL)\n",
    "with open('index_to_word.pkl', 'wb') as f:\n",
    "    pickle.dump(index_to_word, f, pickle.HIGHEST_PROTOCOL)\n",
    "with open('vocab_bag.pkl', 'wb') as f:\n",
    "    pickle.dump(vocab_bag, f, pickle.HIGHEST_PROTOCOL)\n",
    "print('vocab_bag: ', len(vocab_bag))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[list([0, 1, 2]) list([3, 4, 5, 5, 5, 6, 7, 8])\n",
      " list([9, 10, 11, 12, 13, 14, 15, 9, 16, 17])]\n",
      "[list([43347, 1, 1838, 56, 43348])\n",
      " list([43347, 213, 56, 213, 56, 213, 56, 59, 36045, 511, 193, 43348])\n",
      " list([43347, 384, 1480, 34, 201, 77, 35, 1325, 213, 43348])]\n",
      "[list([1, 1838, 56, 43348])\n",
      " list([213, 56, 213, 56, 213, 56, 59, 36045, 511, 193, 43348])\n",
      " list([384, 1480, 34, 201, 77, 35, 1325, 213, 43348])]\n"
     ]
    }
   ],
   "source": [
    "question = np.array([[word_to_index[w] for w in i.split(' ')] for i in question])\n",
    "answer_a = np.array([[word_to_index[w] for w in i.split(' ')] for i in answer_a])\n",
    "answer_b = np.array([[word_to_index[w] for w in i.split(' ')] for i in answer_b])\n",
    "print(question[:3])\n",
    "print(answer_a[:3])\n",
    "print(answer_b[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question:  105575 \n",
      " answer:  105575\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "import numpy as np\n",
    "print('question: ', len(question), '\\n', 'answer: ', len(answer))\n",
    "np.save('question.npy', question[:100000])\n",
    "np.save('answer_a.npy', answer_a[:100000])\n",
    "np.save('answer_b.npy', answer_b[:100000])\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "answer_a.shape:  (100000,)\n",
      "word_to_vec_map:  43349\n",
      "vocab_size:  43350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43348     2  1839    57 43349     0     0     0     0     0     0     0\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [43348   214    57   214    57   214    57    60 36046   512   194 43349\n",
      "      0     0     0     0     0     0     0     0]\n",
      " [43348   385  1481    35   202    78    36  1326   214 43349     0     0\n",
      "      0     0     0     0     0     0     0     0]]\n",
      "[list([2, 1839, 57, 43349])\n",
      " list([214, 57, 214, 57, 214, 57, 60, 36046, 512, 194, 43349])\n",
      " list([385, 1481, 35, 202, 78, 36, 1326, 214, 43349])]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import operator\n",
    "\n",
    "question = np.load('question.npy')\n",
    "answer_a = np.load('answer_a.npy')\n",
    "answer_b = np.load('answer_b.npy')\n",
    "print('answer_a.shape: ', answer_a.shape)\n",
    "with open('word_to_index.pkl', 'rb') as f:\n",
    "    word_to_index = pickle.load(f)\n",
    "\n",
    "for i, j in word_to_index.items():\n",
    "    word_to_index[i] = j + 1\n",
    "\n",
    "index_to_word = {}\n",
    "for key, value in word_to_index.items():\n",
    "    index_to_word[value] = key\n",
    "pad_question = question\n",
    "pad_answer_a = answer_a\n",
    "pad_answer_b = answer_b\n",
    "maxLen = 20\n",
    "for pos, i in enumerate(pad_question):\n",
    "    for pos_, j in enumerate(i):\n",
    "        i[pos_] = j + 1\n",
    "    if(len(i) > maxLen):\n",
    "        pad_question[pos] = i[:maxLen]\n",
    "    \n",
    "for pos, i in enumerate(pad_answer_a):\n",
    "    for pos_, j in enumerate(i):\n",
    "        i[pos_] = j + 1\n",
    "    if(len(i) > maxLen):\n",
    "        pad_answer_a[pos] = i[:maxLen]\n",
    "for pos, i in enumerate(pad_answer_b):\n",
    "    for pos_, j in enumerate(i):\n",
    "        i[pos_] = j + 1\n",
    "    if(len(i) > maxLen):\n",
    "        pad_answer_b[pos] = i[:maxLen]\n",
    "np.save('answer_o.npy', pad_answer_b)        \n",
    "    \n",
    "with open('vocab_bag.pkl', 'rb') as f:\n",
    "    words = pickle.load(f)\n",
    "vocab_size = len(word_to_index) + 1\n",
    "print('word_to_vec_map: ', len(list(words)))\n",
    "print('vocab_size: ', vocab_size)\n",
    "\n",
    "\n",
    "from keras.preprocessing import sequence\n",
    "#后端padding\n",
    "pad_question = sequence.pad_sequences(pad_question, maxlen=maxLen,\n",
    "                                      dtype='int32', padding='post', \n",
    "                                       truncating='post')\n",
    "pad_answer = sequence.pad_sequences(pad_answer_a, maxlen=maxLen,\n",
    "                                 dtype='int32', padding='post',\n",
    "                                 truncating='post')\n",
    "\n",
    "def get_file_list(file_path):\n",
    "    dir_list = os.listdir(file_path)\n",
    "    if not dir_list:\n",
    "        return\n",
    "    else:\n",
    "        dir_list = sorted(dir_list, key=lambda x: os.path.getmtime(os.path.join(file_path, x)))\n",
    "    return dir_list\n",
    "\n",
    "with open('pad_word_to_index.pkl', 'wb') as f:\n",
    "    pickle.dump(word_to_index, f, pickle.HIGHEST_PROTOCOL)\n",
    "with open('pad_index_to_word.pkl', 'wb') as f:\n",
    "    pickle.dump(index_to_word, f, pickle.HIGHEST_PROTOCOL)\n",
    "np.save('pad_question.npy', pad_question)\n",
    "np.save('pad_answer.npy', pad_answer)\n",
    "    \n",
    "print(pad_answer[:3])\n",
    "print(pad_answer_b[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}