feature_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "official-yahoo",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dramatic-amateur",
   "metadata": {},
   "source": [
    "## Incorprate AOA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "particular-subscription",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predicted_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416763</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416764</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416765</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416766</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416767</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>416768 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        predicted_label\n",
       "0                     1\n",
       "1                     1\n",
       "2                     1\n",
       "3                     1\n",
       "4                     1\n",
       "...                 ...\n",
       "416763                0\n",
       "416764                0\n",
       "416765                0\n",
       "416766                0\n",
       "416767                0\n",
       "\n",
       "[416768 rows x 1 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_label = pd.read_csv('train_data_bert_label.csv', header = 0)[['predicted_label']]\n",
    "bert_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "pharmaceutical-palace",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Alternative.spelling</th>\n",
       "      <th>Freq_pm</th>\n",
       "      <th>Dom_PoS_SUBTLEX</th>\n",
       "      <th>Nletters</th>\n",
       "      <th>Nphon</th>\n",
       "      <th>Nsyll</th>\n",
       "      <th>Lemma_highest_PoS</th>\n",
       "      <th>AoA_Kup</th>\n",
       "      <th>Perc_known</th>\n",
       "      <th>AoA_Kup_lem</th>\n",
       "      <th>Perc_known_lem</th>\n",
       "      <th>AoA_Bird_lem</th>\n",
       "      <th>AoA_Bristol_lem</th>\n",
       "      <th>AoA_Cort_lem</th>\n",
       "      <th>AoA_Schock</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>a</td>\n",
       "      <td>20415.27</td>\n",
       "      <td>Article</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>a</td>\n",
       "      <td>2.89</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.89</td>\n",
       "      <td>1.00</td>\n",
       "      <td>3.16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aardvark</th>\n",
       "      <td>aardvark</td>\n",
       "      <td>0.41</td>\n",
       "      <td>Noun</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>aardvark</td>\n",
       "      <td>9.89</td>\n",
       "      <td>1.00</td>\n",
       "      <td>9.89</td>\n",
       "      <td>1.00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abacus</th>\n",
       "      <td>abacus</td>\n",
       "      <td>0.24</td>\n",
       "      <td>Noun</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>abacus</td>\n",
       "      <td>8.69</td>\n",
       "      <td>0.65</td>\n",
       "      <td>8.69</td>\n",
       "      <td>0.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abacuses</th>\n",
       "      <td>abacuses</td>\n",
       "      <td>0.02</td>\n",
       "      <td>Noun</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>abacus</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.69</td>\n",
       "      <td>0.65</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abalone</th>\n",
       "      <td>abalone</td>\n",
       "      <td>0.51</td>\n",
       "      <td>Verb</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>abalone</td>\n",
       "      <td>12.23</td>\n",
       "      <td>0.72</td>\n",
       "      <td>12.23</td>\n",
       "      <td>0.72</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Alternative.spelling   Freq_pm Dom_PoS_SUBTLEX  Nletters  Nphon  \\\n",
       "Word                                                                       \n",
       "a                           a  20415.27         Article         1      1   \n",
       "aardvark             aardvark      0.41            Noun         8      7   \n",
       "abacus                 abacus      0.24            Noun         6      6   \n",
       "abacuses             abacuses      0.02            Noun         8      9   \n",
       "abalone               abalone      0.51            Verb         7      7   \n",
       "\n",
       "          Nsyll Lemma_highest_PoS  AoA_Kup  Perc_known  AoA_Kup_lem  \\\n",
       "Word                                                                  \n",
       "a             1                 a     2.89        1.00         2.89   \n",
       "aardvark      2          aardvark     9.89        1.00         9.89   \n",
       "abacus        3            abacus     8.69        0.65         8.69   \n",
       "abacuses      4            abacus      NaN         NaN         8.69   \n",
       "abalone       4           abalone    12.23        0.72        12.23   \n",
       "\n",
       "          Perc_known_lem  AoA_Bird_lem  AoA_Bristol_lem  AoA_Cort_lem  \\\n",
       "Word                                                                    \n",
       "a                   1.00          3.16              NaN           NaN   \n",
       "aardvark            1.00           NaN              NaN           NaN   \n",
       "abacus              0.65           NaN              NaN           NaN   \n",
       "abacuses            0.65           NaN              NaN           NaN   \n",
       "abalone             0.72           NaN              NaN           NaN   \n",
       "\n",
       "          AoA_Schock  \n",
       "Word                  \n",
       "a                NaN  \n",
       "aardvark         NaN  \n",
       "abacus           NaN  \n",
       "abacuses         NaN  \n",
       "abalone          NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aoa_df = pd.read_csv('AOA_51715_words.csv', header = 0, index_col = 0)\n",
    "aoa_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "unique-transaction",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 51715 entries, a to zygotes\n",
      "Data columns (total 15 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   Alternative.spelling  51714 non-null  object \n",
      " 1   Freq_pm               51046 non-null  float64\n",
      " 2   Dom_PoS_SUBTLEX       50963 non-null  object \n",
      " 3   Nletters              51715 non-null  int64  \n",
      " 4   Nphon                 51715 non-null  int64  \n",
      " 5   Nsyll                 51715 non-null  int64  \n",
      " 6   Lemma_highest_PoS     51714 non-null  object \n",
      " 7   AoA_Kup               31105 non-null  float64\n",
      " 8   Perc_known            31124 non-null  float64\n",
      " 9   AoA_Kup_lem           51695 non-null  float64\n",
      " 10  Perc_known_lem        51715 non-null  float64\n",
      " 11  AoA_Bird_lem          5421 non-null   float64\n",
      " 12  AoA_Bristol_lem       8061 non-null   float64\n",
      " 13  AoA_Cort_lem          8045 non-null   float64\n",
      " 14  AoA_Schock            7056 non-null   float64\n",
      "dtypes: float64(9), int64(3), object(3)\n",
      "memory usage: 6.3+ MB\n"
     ]
    }
   ],
   "source": [
    "aoa_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "built-damage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label\n",
       "0  There is manuscript evidence that Austen conti...      1\n",
       "1  In a remarkable comparative analysis , Mandaea...      1\n",
       "2  Before Persephone was released to Hermes , who...      1\n",
       "3  Cogeneration plants are commonly found in dist...      1\n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = pd.read_csv('WikiLarge_Train.csv', header = 0)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "convertible-vancouver",
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_processing(x):\n",
    "    x = re.sub(r'[^\\w\\s]','', x)\n",
    "    x = x.split(' ')\n",
    "    x = [word.strip().lower() for word in x]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "objective-chinese",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.loc[:, 'processed_text'] = train_data['original_text'].apply(text_processing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cognitive-percentage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "      <td>[there, is, manuscript, evidence, that, austen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "      <td>[in, a, remarkable, comparative, analysis, , m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "      <td>[before, persephone, was, released, to, hermes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "      <td>[cogeneration, plants, are, commonly, found, i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "      <td>[geneva, lrb, , , , , , , , rrb, is, the, seco...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label  \\\n",
       "0  There is manuscript evidence that Austen conti...      1   \n",
       "1  In a remarkable comparative analysis , Mandaea...      1   \n",
       "2  Before Persephone was released to Hermes , who...      1   \n",
       "3  Cogeneration plants are commonly found in dist...      1   \n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1   \n",
       "\n",
       "                                      processed_text  \n",
       "0  [there, is, manuscript, evidence, that, austen...  \n",
       "1  [in, a, remarkable, comparative, analysis, , m...  \n",
       "2  [before, persephone, was, released, to, hermes...  \n",
       "3  [cogeneration, plants, are, commonly, found, i...  \n",
       "4  [geneva, lrb, , , , , , , , rrb, is, the, seco...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "chicken-baking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(aoa_df.loc[:, 'AoA_Kup_lem'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "bridal-charger",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_average_aoa(x):\n",
    "    aoa_value_list = []\n",
    "    \n",
    "    for word in x:\n",
    "        if word:\n",
    "            try:\n",
    "                aoa_value = aoa_df.loc[word, 'AoA_Kup_lem']\n",
    "                aoa_value_list.append(aoa_value)\n",
    "            except:\n",
    "                pass\n",
    "    \n",
    "    if aoa_value_list:\n",
    "        mean_aoa = np.mean(aoa_value_list)\n",
    "    else:\n",
    "        mean_aoa = 0\n",
    "        \n",
    "    return mean_aoa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "advance-arnold",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.loc[:, 'average_AoA_Kup_lem'] = train_data['processed_text'].apply(calculate_average_aoa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "impaired-belfast",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>average_AoA_Kup_lem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "      <td>[there, is, manuscript, evidence, that, austen...</td>\n",
       "      <td>10.441842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "      <td>[in, a, remarkable, comparative, analysis, , m...</td>\n",
       "      <td>13.332857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "      <td>[before, persephone, was, released, to, hermes...</td>\n",
       "      <td>7.664762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "      <td>[cogeneration, plants, are, commonly, found, i...</td>\n",
       "      <td>9.592187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "      <td>[geneva, lrb, , , , , , , , rrb, is, the, seco...</td>\n",
       "      <td>14.498214</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label  \\\n",
       "0  There is manuscript evidence that Austen conti...      1   \n",
       "1  In a remarkable comparative analysis , Mandaea...      1   \n",
       "2  Before Persephone was released to Hermes , who...      1   \n",
       "3  Cogeneration plants are commonly found in dist...      1   \n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1   \n",
       "\n",
       "                                      processed_text  average_AoA_Kup_lem  \n",
       "0  [there, is, manuscript, evidence, that, austen...            10.441842  \n",
       "1  [in, a, remarkable, comparative, analysis, , m...            13.332857  \n",
       "2  [before, persephone, was, released, to, hermes...             7.664762  \n",
       "3  [cogeneration, plants, are, commonly, found, i...             9.592187  \n",
       "4  [geneva, lrb, , , , , , , , rrb, is, the, seco...            14.498214  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cordless-detective",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 416768 entries, 0 to 416767\n",
      "Data columns (total 4 columns):\n",
      " #   Column               Non-Null Count   Dtype  \n",
      "---  ------               --------------   -----  \n",
      " 0   original_text        416768 non-null  object \n",
      " 1   label                416768 non-null  int64  \n",
      " 2   processed_text       416768 non-null  object \n",
      " 3   average_AoA_Kup_lem  416471 non-null  float64\n",
      "dtypes: float64(1), int64(1), object(2)\n",
      "memory usage: 12.7+ MB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "downtown-relay",
   "metadata": {},
   "source": [
    "## Incorprate percentage of basic words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "atlantic-lunch",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dale_chall.txt') as file_obj:\n",
    "    basic_word_list = file_obj.readlines()\n",
    "    cleaned_basic_word_list = [word.lower().strip() for word in basic_word_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "graphic-tuesday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2950"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cleaned_basic_word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "constitutional-captain",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_percentage_of_basic_words(x):\n",
    "    basic_count = 0\n",
    "    word_count = 0\n",
    "    for word in x:\n",
    "        if word: \n",
    "            word_count += 1\n",
    "            if word in cleaned_basic_word_list:\n",
    "                basic_count += 1\n",
    "    try:\n",
    "        percentage = basic_count / word_count\n",
    "    except:\n",
    "        percentage = 1\n",
    "        \n",
    "    return percentage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "different-union",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.loc[:, 'percentage_of_basic_words'] = train_data['processed_text'].apply(calculate_percentage_of_basic_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "conservative-border",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>average_AoA_Kup_lem</th>\n",
       "      <th>percentage_of_basic_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "      <td>[there, is, manuscript, evidence, that, austen...</td>\n",
       "      <td>10.441842</td>\n",
       "      <td>0.552632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "      <td>[in, a, remarkable, comparative, analysis, , m...</td>\n",
       "      <td>13.332857</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "      <td>[before, persephone, was, released, to, hermes...</td>\n",
       "      <td>7.664762</td>\n",
       "      <td>0.619048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "      <td>[cogeneration, plants, are, commonly, found, i...</td>\n",
       "      <td>9.592187</td>\n",
       "      <td>0.312500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "      <td>[geneva, lrb, , , , , , , , rrb, is, the, seco...</td>\n",
       "      <td>14.498214</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label  \\\n",
       "0  There is manuscript evidence that Austen conti...      1   \n",
       "1  In a remarkable comparative analysis , Mandaea...      1   \n",
       "2  Before Persephone was released to Hermes , who...      1   \n",
       "3  Cogeneration plants are commonly found in dist...      1   \n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1   \n",
       "\n",
       "                                      processed_text  average_AoA_Kup_lem  \\\n",
       "0  [there, is, manuscript, evidence, that, austen...            10.441842   \n",
       "1  [in, a, remarkable, comparative, analysis, , m...            13.332857   \n",
       "2  [before, persephone, was, released, to, hermes...             7.664762   \n",
       "3  [cogeneration, plants, are, commonly, found, i...             9.592187   \n",
       "4  [geneva, lrb, , , , , , , , rrb, is, the, seco...            14.498214   \n",
       "\n",
       "   percentage_of_basic_words  \n",
       "0                   0.552632  \n",
       "1                   0.285714  \n",
       "2                   0.619048  \n",
       "3                   0.312500  \n",
       "4                   0.500000  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "governmental-lafayette",
   "metadata": {},
   "source": [
    "## Incorprate Concreteness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "constitutional-estimate",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bigram</th>\n",
       "      <th>Conc.M</th>\n",
       "      <th>Conc.SD</th>\n",
       "      <th>Unknown</th>\n",
       "      <th>Total</th>\n",
       "      <th>Percent_known</th>\n",
       "      <th>SUBTLEX</th>\n",
       "      <th>Dom_Pos</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>roadsweeper</th>\n",
       "      <td>0</td>\n",
       "      <td>4.85</td>\n",
       "      <td>0.37</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>traindriver</th>\n",
       "      <td>0</td>\n",
       "      <td>4.54</td>\n",
       "      <td>0.71</td>\n",
       "      <td>3</td>\n",
       "      <td>29</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tush</th>\n",
       "      <td>0</td>\n",
       "      <td>4.45</td>\n",
       "      <td>1.01</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "      <td>0.88</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hairdress</th>\n",
       "      <td>0</td>\n",
       "      <td>3.93</td>\n",
       "      <td>1.28</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pharmaceutics</th>\n",
       "      <td>0</td>\n",
       "      <td>3.77</td>\n",
       "      <td>1.41</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unenvied</th>\n",
       "      <td>0</td>\n",
       "      <td>1.21</td>\n",
       "      <td>0.62</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agnostically</th>\n",
       "      <td>0</td>\n",
       "      <td>1.20</td>\n",
       "      <td>0.50</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conceptualistic</th>\n",
       "      <td>0</td>\n",
       "      <td>1.18</td>\n",
       "      <td>0.50</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conventionalism</th>\n",
       "      <td>0</td>\n",
       "      <td>1.18</td>\n",
       "      <td>0.48</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>essentialness</th>\n",
       "      <td>0</td>\n",
       "      <td>1.04</td>\n",
       "      <td>0.20</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39954 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Bigram  Conc.M  Conc.SD  Unknown  Total  Percent_known  \\\n",
       "Word                                                                      \n",
       "roadsweeper           0    4.85     0.37        1     27           0.96   \n",
       "traindriver           0    4.54     0.71        3     29           0.90   \n",
       "tush                  0    4.45     1.01        3     25           0.88   \n",
       "hairdress             0    3.93     1.28        0     29           1.00   \n",
       "pharmaceutics         0    3.77     1.41        4     26           0.85   \n",
       "...                 ...     ...      ...      ...    ...            ...   \n",
       "unenvied              0    1.21     0.62        1     30           0.97   \n",
       "agnostically          0    1.20     0.50        2     27           0.93   \n",
       "conceptualistic       0    1.18     0.50        4     26           0.85   \n",
       "conventionalism       0    1.18     0.48        1     29           0.97   \n",
       "essentialness         0    1.04     0.20        2     26           0.92   \n",
       "\n",
       "                 SUBTLEX Dom_Pos  \n",
       "Word                              \n",
       "roadsweeper            0       0  \n",
       "traindriver            0       0  \n",
       "tush                  66       0  \n",
       "hairdress              1       0  \n",
       "pharmaceutics          0       0  \n",
       "...                  ...     ...  \n",
       "unenvied               0     NaN  \n",
       "agnostically           0     NaN  \n",
       "conceptualistic        0     NaN  \n",
       "conventionalism        0     NaN  \n",
       "essentialness          0     NaN  \n",
       "\n",
       "[39954 rows x 8 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concreteness_df = pd.read_csv('Concreteness_ratings_Brysbaert_et_al_BRM.txt', sep = '\\t', header = 0, index_col = 0)\n",
    "concreteness_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "secondary-hartford",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 39954 entries, roadsweeper to essentialness\n",
      "Data columns (total 8 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Bigram         39954 non-null  int64  \n",
      " 1   Conc.M         39954 non-null  float64\n",
      " 2   Conc.SD        39954 non-null  float64\n",
      " 3   Unknown        39954 non-null  int64  \n",
      " 4   Total          39954 non-null  int64  \n",
      " 5   Percent_known  39954 non-null  float64\n",
      " 6   SUBTLEX        39954 non-null  int64  \n",
      " 7   Dom_Pos        28707 non-null  object \n",
      "dtypes: float64(3), int64(4), object(1)\n",
      "memory usage: 2.7+ MB\n"
     ]
    }
   ],
   "source": [
    "concreteness_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "auburn-preference",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_average_concrete_mean(x):\n",
    "    aoa_value_list = []\n",
    "    min_conc_value = np.min(concreteness_df.loc[:, 'Conc.M'])\n",
    "    for word in x:\n",
    "        if word:\n",
    "            try:\n",
    "                aoa_value = concreteness_df.loc[word, 'Conc.M']\n",
    "                aoa_value_list.append(aoa_value)\n",
    "            except:\n",
    "                pass\n",
    "    \n",
    "    if aoa_value_list:\n",
    "        mean_aoa = np.mean(aoa_value_list)\n",
    "    else:\n",
    "        mean_aoa = np.nan\n",
    "        \n",
    "    return mean_aoa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "wicked-baghdad",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.loc[:, 'average_concreteness'] = train_data['processed_text'].apply(calculate_average_concrete_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "painful-locator",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>average_AoA_Kup_lem</th>\n",
       "      <th>percentage_of_basic_words</th>\n",
       "      <th>average_concreteness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "      <td>[there, is, manuscript, evidence, that, austen...</td>\n",
       "      <td>10.441842</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>1.974211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "      <td>[in, a, remarkable, comparative, analysis, , m...</td>\n",
       "      <td>13.332857</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>1.655714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "      <td>[before, persephone, was, released, to, hermes...</td>\n",
       "      <td>7.664762</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>2.106667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "      <td>[cogeneration, plants, are, commonly, found, i...</td>\n",
       "      <td>9.592187</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>2.324375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "      <td>[geneva, lrb, , , , , , , , rrb, is, the, seco...</td>\n",
       "      <td>14.498214</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.749643</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label  \\\n",
       "0  There is manuscript evidence that Austen conti...      1   \n",
       "1  In a remarkable comparative analysis , Mandaea...      1   \n",
       "2  Before Persephone was released to Hermes , who...      1   \n",
       "3  Cogeneration plants are commonly found in dist...      1   \n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1   \n",
       "\n",
       "                                      processed_text  average_AoA_Kup_lem  \\\n",
       "0  [there, is, manuscript, evidence, that, austen...            10.441842   \n",
       "1  [in, a, remarkable, comparative, analysis, , m...            13.332857   \n",
       "2  [before, persephone, was, released, to, hermes...             7.664762   \n",
       "3  [cogeneration, plants, are, commonly, found, i...             9.592187   \n",
       "4  [geneva, lrb, , , , , , , , rrb, is, the, seco...            14.498214   \n",
       "\n",
       "   percentage_of_basic_words  average_concreteness  \n",
       "0                   0.552632              1.974211  \n",
       "1                   0.285714              1.655714  \n",
       "2                   0.619048              2.106667  \n",
       "3                   0.312500              2.324375  \n",
       "4                   0.500000              1.749643  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "recreational-boards",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(concreteness_df.loc[:, 'Percent_known'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "emotional-welding",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_percentage_known_mean(x):\n",
    "    aoa_value_list = []\n",
    "    min_known_value = np.min(concreteness_df.loc[:, 'Percent_known'])\n",
    "    for word in x:\n",
    "        if word:\n",
    "            try:\n",
    "                aoa_value = concreteness_df.loc[word, 'Percent_known']\n",
    "                aoa_value_list.append(aoa_value)\n",
    "            except:\n",
    "                pass\n",
    "    \n",
    "    if aoa_value_list:\n",
    "        mean_aoa = np.mean(aoa_value_list)\n",
    "    else:\n",
    "        mean_aoa = 1\n",
    "        \n",
    "    return mean_aoa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "announced-stupid",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.loc[:, 'average_percentage_known'] = train_data['processed_text'].apply(calculate_percentage_known_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "limited-complex",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>average_AoA_Kup_lem</th>\n",
       "      <th>percentage_of_basic_words</th>\n",
       "      <th>average_concreteness</th>\n",
       "      <th>average_percentage_known</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is manuscript evidence that Austen conti...</td>\n",
       "      <td>1</td>\n",
       "      <td>[there, is, manuscript, evidence, that, austen...</td>\n",
       "      <td>10.441842</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>1.974211</td>\n",
       "      <td>0.995926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In a remarkable comparative analysis , Mandaea...</td>\n",
       "      <td>1</td>\n",
       "      <td>[in, a, remarkable, comparative, analysis, , m...</td>\n",
       "      <td>13.332857</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>1.655714</td>\n",
       "      <td>0.994167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Before Persephone was released to Hermes , who...</td>\n",
       "      <td>1</td>\n",
       "      <td>[before, persephone, was, released, to, hermes...</td>\n",
       "      <td>7.664762</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>2.106667</td>\n",
       "      <td>0.992647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cogeneration plants are commonly found in dist...</td>\n",
       "      <td>1</td>\n",
       "      <td>[cogeneration, plants, are, commonly, found, i...</td>\n",
       "      <td>9.592187</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>2.324375</td>\n",
       "      <td>0.993333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Geneva -LRB- , ; , ; , ; ; -RRB- is the second...</td>\n",
       "      <td>1</td>\n",
       "      <td>[geneva, lrb, , , , , , , , rrb, is, the, seco...</td>\n",
       "      <td>14.498214</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.749643</td>\n",
       "      <td>0.989333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_text  label  \\\n",
       "0  There is manuscript evidence that Austen conti...      1   \n",
       "1  In a remarkable comparative analysis , Mandaea...      1   \n",
       "2  Before Persephone was released to Hermes , who...      1   \n",
       "3  Cogeneration plants are commonly found in dist...      1   \n",
       "4  Geneva -LRB- , ; , ; , ; ; -RRB- is the second...      1   \n",
       "\n",
       "                                      processed_text  average_AoA_Kup_lem  \\\n",
       "0  [there, is, manuscript, evidence, that, austen...            10.441842   \n",
       "1  [in, a, remarkable, comparative, analysis, , m...            13.332857   \n",
       "2  [before, persephone, was, released, to, hermes...             7.664762   \n",
       "3  [cogeneration, plants, are, commonly, found, i...             9.592187   \n",
       "4  [geneva, lrb, , , , , , , , rrb, is, the, seco...            14.498214   \n",
       "\n",
       "   percentage_of_basic_words  average_concreteness  average_percentage_known  \n",
       "0                   0.552632              1.974211                  0.995926  \n",
       "1                   0.285714              1.655714                  0.994167  \n",
       "2                   0.619048              2.106667                  0.992647  \n",
       "3                   0.312500              2.324375                  0.993333  \n",
       "4                   0.500000              1.749643                  0.989333  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "funded-queens",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 416768 entries, 0 to 416767\n",
      "Data columns (total 7 columns):\n",
      " #   Column                     Non-Null Count   Dtype  \n",
      "---  ------                     --------------   -----  \n",
      " 0   original_text              416768 non-null  object \n",
      " 1   label                      416768 non-null  int64  \n",
      " 2   processed_text             416768 non-null  object \n",
      " 3   average_AoA_Kup_lem        416471 non-null  float64\n",
      " 4   percentage_of_basic_words  416479 non-null  float64\n",
      " 5   average_concreteness       416479 non-null  float64\n",
      " 6   average_percentage_known   404958 non-null  float64\n",
      "dtypes: float64(4), int64(1), object(2)\n",
      "memory usage: 22.3+ MB\n"
     ]
    }
   ],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "sitting-episode",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv('train_data_with_feature_ver_2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "recent-spoke",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_features(train_data):\n",
    "    \n",
    "    train_data.loc[:, 'processed_text'] = train_data['original_text'].apply(text_processing)\n",
    "    train_data.loc[:, 'text_length'] = train_data['processed_text'].apply(lambda x: len(x))\n",
    "    train_data['text_length'] = pd.cut(train_data['text_length'], bins = [0, 10, 20, 30, 40, 100], labels = [0, 1, 2, 3, 4])\n",
    "    train_data['text_length'] = train_data['text_length'].astype('int64')\n",
    "    train_data.loc[:, 'average_AoA_Kup_lem'] = train_data['processed_text'].apply(calculate_average_aoa)\n",
    "    train_data.loc[:, 'percentage_of_basic_words'] = train_data['processed_text'].apply(calculate_percentage_of_basic_words)\n",
    "    train_data.loc[:, 'average_concreteness'] = train_data['processed_text'].apply(calculate_average_concrete_mean)\n",
    "    train_data.loc[:, 'average_percentage_known'] = train_data['processed_text'].apply(calculate_percentage_known_mean)\n",
    "\n",
    "    return train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "designing-wrestling",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('WikiLarge_Test.csv', header = 0, index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "varied-virgin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2011</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2011</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1997</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.636</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119087</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119088</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119089</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119090</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119091</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>119092 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       original_text  label\n",
       "id                         \n",
       "0              -2011    NaN\n",
       "1              -2011    NaN\n",
       "2              -2000    NaN\n",
       "3              -1997    NaN\n",
       "4              1.636    NaN\n",
       "...              ...    ...\n",
       "119087        #NAME?    NaN\n",
       "119088        #NAME?    NaN\n",
       "119089        #NAME?    NaN\n",
       "119090        #NAME?    NaN\n",
       "119091        #NAME?    NaN\n",
       "\n",
       "[119092 rows x 2 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "former-husband",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_text</th>\n",
       "      <th>label</th>\n",
       "      <th>processed_text</th>\n",
       "      <th>text_length</th>\n",
       "      <th>average_AoA_Kup_lem</th>\n",
       "      <th>percentage_of_basic_words</th>\n",
       "      <th>average_concreteness</th>\n",
       "      <th>average_percentage_known</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[2011]</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[2011]</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[2000]</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1997</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[1997]</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[1636]</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119087</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[name]</td>\n",
       "      <td>0</td>\n",
       "      <td>3.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119088</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[name]</td>\n",
       "      <td>0</td>\n",
       "      <td>3.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119089</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[name]</td>\n",
       "      <td>0</td>\n",
       "      <td>3.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119090</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[name]</td>\n",
       "      <td>0</td>\n",
       "      <td>3.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119091</th>\n",
       "      <td>#NAME?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[name]</td>\n",
       "      <td>0</td>\n",
       "      <td>3.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>119092 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       original_text  label processed_text  text_length  average_AoA_Kup_lem  \\\n",
       "id                                                                             \n",
       "0              -2011    NaN         [2011]            0                 0.00   \n",
       "1              -2011    NaN         [2011]            0                 0.00   \n",
       "2              -2000    NaN         [2000]            0                 0.00   \n",
       "3              -1997    NaN         [1997]            0                 0.00   \n",
       "4              1.636    NaN         [1636]            0                 0.00   \n",
       "...              ...    ...            ...          ...                  ...   \n",
       "119087        #NAME?    NaN         [name]            0                 3.68   \n",
       "119088        #NAME?    NaN         [name]            0                 3.68   \n",
       "119089        #NAME?    NaN         [name]            0                 3.68   \n",
       "119090        #NAME?    NaN         [name]            0                 3.68   \n",
       "119091        #NAME?    NaN         [name]            0                 3.68   \n",
       "\n",
       "        percentage_of_basic_words  average_concreteness  \\\n",
       "id                                                        \n",
       "0                             0.0                   NaN   \n",
       "1                             0.0                   NaN   \n",
       "2                             0.0                   NaN   \n",
       "3                             0.0                   NaN   \n",
       "4                             0.0                   NaN   \n",
       "...                           ...                   ...   \n",
       "119087                        1.0                   3.5   \n",
       "119088                        1.0                   3.5   \n",
       "119089                        1.0                   3.5   \n",
       "119090                        1.0                   3.5   \n",
       "119091                        1.0                   3.5   \n",
       "\n",
       "        average_percentage_known  \n",
       "id                                \n",
       "0                           1.00  \n",
       "1                           1.00  \n",
       "2                           1.00  \n",
       "3                           1.00  \n",
       "4                           1.00  \n",
       "...                          ...  \n",
       "119087                      0.96  \n",
       "119088                      0.96  \n",
       "119089                      0.96  \n",
       "119090                      0.96  \n",
       "119091                      0.96  \n",
       "\n",
       "[119092 rows x 8 columns]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = generate_features(train_data)\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "integrated-grocery",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.to_csv(\"test_data_with_feature_ver_3.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fallen-engagement",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}