NLP 2 - TFIDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classificação de Texto utilizando TFIDF\n",
    "Autor: Lucas Kenji Sudo\n",
    "\n",
    "E-mail: lucaskenjis@gmail.com\n",
    "\n",
    "Linkedin : linkedin.com/in/lucas-sudo-701647121/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import re\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "# Imports para NLP\n",
    "#from nltk import word_tokenize\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk import regexp_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "start_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('IMDB Dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50000</td>\n",
       "      <td>50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>49582</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Loved today's show!!! It was a variety and not...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>5</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   review sentiment\n",
       "count                                               50000     50000\n",
       "unique                                              49582         2\n",
       "top     Loved today's show!!! It was a variety and not...  negative\n",
       "freq                                                    5     25000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remover review duplicadas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.review.duplicated()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remover colunas null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df[~df.review.isnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Limpeza"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Método para limpeza de texto aplicando:\n",
    "    - replace para um caractere específico\n",
    "    - isdigit para verificar se a palavra é um digito e assim, remove-la.\n",
    "    - limpeza de tag html usando BeautifulSoup\n",
    "    - regexp_tokenize para transformar textos em tokens, e coletar apenas palavras através do regex [\\w]+. Ignorando acentos.\n",
    "    - .lower() para deixar as palavras minusculas\n",
    "    - stopwords para remover palavras que não agregam informações\n",
    "    - stemming para extrair o radical das palavras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.download('stopwords')\n",
    "ps = PorterStemmer()\n",
    "eng_stpw = set(stopwords.words('english'))\n",
    "\n",
    "def padronizardados(text):\n",
    "    # remover caractere especifico\n",
    "    text = text.replace('\\ ','')\n",
    "    # remover digitos\n",
    "    text = ''.join([i for i in text if not i.isdigit()])\n",
    "    # remover tags html\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    #tokenization e lower case\n",
    "    text = regexp_tokenize(soup.get_text().lower(),\"[\\w]+\")\n",
    "    #remover stopwords\n",
    "    temp = []\n",
    "    for t in text:\n",
    "        if t not in eng_stpw:\n",
    "            #aplicar o stemming\n",
    "            temp.append(ps.stem(t))\n",
    "    new_text = (' '.join(temp))\n",
    "    \n",
    "    return new_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Texto antes do método"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Taut and organically gripping, Edward Dmytryk\\'s Crossfire is a distinctive suspense thriller, an unlikely \"message\" movie using the look and devices of the noir cycle.<br /><br />Bivouacked in Washington, DC, a company of soldiers cope with their restlessness by hanging out in bars. Three of them end up at a stranger\\'s apartment where Robert Ryan, drunk and belligerent, beats their host (Sam Levene) to death because he happens to be Jewish. Police detective Robert Young investigates with the help of Robert Mitchum, who\\'s assigned to Ryan\\'s outfit. Suspicion falls on the second of the three (George Cooper), who has vanished. Ryan slays the third buddy (Steve Brodie) to insure his silence before Young closes in.<br /><br />Abetted by a superior script by John Paxton, Dmytryk draws precise performances from his three starring Bobs. Ryan, naturally, does his prototypical Angry White Male (and to the hilt), while Mitchum underplays with his characteristic alert nonchalance (his role, however, is not central); Young may never have been better. Gloria Grahame gives her first fully-fledged rendition of the smart-mouthed, vulnerable tramp, and, as a sad sack who\\'s leeched into her life, Paul Kelly haunts us in a small, peripheral role that he makes memorable.<br /><br />The politically engaged Dmytryk perhaps inevitably succumbs to sermonizing, but it\\'s pretty much confined to Young\\'s reminiscence of how his Irish grandfather died at the hands of bigots a century earlier (thus, incidentally, stretching chronology to the limit). At least there\\'s no attempt to render an explanation, however glib, of why Ryan hates Jews (and hillbillies and...).<br /><br />Curiously, Crossfire survives even the major change wrought upon it -- the novel it\\'s based on (Richard Brooks\\' The Brick Foxhole) dealt with a gay-bashing murder. But homosexuality in 1947 was still Beyond The Pale. News of the Holocaust had, however, begun to emerge from the ashes of Europe, so Hollywood felt emboldened to register its protest against anti-Semitism (the studios always quaked at the prospect of offending any potential ticket buyer).<br /><br />But while the change from homophobia to anti-Semitism works in general, the specifics don\\'t fit so smoothly. The victim\\'s chatting up a lonesome, drunk young soldier then inviting him back home looks odd, even though (or especially since) there\\'s a girlfriend in tow. It raises the question whether this scenario was retained inadvertently or left in as a discreet tip-off to the original engine generating Ryan\\'s murderous rage.'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.review[30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Aplicando método de limpeza aos textos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.review = df.review.apply(padronizardados)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Texto depois do método"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'taut organ grip edward dmytryk crossfir distinct suspens thriller unlik messag movi use look devic noir cycl bivouack washington dc compani soldier cope restless hang bar three end stranger apart robert ryan drunk belliger beat host sam leven death happen jewish polic detect robert young investig help robert mitchum assign ryan outfit suspicion fall second three georg cooper vanish ryan slay third buddi steve brodi insur silenc young close abet superior script john paxton dmytryk draw precis perform three star bob ryan natur prototyp angri white male hilt mitchum underplay characterist alert nonchal role howev central young may never better gloria graham give first fulli fledg rendit smart mouth vulner tramp sad sack leech life paul kelli haunt us small peripher role make memor polit engag dmytryk perhap inevit succumb sermon pretti much confin young reminisc irish grandfath die hand bigot centuri earlier thu incident stretch chronolog limit least attempt render explan howev glib ryan hate jew hillbilli curious crossfir surviv even major chang wrought upon novel base richard brook brick foxhol dealt gay bash murder homosexu still beyond pale news holocaust howev begun emerg ash europ hollywood felt embolden regist protest anti semit studio alway quak prospect offend potenti ticket buyer chang homophobia anti semit work gener specif fit smoothli victim chat lonesom drunk young soldier invit back home look odd even though especi sinc girlfriend tow rais question whether scenario retain inadvert left discreet tip origin engin gener ryan murder rage'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.review[30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transformar sentiments em 0 e 1 (Label encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "tag = le.fit_transform(df.sentiment)\n",
    "tag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Com TFIDF\n",
    "   Ao construir modelos para análise de texto não podemos trabalhar diretamente com os dados na forma textual, pois o computador trabalha com dados na forma numérica. Para analisar dados de texto é preciso separar o texto em palavras ou em conjunto de palavras, através do processo de tokenization, e transformar essas palavras em representação numérica, através do processo de extração de características (features).\n",
    "    Nesse jupyter notebook irei utilizar a técnica TF-IDF.\n",
    "    https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html#sklearn.feature_extraction.text.TfidfVectorizer\n",
    "    O conceito desse método é dar destaque para as palavras que não aparecem com tanta frequência nos documentos. Essa abordagem supre o problema do CountVectorizer; palavras com alta frequência como \"a\" e \"the\" terão pontuações muito baixas (pois serão exibidas em muitos documentos), enquanto palavras com pouca frequência terão pontuações mais altas e, portanto, elas serão as que o modelo identificará como importantes e tentará aprender.\n",
    "    \n",
    "   Para o cálculo, o método utiliza da sequinte equação: \n",
    "   * TF= (frequencia da palavra no documento/quantidade de palavras no documento) \n",
    "   * IDF= log(quantidade de documentos/ quantidade de documentos onde a palavra está presente)\n",
    "   * TFIDF= TF * IDF\n",
    "   \n",
    "   ![](tfidf-calc.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'esse': 1, 'filme': 2, 'bom': 0, 'ótimo': 3}\n",
      "['bom', 'esse', 'filme', 'ótimo']\n"
     ]
    }
   ],
   "source": [
    "tfidf = TfidfVectorizer()\n",
    "corpus = ['Esse filme é bom bom','Esse filme é ótimo']\n",
    "\n",
    "x = tfidf.fit(corpus)\n",
    "#Vocabulario com os id inteiros fixos\n",
    "print(x.vocabulary_) \n",
    "# Features\n",
    "print(tfidf.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (2, 4) \n",
      "\n",
      "array,vocab   freq\n",
      "  (0, 2)\t0.31779953783628945\n",
      "  (0, 1)\t0.31779953783628945\n",
      "  (0, 0)\t0.8933123236036103\n",
      "  (1, 3)\t0.7049094889309326\n",
      "  (1, 2)\t0.5015489070943787\n",
      "  (1, 1)\t0.5015489070943787\n",
      "[[0.89331232 0.31779954 0.31779954 0.        ]\n",
      " [0.         0.50154891 0.50154891 0.70490949]]\n"
     ]
    }
   ],
   "source": [
    "y = tfidf.transform(corpus)\n",
    "print('shape:',y.shape,'\\n')\n",
    "print('array,vocab   freq')\n",
    "print(y)\n",
    "print(y.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Split dados\n",
    "    - Optei por separar os dados em treino, validação e teste"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "  \n",
    "X_train, X_test, y_train, y_test  = train_test_split(df.review, df.sentiment, test_size=0.2, random_state=1)\n",
    "\n",
    "X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, test_size=0.5, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TFIDF utilizando ngram no range(1,2) para procurar palavras que se complementam\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(ngram_range=(1, 2),max_features=10000)\n",
    "\n",
    "X_train = tfidf.fit_transform(X_train)\n",
    "X_val = tfidf.transform(X_val)\n",
    "X_test = tfidf.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tfidf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-326a55804881>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvocabulary_\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'like'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'tfidf' is not defined"
     ]
    }
   ],
   "source": [
    "tfidf.vocabulary_['like']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 7597)\t0.08320292352423787\n",
      "  (0, 4826)\t0.08775601865145748\n",
      "  (0, 8028)\t0.0855725431794635\n",
      "  (0, 2651)\t0.09209014594305069\n",
      "  (0, 1269)\t0.05476272873806238\n",
      "  (0, 6695)\t0.06669894520110206\n",
      "  (0, 2917)\t0.03935453908241891\n",
      "  (0, 752)\t0.06839229283665357\n",
      "  (0, 7741)\t0.05132108595108667\n",
      "  (0, 4899)\t0.08251664320962881\n",
      "  (0, 4564)\t0.08512063791148733\n",
      "  (0, 1580)\t0.0850503344358566\n",
      "  (0, 1915)\t0.09974738058419905\n",
      "  (0, 8212)\t0.09364266260053367\n",
      "  (0, 861)\t0.14431694763743863\n",
      "  (0, 8850)\t0.06132429682166089\n",
      "  (0, 9493)\t0.10436328376994532\n",
      "  (0, 3237)\t0.10139561493343953\n",
      "  (0, 6337)\t0.1671120115956271\n",
      "  (0, 9177)\t0.07983519863409935\n",
      "  (0, 348)\t0.07627950924437121\n",
      "  (0, 9117)\t0.0687337482030325\n",
      "  (0, 9798)\t0.08069498610009267\n",
      "  (0, 5316)\t0.08603908899599456\n",
      "  (0, 6763)\t0.12732315941836397\n",
      "  :\t:\n",
      "  (0, 6476)\t0.12859065587119428\n",
      "  (0, 7174)\t0.12825105465824163\n",
      "  (0, 8667)\t0.14882267516665254\n",
      "  (0, 3448)\t0.174975798091872\n",
      "  (0, 7016)\t0.13253321048086267\n",
      "  (0, 5548)\t0.1524152933314979\n",
      "  (0, 7993)\t0.13010381951633013\n",
      "  (0, 3805)\t0.10014401668986092\n",
      "  (0, 7053)\t0.1685008425996963\n",
      "  (0, 2780)\t0.09528455475176421\n",
      "  (0, 1921)\t0.18170129287306408\n",
      "  (0, 2313)\t0.1519906494457259\n",
      "  (0, 2478)\t0.1735837697354196\n",
      "  (0, 9173)\t0.18250794319889393\n",
      "  (0, 4016)\t0.10306608300668306\n",
      "  (0, 9027)\t0.16131694299614016\n",
      "  (0, 9602)\t0.12230682272593806\n",
      "  (0, 8800)\t0.10920945491622781\n",
      "  (0, 4036)\t0.09888467422847831\n",
      "  (0, 1560)\t0.13561399312474914\n",
      "  (0, 5956)\t0.177777580827081\n",
      "  (0, 4508)\t0.1532860180744708\n",
      "  (0, 864)\t0.1698065350661602\n",
      "  (0, 7330)\t0.15668733866259127\n",
      "  (0, 6705)\t0.14912262257953615\n"
     ]
    }
   ],
   "source": [
    "print(X_train[30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature selection + L1 e Regressao logistica\n",
    "A seleção de atributos ( Feature Selection) é uma técnica responsável por selecionar os atributos mais relevantes/importantes para a criação do modelo. Sua utilização trás vantagens como: \n",
    "   - Minimizar o problema de overfitting sem perder muita informação.\n",
    "   - Diminuir o tempo de treinamento.\n",
    "   \n",
    "Para seleção de features foi utilizado a regularização L1, também conhecida como Lasso regression. O L1 penaliza os coeficientes com um alto grau de correlação entra si jogando seu valor a ZERO, ou seja, removendo a feature e diminuindo a complexidade do modelo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17.3 minutos \n",
      "\n",
      "17.34 minutos \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lucaskenjis\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\sag.py:334: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17.76 minutos \n",
      "\n",
      "19.68 minutos \n",
      "\n",
      "22.54 minutos \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cs=[.01,.1,1,10,100]\n",
    "#\n",
    "summary=[]\n",
    "\n",
    "for c in cs:\n",
    "    \n",
    "    #seleção de atributos\n",
    "    logreg = LogisticRegression(solver='saga', penalty='l1',C=c, max_iter=100).fit(X_train, y_train)\n",
    "    select_features = SelectFromModel(logreg, prefit=True)\n",
    "    \n",
    "    X_train_sel=select_features.transform(X_train)\n",
    "    X_test_sel=select_features.transform(X_test)\n",
    "    X_val_sel=select_features.transform(X_val)\n",
    "\n",
    "    #fittando o modelo\n",
    "    model_tfidf = LogisticRegression(solver='saga', max_iter=100).fit(X_train_sel, y_train)\n",
    "    \n",
    "    #avaliando acurácia\n",
    "    model_tfidf_score = model_tfidf.score(X_val_sel, y_val)\n",
    "    \n",
    "    #resumo da validação\n",
    "    summary.append((c,np.shape(X_train_sel)[1],model_tfidf_score))\n",
    "    \n",
    "    print(round((time.time() - start_time)/60,2),\"minutos \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.01 Features=5 Acc=0.6634\n",
      "C=0.10 Features=154 Acc=0.8484\n",
      "C=1.00 Features=1659 Acc=0.8905\n",
      "C=10.00 Features=7331 Acc=0.8974\n",
      "C=100.00 Features=9881 Acc=0.9002\n"
     ]
    }
   ],
   "source": [
    "for i in summary:\n",
    "  print(\"C=%.2f Features=%d Acc=%3.4f\" %i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Criação do modelo com o melhor peso C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26.19 minutos \n",
      "\n"
     ]
    }
   ],
   "source": [
    "logreg = LogisticRegression(solver='saga', penalty='l1',C=100, max_iter=100).fit(X_train, y_train)\n",
    "select_features = SelectFromModel(logreg, prefit=True)\n",
    "    \n",
    "X_train_sel=select_features.transform(X_train)\n",
    "X_test_sel=select_features.transform(X_test)\n",
    "X_val_sel=select_features.transform(X_val)\n",
    "\n",
    "model_tfidf = LogisticRegression(solver='saga', max_iter=100).fit(X_train_sel, y_train)\n",
    "    \n",
    "model_tfidf_score = model_tfidf.score(X_val_sel, y_val)\n",
    "    \n",
    "summary.append((c,np.shape(X_train_sel)[1],model_tfidf_score))\n",
    "    \n",
    "print(round((time.time() - start_time)/60,2),\"minutos \\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testando o modelo criado com as variáveis teste"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acurácia na base de teste=0.889 \n",
      "\n",
      "[[2177  229]\n",
      " [ 319 2233]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Acurácia na base de teste=%3.3f \\n\" % model_tfidf.score(X_test_sel, y_test))\n",
    "\n",
    "y_pred = model_tfidf.predict(X_test_sel)\n",
    "print(confusion_matrix(y_pred,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}