05downstream.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Beyond counting words: Working with word embeddings\n",
    "\n",
    "Workshop by Damian Trilling\n",
    "\n",
    "This notebook illustrates how we can use embeddings in Machine Learning tasks.\n",
    "\n",
    "As always, we first import neccesary modules. We also get our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install embeddingvectorizer    # you need to install this module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Supervised text classification\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn import metrics\n",
    "import joblib\n",
    "import eli5\n",
    "from nltk.sentiment import vader\n",
    "\n",
    "from embeddingvectorizer import EmbeddingCountVectorizer, EmbeddingTfidfVectorizer\n",
    "import embeddingvectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "\n",
    "# general\n",
    "import numpy as np\n",
    "import re\n",
    "# word embedding stuff\n",
    "import gensim\n",
    "import gensim.downloader as api\n",
    "from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex\n",
    "from gensim.corpora import Dictionary\n",
    "# from gensim.models import WordEmbeddingSimilarityIndex\n",
    "\n",
    "# data\n",
    "from courseutils import get_review_data\n",
    "\n",
    "# lets get more output\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'4.0.1'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gensim.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cached file reviewdata.pickle.bz2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-04-13 10:39:17,286 : INFO : loading Word2Vec object from mymodel\n",
      "2021-04-13 10:39:17,302 : INFO : loading wv recursively from mymodel.wv.* with mmap=None\n",
      "2021-04-13 10:39:17,302 : INFO : loading vectors from mymodel.wv.vectors.npy with mmap=None\n",
      "2021-04-13 10:39:17,388 : INFO : loading syn1neg from mymodel.syn1neg.npy with mmap=None\n",
      "2021-04-13 10:39:17,519 : INFO : setting ignored attribute cum_table to None\n",
      "2021-04-13 10:39:18,002 : INFO : Word2Vec lifecycle event {'fname': 'mymodel', 'datetime': '2021-04-13T10:39:18.002702', 'gensim': '4.0.1', 'python': '3.8.5 (default, Jan 27 2021, 15:41:15) \\n[GCC 9.3.0]', 'platform': 'Linux-5.4.0-70-generic-x86_64-with-glibc2.29', 'event': 'loaded'}\n"
     ]
    }
   ],
   "source": [
    "# get data\n",
    "reviews_train, reviews_test, y_train, y_test = get_review_data()\n",
    "\n",
    "reviews_train, y_train = shuffle(reviews_train, y_train, random_state=42)\n",
    "reviews_test, y_test = shuffle(reviews_test, y_test, random_state=42)\n",
    "\n",
    "# get word embedding model\n",
    "\n",
    "# pretrained:\n",
    "# wv = api.load('word2vec-google-news-300')\n",
    "# wv = api.load(\"glove-wiki-gigaword-300\")\n",
    "\n",
    "# or our own:\n",
    "wv = gensim.models.Word2Vec.load(\"mymodel\").wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# explore data here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 1: Document similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "termsim_index = WordEmbeddingSimilarityIndex(wv)\n",
    "documents = [re.split(r\"\\W\",e.lower()) for e in reviews_train[:100]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-04-13 10:39:18,059 : INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
      "2021-04-13 10:39:18,110 : INFO : built Dictionary(4674 unique tokens: ['', 'a', 'amigos', 'an', 'any']...) from 100 documents (total 27348 corpus positions)\n",
      "2021-04-13 10:39:18,111 : INFO : Dictionary lifecycle event {'msg': \"built Dictionary(4674 unique tokens: ['', 'a', 'amigos', 'an', 'any']...) from 100 documents (total 27348 corpus positions)\", 'datetime': '2021-04-13T10:39:18.111860', 'gensim': '4.0.1', 'python': '3.8.5 (default, Jan 27 2021, 15:41:15) \\n[GCC 9.3.0]', 'platform': 'Linux-5.4.0-70-generic-x86_64-with-glibc2.29', 'event': 'created'}\n",
      "2021-04-13 10:39:18,189 : INFO : constructing a sparse term similarity matrix using WordEmbeddingSimilarityIndex(keyedvectors=<gensim.models.keyedvectors.KeyedVectors object at 0x7f55186a5d90>, threshold=0.0, exponent=2.0, kwargs={})\n",
      "2021-04-13 10:39:18,190 : INFO : iterating over columns in dictionary order\n",
      "100%|██████████| 4674/4674 [00:53<00:00, 88.01it/s] \n",
      "2021-04-13 10:40:11,305 : INFO : constructed a sparse term similarity matrix with 0.835566% density\n"
     ]
    }
   ],
   "source": [
    "id2word = Dictionary(documents)\n",
    "bow_corpus = [id2word.doc2bow(document) for document in documents]\n",
    "similarity_matrix = SparseTermSimilarityMatrix(termsim_index, id2word)  # construct similarity matrix\n",
    "docsim_index = SoftCosineSimilarity(bow_corpus, similarity_matrix, num_best=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = re.split(\"\\W\",'''Pulp Fiction may be the single best film ever made, and quite appropriately it is by one of the most \n",
    "creative directors of all time, Quentin Tarantino. This movie is amazing from the beginning definition of pulp to\n",
    "the end credits and boasts one of the best casts ever assembled with the likes of Bruce Willis, Samuel L. Jackson, \n",
    "John Travolta, Uma Thurman, Harvey Keitel, Tim Roth and Christopher Walken. The dialog is surprisingly humorous for\n",
    "this type of film, and I think that's what has made it so successful. Wrongfully denied the many Oscars it was \n",
    "nominated for, Pulp Fiction is by far the best film of the 90s and no Tarantino film has surpassed the quality of\n",
    "this movie (although Kill Bill came close). As far as I'm concerned this is the top film of all-time and definitely \n",
    "deserves a watch if you haven't seen it.\n",
    "'''.lower())\n",
    "sims = docsim_index[id2word.doc2bow(query)]                                                                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# or let's take a  the first, second, or whatever docuemnt itself\n",
    "\n",
    "docindex = 2\n",
    "\n",
    "sims = docsim_index[id2word.doc2bow(documents[docindex])]      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('after watching this movie i was honestly disappointed   not because of the actors  story or directing   i was disappointed by this film advertisements  br    br   the trailers were suggesting that the battalion  have chosen the third way out  other than surrender or die  polish infos were even misguiding that they had the choice between being killed by own artillery or german guns  they even translated the title wrong as  misplaced battalion    this have tickled the right spot and i bought the movie  br    br   the disappointment started when i realized that the third way is to just sit down and count dead bodies followed by sitting down and counting dead bodies    then i began to think  hey  this story can t be that simple    i bet this clever officer will find some cunning way to save what left of his troops   well  he didn t  they were just sitting and waiting for something to happen  and so was i  br    br   the story was based on real events of world war i  so the writers couldn t make much use of their imagination  but even thought i found this movie really unchallenging and even a little bit boring  and as i wrote in the first place   it isn t fault of actors  writers or director   their marketing people have raised my expectations high above the level that this movie could cope with ',\n",
       " 'After watching this movie I was honestly disappointed - not because of the actors, story or directing - I was disappointed by this film advertisements.<br /><br />The trailers were suggesting that the battalion \"have chosen the third way out\" other than surrender or die (Polish infos were even misguiding that they had the choice between being killed by own artillery or German guns, they even translated the title wrong as \"misplaced battalion\"). This have tickled the right spot and I bought the movie.<br /><br />The disappointment started when I realized that the third way is to just sit down and count dead bodies followed by sitting down and counting dead bodies... Then I began to think \"hey, this story can\\'t be that simple... I bet this clever officer will find some cunning way to save what left of his troops\". Well, he didn\\'t, they were just sitting and waiting for something to happen. And so was I.<br /><br />The story was based on real events of World War I, so the writers couldn\\'t make much use of their imagination, but even thought I found this movie really unchallenging and even a little bit boring. And as I wrote in the first place - it isn\\'t fault of actors, writers or director - their marketing people have raised my expectations high above the level that this movie could cope with.')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check wether everything's ok\n",
    "\" \".join(documents[docindex]), reviews_train[docindex]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This review has a similarity of 1.0 with our query:\n",
      "After watching this movie I was honestly disappointed - not because of the actors, story or directing - I was disappointed by this film advertisements.<br /><br />The trailers were suggesting that the battalion \"have chosen the third way out\" other than surrender or die (Polish infos were even misguiding that they had the choice between being killed by own artillery or German guns, they even translated the title wrong as \"misplaced battalion\"). This have tickled the right spot and I bought the movie.<br /><br />The disappointment started when I realized that the third way is to just sit down and count dead bodies followed by sitting down and counting dead bodies... Then I began to think \"hey, this story can't be that simple... I bet this clever officer will find some cunning way to save what left of his troops\". Well, he didn't, they were just sitting and waiting for something to happen. And so was I.<br /><br />The story was based on real events of World War I, so the writers couldn't\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9548406004905701 with our query:\n",
      "I don't think I've ever felt this let down by a film before. After loving Guy Ritchie's two previous films (I don't count Swept Away - he was pussy blind), I was so looking forward to seeing this. <br /><br />The reviews were poor, then again, I don't trust the press anyway. More worrying was the fact that the internet buzz was that this was a bit of a stinker, so it was with some trepidation I handed over my £4.80 yesterday afternoon.<br /><br />I'm not even going to try to explain this film, mainly because I haven't got a clue what was going on and at one point I was honestly close to standing up and asking if it was just me who didn't get it! <br /><br />Unfortunately I think Ritchie seems to have fallen into his wife's trap of taking himself far too seriously.It seems it wasn't good enough for him to make films with good plots, laughs, snappy dialogue and good characters. It's almost as if he had a checklist of films he wanted to rip off, here are some of the ones I noticed:<br /><\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9445935487747192 with our query:\n",
      "I had high expectations of this movie (the title, translated, is \"How We Get Rid of the Others\"). After all, the concept is great: a near future in which the ruling elite has taken the consequence of the right-wing government's constant verbal and legislative persecution of so-called freeloaders and the left wing in general, and decided to just kill off everyone who cannot prove that they're contributing something to the establishment (the establishment being called \"the common good\", but actually meaning the interests of the ruling capitalist ideology).<br /><br />Very cool idea! Ideal for biting satire! Only, this movie completely blows its chance. The satire comes out only in a few scenes and performances of absurdity, but this satire is not sustained; it is neither sharp nor witty. And for an alleged comedy, the movie has nearly no funny scenes. The comedy, I assume, is supposed to be in the absurdity of the situations, but the situations are largely uncomfortable and over-serious,\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9442602396011353 with our query:\n",
      "Hello there,<br /><br />This is my first post in IMDb even though I use it as a reference for quite a while. I would therefore like to salute you all. The fact that I am a Greek is inevitably going to affect my judgement I hope not to your annoyance.<br /><br />I spent 2 years of my life, (all we Greeks did actually), analysing Omirus epos (and not Homers as you see everywhere), rhyme by rhyme. If I recall well it was Iliada (Iliad) on 8th grade and Odysseia (Odyssey) on 9th grade. Warner's Troy, was a big disappointment to me and my fellow Greeks around the campus (I study in the UK).<br /><br />Iliad epos is one of the very best literature works ever made. It was composed by a Greek poet Omirus a whole 400 years after the actual war. Historians put Trojan war around 1200 BC, and the actual reason of the war not being Helen's beauty but the strategically crucial position of Troy. That said one may now understand that Omirus epos is not presenting the actual events (as it's not accurat\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.940505862236023 with our query:\n",
      "I had to watch this in school. And to sum it up...<br /><br />Talentless actors, talentless script, and a talentless director.<br /><br />This movie is such a waste of your time. Don't even watch the movie. Don't bother. You will be so disappointed. My teacher said it was supposed to be good. How wrong she was. She even slept through it a little. The movie's actors were just bad. The best actor in there was the old man and that's not saying much. It's has horrible plot with awful characters. So unrealistic and I can honestly said it had no point. The script was unemotional and confusing. There was points in the movie when I furrowed my brows and said, \"What?\". Also there were just too many loose ties and plot holes. It was just absolutely horrendous.\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.940114438533783 with our query:\n",
      "Johnny and Jeremy are vampires of sorts. Minus the fangs, of course. They're dark, bitter creatures with nothing better to do than to spread their own misery. Through their charms (namely a sharp tongue and a fat wallet, respectively) they seduce desperate souls, who they proceed to torment and victimize. That's more or less the basis of this black comedy, as I understand it.<br /><br />It's not a blend of black humor that I can easily subscribe to, partly because it bothers me to imagine the audience rooting for the sleazy, main character. I did enjoy, however, the sound and the melody of the rapid-fire (and supposedly very witty) remarks. I was very impressed by the cast's strong acting, particularly David Thelis's; only the character of Jeremy seemed too bi-dimensional. The photography and the music, both dramatic and somber, work very well together. <br /><br />What really turns me off about \"Naked\" (and the main reason I'd never recommend it to anyone) is the way it repeatedly see\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9362901449203491 with our query:\n",
      "I gave this film 10 not because it is a superbly consistent movie, but for it's pure ability to evoke emotions in its audience. The story of one-woman's-struggle-against-all-odds is an old cliché by now, but very few films have carried it off with so much warmth and sincerity as The Color Purple.<br /><br />It also showed a different side to the African-American experience - showing that after slaves were granted freedom many fell into the ways of the hated 'white man' and were abusive of their own people. I find this an important point as it goes against the portray-white-on-black-violence-and-win-an-Oscar trend.<br /><br />Also the acting performances are superb - especially Oprah who I now have a new found respect for.<br /><br />Well worth watching - but keep some tissue handy.\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.936143159866333 with our query:\n",
      "I find it hard to believe that anyone would put this movie in the same context as the Exorcist. Where the Exorcist was subtle and creepy, Stigmata was blunt, clumsy, and way too formulaic.<br /><br />This is one of the most visually beautiful films I've seen in a while, but the imagery does not make up for the downward spiral of patronizing exposition that makes it unbearable. <br /><br />My interest in this movie was peaked when it was compared to The Exorcist, and my visit to the official web site increased that interest. The web site had many tales of \"actual\" stigmata throughout history. However, scene by scene, the movie is so obsessed by its quest for \"genuineness\" that it becomes comical at first, then outright hard to watch toward the end. I began getting suspicious when the priest charged with investigating potential miracles walks into the beauty parlor where our would-be heroine cuts hair and, evidently, flirts with priests.<br /><br />The plot: A woman without faith in God \n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9357336163520813 with our query:\n",
      "I enjoyed this film. I thought it was an excellent political thriller about something that's never happened before - a Secret Service agent going bad and involved in an assassination plot. Unfortunately, for Michael Douglas' character, \"Pete Garrison,\" they think HE's the mole but he isn't. <br /><br />He's just a morally-flawed agent having an affair with the First Lady! Since he's doing that, he's unable to give an acceptable polygraph exam and that makes him suspect number one when it's revealed there is a plot to kill the President.<br /><br />\"Garrison\" is forced to go on the lam but at the same time he's still trying to do the right thing by protecting the President. Douglas does a fine job in this role. I don't always care the people he plays but he's an excellent actor. Keifer Sutherland (\"David Breckinridge\") is equally as good (at least in here) as the fellow SS boss who hunts down Douglas until convinced he has been telling the truth. When he does the two of them work togeth\n",
      "\n",
      "*************************************************************\n",
      "\n",
      "This review has a similarity of 0.9355818629264832 with our query:\n",
      "The problem with the 1985 version of this movie is simple; Indiana Jones was so closely modeled after Alan Quartermain (or at least is an Alan Quartermain TYPE of character), that the '85 director made the mistake of plundering the IJ movies for dialog and story far too deeply. What you got as a finished product was a jumbled mess of the name Alan Quartermain, in an uneven hodge podge of a cheaply imitated IJ saga (with a touch of Austin Powers-esquire cheese here and there). <br /><br />It was labeled by many critics to have been a \"great parody,\" or \"unintentional comedy.\" Unintentional is the word. This movie was never intended to be humorous; witty, yes, but not humorous. Unfortunately, it's witless rather than witty.<br /><br />With this new M4TV mini-series, you get much more story, character development of your lead, solid portrayals, and a fine, even, entertaining blend. This story is a bit long; much longer than its predecessors, but deservedly so as this version carries a rea\n",
      "\n",
      "*************************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for index, similarity in sims:\n",
    "    print(f\"This review has a similarity of {similarity} with our query:\")\n",
    "    print(reviews_train[index][:1000])\n",
    "    print(\"\\n*************************************************************\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 2: Supervised Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A classical model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.85      0.87      0.86     12500\n",
      "         pos       0.87      0.85      0.86     12500\n",
      "\n",
      "    accuracy                           0.86     25000\n",
      "   macro avg       0.86      0.86      0.86     25000\n",
      "weighted avg       0.86      0.86      0.86     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(stop_words='english')\n",
    "X_train = vectorizer.fit_transform(reviews_train)\n",
    "X_test = vectorizer.transform(reviews_test)\n",
    "\n",
    "logreg = LogisticRegression(solver='liblinear')\n",
    "logreg.fit(X_train, y_train)\n",
    "\n",
    "y_pred = logreg.predict(X_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's discuss\n",
    "\n",
    "- what happened here under the hood?\n",
    "- How many features do we have?\n",
    "- How does X_train \"look\" like?\n",
    "\n",
    "**write your conclusions here**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's rewrite this into a pipeline (for easier use), and let's use a TfIDF vectorizer instead. This is probably as good as it can get."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.88      0.88      0.88     12500\n",
      "         pos       0.88      0.88      0.88     12500\n",
      "\n",
      "    accuracy                           0.88     25000\n",
      "   macro avg       0.88      0.88      0.88     25000\n",
      "weighted avg       0.88      0.88      0.88     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "traditionalpipe = Pipeline([('vectorizer', TfidfVectorizer()),\n",
    "                    ('logreg',LogisticRegression(solver='liblinear'))])\n",
    "\n",
    "traditionalpipe.fit(reviews_train, y_train)\n",
    "y_pred = traditionalpipe.predict(reviews_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**It's not the topic of today, but once we have such a pipeline, we can use a so-called gridsearch to find the optimal settings. For more info, see https://github.com/damian0604/bdaca/blob/master/12ec/week10/lecture10.pdf**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's use embeddings as input instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAKE SURE THAT YOU KNOW WHICH MODEL YOU ARE WORKING ON - can use either self-trained or pre-trained model\n",
    "\n",
    "# we need to convert `wv` to a slightliy different format:\n",
    "w2vmodel = dict(zip(wv.index2word, wv.vectors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.90      0.73      0.80     12500\n",
      "         pos       0.77      0.92      0.84     12500\n",
      "\n",
      "    accuracy                           0.82     25000\n",
      "   macro avg       0.83      0.82      0.82     25000\n",
      "weighted avg       0.83      0.82      0.82     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mypipe = Pipeline([('vectorizer', embeddingvectorizer.EmbeddingTfidfVectorizer(w2vmodel, operator='mean')),\n",
    "                    ('svm', \n",
    "                     SGDClassifier(loss='hinge', penalty='l2', tol=1e-4, alpha=1e-6, max_iter=1000, random_state=42))])\n",
    "\n",
    "# Generate BOW representation of word counts\n",
    "mypipe.fit(reviews_train, y_train)\n",
    "y_pred = mypipe.predict(reviews_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.83      0.83      0.83     12500\n",
      "         pos       0.83      0.83      0.83     12500\n",
      "\n",
      "    accuracy                           0.83     25000\n",
      "   macro avg       0.83      0.83      0.83     25000\n",
      "weighted avg       0.83      0.83      0.83     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mypipe = Pipeline([('vectorizer', embeddingvectorizer.EmbeddingTfidfVectorizer(w2vmodel, operator='mean')),\n",
    "                    ('logreg', LogisticRegression(solver='liblinear'))])\n",
    "\n",
    "# Generate BOW representation of word counts\n",
    "mypipe.fit(reviews_train, y_train)\n",
    "y_pred = mypipe.predict(reviews_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.83      0.83      0.83     12500\n",
      "         pos       0.83      0.83      0.83     12500\n",
      "\n",
      "    accuracy                           0.83     25000\n",
      "   macro avg       0.83      0.83      0.83     25000\n",
      "weighted avg       0.83      0.83      0.83     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mypipe = Pipeline([('vectorizer', embeddingvectorizer.EmbeddingCountVectorizer(w2vmodel, operator='mean')),\n",
    "                    ('logreg', LogisticRegression(solver='liblinear'))])\n",
    "\n",
    "# Generate BOW representation of word counts\n",
    "mypipe.fit(reviews_train, y_train)\n",
    "y_pred = mypipe.predict(reviews_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's discuss\n",
    "\n",
    "- what happened here under the hood?\n",
    "- How many features do we have?\n",
    "- How does X_train \"look\" like?\n",
    "\n",
    "**write your conclusions here**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## There's a reason why the classical approach worked so good that the embedding approach couldn't add anything.\n",
    "\n",
    "- can you see what?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_train_short = reviews_train[:100]\n",
    "reviews_test_short = reviews_test[:100] \n",
    "y_train_short = y_train[:100] \n",
    "y_test_short = y_test[:100] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.85      0.47      0.60        47\n",
      "         pos       0.66      0.92      0.77        53\n",
      "\n",
      "    accuracy                           0.71       100\n",
      "   macro avg       0.75      0.70      0.69       100\n",
      "weighted avg       0.75      0.71      0.69       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# traditional, short dataset\n",
    "traditionalpipe.fit(reviews_train_short, y_train_short )\n",
    "y_pred_short = traditionalpipe.predict(reviews_test_short)\n",
    "\n",
    "print(metrics.classification_report(y_test_short, y_pred_short))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         neg       0.75      0.70      0.73        47\n",
      "         pos       0.75      0.79      0.77        53\n",
      "\n",
      "    accuracy                           0.75       100\n",
      "   macro avg       0.75      0.75      0.75       100\n",
      "weighted avg       0.75      0.75      0.75       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# with embeddings, shor tdataset\n",
    "mypipe.fit(reviews_train_short, y_train_short )\n",
    "y_pred_short = mypipe.predict(reviews_test_short)\n",
    "\n",
    "print(metrics.classification_report(y_test_short, y_pred_short))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**write your conclusions here. Why do you think that the relative advantage turns around with the smaller dataset?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}