praise_categorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.express as px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = '../tec-rewards/distribution_rounds/'\n",
    "rounds = np.arange(1,8)\n",
    "# load all the praise\n",
    "allpraise_df = pd.DataFrame()\n",
    "for kr in rounds:\n",
    "    df =pd.read_csv(f'{datadir}/round-{kr}/distribution_results/raw_csv_exports/extended_praise_data.csv')\n",
    "    allpraise_df=pd.concat([allpraise_df,df[['REASON','AVG SCORE','TO USER ACCOUNT','DATE']]],axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>REASON</th>\n",
       "      <th>AVG SCORE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>for making edits in the welcome text</td>\n",
       "      <td>50.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for making edits in the welcome text.</td>\n",
       "      <td>4.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>for offering to help us improve some designs f...</td>\n",
       "      <td>27.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for invite me to play some music</td>\n",
       "      <td>7.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>for sharing material about TEC simulator and c...</td>\n",
       "      <td>21.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1726</th>\n",
       "      <td>for attending the Twitter planning TEAM call a...</td>\n",
       "      <td>10.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1727</th>\n",
       "      <td>for attending the Twitter planning TEAM call a...</td>\n",
       "      <td>3.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1728</th>\n",
       "      <td>for attending the Twitter planning TEAM call a...</td>\n",
       "      <td>5.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1729</th>\n",
       "      <td>for hopping into our weekly Communitas and giv...</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1730</th>\n",
       "      <td>for DAOing it in coindesk's article</td>\n",
       "      <td>11.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9690 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 REASON  AVG SCORE\n",
       "0                  for making edits in the welcome text      50.00\n",
       "1                 for making edits in the welcome text.       4.67\n",
       "2     for offering to help us improve some designs f...      27.00\n",
       "3                      for invite me to play some music       7.67\n",
       "4     for sharing material about TEC simulator and c...      21.00\n",
       "...                                                 ...        ...\n",
       "1726  for attending the Twitter planning TEAM call a...      10.33\n",
       "1727  for attending the Twitter planning TEAM call a...       3.67\n",
       "1728  for attending the Twitter planning TEAM call a...       5.50\n",
       "1729  for hopping into our weekly Communitas and giv...       4.00\n",
       "1730                for DAOing it in coindesk's article      11.33\n",
       "\n",
       "[9690 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allpraise_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import nltk\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "nltk.download('stopwords')\n",
    "\n",
    "# cleaning master function\n",
    "def clean_praise(praise):\n",
    "    # code adapted from: https://ourcodingclub.github.io/tutorials/topic-modelling-python/\n",
    "    my_stopwords = nltk.corpus.stopwords.words('english')\n",
    "    word_rooter = nltk.stem.snowball.PorterStemmer(ignore_stopwords=False).stem # clean words to the \"stem\" (e.g. words->word, talked->talk)\n",
    "    my_punctuation = '!\"$%&\\'()*+,-./:;<=>?[\\\\]^_`{|}~•@'\n",
    "\n",
    "    praise = praise.lower() # lower case\n",
    "    praise = re.sub('['+my_punctuation + ']+', ' ', praise) # strip punctuation\n",
    "    praise = re.sub('\\s+', ' ', praise) #remove double spacing\n",
    "    praise = re.sub('([0-9]+)', '', praise) # remove numbers\n",
    "    praise_token_list = [word for word in praise.split(' ')\n",
    "                            if word not in my_stopwords] # remove stopwords\n",
    "\n",
    "    praise_token_list = [word_rooter(word) if '#' not in word else word\n",
    "                        for word in praise_token_list] # apply word rooter\n",
    "\n",
    "    praise = ' '.join(praise_token_list)\n",
    "    return praise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# topic modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## clean the language data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mRunning cells with 'Python 3.10.4 64-bit' requires ipykernel package.\n",
      "Run the following command to install 'ipykernel' into the Python environment. \n",
      "Command: '/bin/python3 -m pip install ipykernel -U --user --force-reinstall'"
     ]
    }
   ],
   "source": [
    "cleaned_praise = allpraise_df['REASON'].apply(clean_praise)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhiwei/anaconda3/envs/rad_env/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# the vectorizer object will be used to transform text to vector form\n",
    "vectorizer = CountVectorizer(max_df=0.9, min_df=5, token_pattern='\\w+|\\$[\\d\\.]+|\\S+') # remove words appear less than 5 times or more than 90%\n",
    "\n",
    "# apply transformation\n",
    "tf = vectorizer.fit_transform(cleaned_praise).toarray() # term frequency\n",
    "\n",
    "# tf_feature_names tells us what word each column in the matric represents\n",
    "tf_feature_names = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1269"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tf_feature_names) # total number of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "number_of_topics = 10\n",
    "\n",
    "model = LatentDirichletAllocation(n_components=number_of_topics, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LatentDirichletAllocation(random_state=0)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LatentDirichletAllocation</label><div class=\"sk-toggleable__content\"><pre>LatentDirichletAllocation(random_state=0)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LatentDirichletAllocation(random_state=0)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(tf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_topics(model, feature_names, no_top_words):\n",
    "    topic_dict = {}\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        topic_dict[\"Topic %d words\" % (topic_idx)]= ['{}'.format(feature_names[i])\n",
    "                        for i in topic.argsort()[:-no_top_words - 1:-1]]\n",
    "        topic_dict[\"Topic %d weights\" % (topic_idx)]= ['{:.1f}'.format(topic[i])\n",
    "                        for i in topic.argsort()[:-no_top_words - 1:-1]]\n",
    "    return pd.DataFrame(topic_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic 0 words</th>\n",
       "      <th>Topic 0 weights</th>\n",
       "      <th>Topic 1 words</th>\n",
       "      <th>Topic 1 weights</th>\n",
       "      <th>Topic 2 words</th>\n",
       "      <th>Topic 2 weights</th>\n",
       "      <th>Topic 3 words</th>\n",
       "      <th>Topic 3 weights</th>\n",
       "      <th>Topic 4 words</th>\n",
       "      <th>Topic 4 weights</th>\n",
       "      <th>Topic 5 words</th>\n",
       "      <th>Topic 5 weights</th>\n",
       "      <th>Topic 6 words</th>\n",
       "      <th>Topic 6 weights</th>\n",
       "      <th>Topic 7 words</th>\n",
       "      <th>Topic 7 weights</th>\n",
       "      <th>Topic 8 words</th>\n",
       "      <th>Topic 8 weights</th>\n",
       "      <th>Topic 9 words</th>\n",
       "      <th>Topic 9 weights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tec</td>\n",
       "      <td>864.3</td>\n",
       "      <td>work</td>\n",
       "      <td>305.1</td>\n",
       "      <td>call</td>\n",
       "      <td>459.9</td>\n",
       "      <td>common</td>\n",
       "      <td>1295.3</td>\n",
       "      <td>prais</td>\n",
       "      <td>286.0</td>\n",
       "      <td>work</td>\n",
       "      <td>184.0</td>\n",
       "      <td>join</td>\n",
       "      <td>332.2</td>\n",
       "      <td>insight</td>\n",
       "      <td>100.3</td>\n",
       "      <td>work</td>\n",
       "      <td>397.7</td>\n",
       "      <td>param</td>\n",
       "      <td>641.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>commun</td>\n",
       "      <td>728.1</td>\n",
       "      <td>alway</td>\n",
       "      <td>181.0</td>\n",
       "      <td>attend</td>\n",
       "      <td>367.3</td>\n",
       "      <td>forum</td>\n",
       "      <td>777.3</td>\n",
       "      <td>propos</td>\n",
       "      <td>189.2</td>\n",
       "      <td>steward</td>\n",
       "      <td>166.9</td>\n",
       "      <td>graviton</td>\n",
       "      <td>239.1</td>\n",
       "      <td>amaz</td>\n",
       "      <td>98.4</td>\n",
       "      <td>reward</td>\n",
       "      <td>345.6</td>\n",
       "      <td>comm</td>\n",
       "      <td>508.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>call</td>\n",
       "      <td>727.2</td>\n",
       "      <td>’</td>\n",
       "      <td>173.7</td>\n",
       "      <td>commun</td>\n",
       "      <td>366.5</td>\n",
       "      <td>week</td>\n",
       "      <td>700.4</td>\n",
       "      <td>help</td>\n",
       "      <td>134.8</td>\n",
       "      <td>great</td>\n",
       "      <td>153.6</td>\n",
       "      <td>train</td>\n",
       "      <td>220.1</td>\n",
       "      <td>share</td>\n",
       "      <td>98.2</td>\n",
       "      <td>dashboard</td>\n",
       "      <td>342.6</td>\n",
       "      <td>parti</td>\n",
       "      <td>399.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>join</td>\n",
       "      <td>504.5</td>\n",
       "      <td>support</td>\n",
       "      <td>169.0</td>\n",
       "      <td>wg</td>\n",
       "      <td>272.5</td>\n",
       "      <td>token</td>\n",
       "      <td>697.1</td>\n",
       "      <td>get</td>\n",
       "      <td>103.2</td>\n",
       "      <td>post</td>\n",
       "      <td>119.8</td>\n",
       "      <td>call</td>\n",
       "      <td>200.3</td>\n",
       "      <td>te</td>\n",
       "      <td>94.5</td>\n",
       "      <td>system</td>\n",
       "      <td>284.8</td>\n",
       "      <td>wg</td>\n",
       "      <td>384.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http</td>\n",
       "      <td>362.6</td>\n",
       "      <td>help</td>\n",
       "      <td>154.6</td>\n",
       "      <td>sync</td>\n",
       "      <td>248.0</td>\n",
       "      <td>help</td>\n",
       "      <td>674.9</td>\n",
       "      <td>work</td>\n",
       "      <td>99.7</td>\n",
       "      <td>give</td>\n",
       "      <td>109.0</td>\n",
       "      <td>orient</td>\n",
       "      <td>187.5</td>\n",
       "      <td>meet</td>\n",
       "      <td>78.9</td>\n",
       "      <td>common</td>\n",
       "      <td>254.5</td>\n",
       "      <td>tec</td>\n",
       "      <td>380.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>youtu</td>\n",
       "      <td>345.1</td>\n",
       "      <td>ccd</td>\n",
       "      <td>128.3</td>\n",
       "      <td>weekli</td>\n",
       "      <td>176.7</td>\n",
       "      <td>engin</td>\n",
       "      <td>665.1</td>\n",
       "      <td>initi</td>\n",
       "      <td>97.2</td>\n",
       "      <td>hatch</td>\n",
       "      <td>105.4</td>\n",
       "      <td>today</td>\n",
       "      <td>183.4</td>\n",
       "      <td>twitter</td>\n",
       "      <td>72.3</td>\n",
       "      <td>upgrad</td>\n",
       "      <td>125.3</td>\n",
       "      <td>work</td>\n",
       "      <td>366.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>look</td>\n",
       "      <td>120.0</td>\n",
       "      <td>develop</td>\n",
       "      <td>108.0</td>\n",
       "      <td>communita</td>\n",
       "      <td>158.2</td>\n",
       "      <td>thank</td>\n",
       "      <td>664.2</td>\n",
       "      <td>share</td>\n",
       "      <td>92.9</td>\n",
       "      <td>question</td>\n",
       "      <td>100.8</td>\n",
       "      <td>soft</td>\n",
       "      <td>132.1</td>\n",
       "      <td>tec</td>\n",
       "      <td>72.1</td>\n",
       "      <td>push</td>\n",
       "      <td>68.6</td>\n",
       "      <td>attend</td>\n",
       "      <td>303.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>forward</td>\n",
       "      <td>109.5</td>\n",
       "      <td>make</td>\n",
       "      <td>105.4</td>\n",
       "      <td>reward</td>\n",
       "      <td>147.1</td>\n",
       "      <td>past</td>\n",
       "      <td>657.1</td>\n",
       "      <td>research</td>\n",
       "      <td>88.8</td>\n",
       "      <td>time</td>\n",
       "      <td>99.7</td>\n",
       "      <td>gov</td>\n",
       "      <td>126.1</td>\n",
       "      <td>thought</td>\n",
       "      <td>69.1</td>\n",
       "      <td>make</td>\n",
       "      <td>58.1</td>\n",
       "      <td>call</td>\n",
       "      <td>235.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>prais</td>\n",
       "      <td>95.1</td>\n",
       "      <td>tec</td>\n",
       "      <td>100.2</td>\n",
       "      <td>join</td>\n",
       "      <td>146.6</td>\n",
       "      <td>te</td>\n",
       "      <td>647.7</td>\n",
       "      <td>bot</td>\n",
       "      <td>83.9</td>\n",
       "      <td>make</td>\n",
       "      <td>98.5</td>\n",
       "      <td>week</td>\n",
       "      <td>118.9</td>\n",
       "      <td>make</td>\n",
       "      <td>67.7</td>\n",
       "      <td>config</td>\n",
       "      <td>54.1</td>\n",
       "      <td>host</td>\n",
       "      <td>197.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>orient</td>\n",
       "      <td>91.7</td>\n",
       "      <td>design</td>\n",
       "      <td>92.6</td>\n",
       "      <td>last</td>\n",
       "      <td>145.8</td>\n",
       "      <td>commun</td>\n",
       "      <td>645.4</td>\n",
       "      <td>incred</td>\n",
       "      <td>82.3</td>\n",
       "      <td>lead</td>\n",
       "      <td>94.9</td>\n",
       "      <td>amaz</td>\n",
       "      <td>102.1</td>\n",
       "      <td>commun</td>\n",
       "      <td>66.7</td>\n",
       "      <td>commun</td>\n",
       "      <td>53.1</td>\n",
       "      <td>today</td>\n",
       "      <td>191.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Topic 0 words Topic 0 weights Topic 1 words Topic 1 weights Topic 2 words  \\\n",
       "0           tec           864.3          work           305.1          call   \n",
       "1        commun           728.1         alway           181.0        attend   \n",
       "2          call           727.2             ’           173.7        commun   \n",
       "3          join           504.5       support           169.0            wg   \n",
       "4          http           362.6          help           154.6          sync   \n",
       "5         youtu           345.1           ccd           128.3        weekli   \n",
       "6          look           120.0       develop           108.0     communita   \n",
       "7       forward           109.5          make           105.4        reward   \n",
       "8         prais            95.1           tec           100.2          join   \n",
       "9        orient            91.7        design            92.6          last   \n",
       "\n",
       "  Topic 2 weights Topic 3 words Topic 3 weights Topic 4 words Topic 4 weights  \\\n",
       "0           459.9        common          1295.3         prais           286.0   \n",
       "1           367.3         forum           777.3        propos           189.2   \n",
       "2           366.5          week           700.4          help           134.8   \n",
       "3           272.5         token           697.1           get           103.2   \n",
       "4           248.0          help           674.9          work            99.7   \n",
       "5           176.7         engin           665.1         initi            97.2   \n",
       "6           158.2         thank           664.2         share            92.9   \n",
       "7           147.1          past           657.1      research            88.8   \n",
       "8           146.6            te           647.7           bot            83.9   \n",
       "9           145.8        commun           645.4        incred            82.3   \n",
       "\n",
       "  Topic 5 words Topic 5 weights Topic 6 words Topic 6 weights Topic 7 words  \\\n",
       "0          work           184.0          join           332.2       insight   \n",
       "1       steward           166.9      graviton           239.1          amaz   \n",
       "2         great           153.6         train           220.1         share   \n",
       "3          post           119.8          call           200.3            te   \n",
       "4          give           109.0        orient           187.5          meet   \n",
       "5         hatch           105.4         today           183.4       twitter   \n",
       "6      question           100.8          soft           132.1           tec   \n",
       "7          time            99.7           gov           126.1       thought   \n",
       "8          make            98.5          week           118.9          make   \n",
       "9          lead            94.9          amaz           102.1        commun   \n",
       "\n",
       "  Topic 7 weights Topic 8 words Topic 8 weights Topic 9 words Topic 9 weights  \n",
       "0           100.3          work           397.7         param           641.8  \n",
       "1            98.4        reward           345.6          comm           508.3  \n",
       "2            98.2     dashboard           342.6         parti           399.6  \n",
       "3            94.5        system           284.8            wg           384.1  \n",
       "4            78.9        common           254.5           tec           380.8  \n",
       "5            72.3        upgrad           125.3          work           366.6  \n",
       "6            72.1          push            68.6        attend           303.4  \n",
       "7            69.1          make            58.1          call           235.9  \n",
       "8            67.7        config            54.1          host           197.2  \n",
       "9            66.7        commun            53.1         today           191.6  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_top_words = 10\n",
    "display_topics(model, tf_feature_names, no_top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: get for each praise which  topic does it belong to\n",
    "# Then, roughly check if the score relates to the praise\n",
    "\n",
    "# Finally, compare using just some keywords for that."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# just categorize by keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "among 9690 praises, 9481 have scores more than 0. Only include them. Next, clean them up.\n"
     ]
    }
   ],
   "source": [
    "nonzerodf = allpraise_df.loc[allpraise_df['AVG SCORE']>0]\n",
    "nonzerodf.insert(0,'CLEANED REASON',nonzerodf['REASON'].apply(clean_praise))\n",
    "\n",
    "print(f'among {len(allpraise_df)} praises, {len(nonzerodf)} have scores more than 0. Only include them. Next, clean them up.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from re import search # for searching sub strings\n",
    "type_keywords = {'attendance':'join|attend|show up|participat','discussion':'question|ask|discuss|discussion','work':'help|work|design|make|write|hack|edit','lead':'host|lead|initiate|form|organize|steward','share':'share|spread','twitter':'twitter|tweet','hack':'hack|test','general':'support|awesome','IRL':'trip|conference'}\n",
    "allcategs = []\n",
    "for kr,row in nonzerodf.iterrows():\n",
    "    category = []\n",
    "    praise = row['CLEANED REASON'].lower()\n",
    "    for praise_type,keywords in type_keywords.items():\n",
    "        if search(keywords,praise):\n",
    "            category.append(praise_type)\n",
    "    if len(category):\n",
    "        allcategs.append(category)\n",
    "    else:\n",
    "        allcategs.append(np.nan)\n",
    "category_df = pd.concat([nonzerodf.reset_index(), pd.DataFrame({\"category\":allcategs})],axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3247 out of 9481 uncategorized\n"
     ]
    }
   ],
   "source": [
    "category_df.loc[category_df['category'].isnull()].to_csv('uncateogrized.csv')\n",
    "print(f\"{sum(category_df['category'].isnull())} out of {len(category_df)} praises uncategorized\")\n",
    "category_df.to_csv('categorized_praise.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "attendance              join|attend|show up|participat\n",
       "discussion             question|ask|discuss|discussion\n",
       "work             help|work|design|make|write|hack|edit\n",
       "lead          host|lead|initiate|form|organize|steward\n",
       "share                                     share|spread\n",
       "twitter                                  twitter|tweet\n",
       "hack                                         hack|test\n",
       "general                                support|awesome\n",
       "IRL                                    trip|conference\n",
       "dtype: object"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(type_keywords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# analysis based on categorization\n",
    "When there's a praise matching more than one category, they will be counted multiple times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "categ_praise_scores = {k:[] for k in type_keywords.keys()}\n",
    "\n",
    "for kr,row in category_df.iterrows():\n",
    "    if type(row['category']) is list:\n",
    "        for key in row['category']:\n",
    "            categ_praise_scores[key] += [{'praise':row['REASON'],'avg_score':row['AVG SCORE'],'receiver':row['TO USER ACCOUNT'],'date':row['DATE']}]\n",
    "categ_praise_scores_df = dict.fromkeys(type_keywords.keys())\n",
    "for key, item in categ_praise_scores.items():\n",
    "    categ_praise_scores_df[key]= pd.DataFrame(item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  the average, min, max score of each categorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "categ_stats = dict.fromkeys(type_keywords.keys())\n",
    "for categ in categ_praise_scores_df.keys():\n",
    "    categ_stats[categ] = {'mean':np.mean(categ_praise_scores_df[categ]['avg_score']),\n",
    "                            'max':np.max(categ_praise_scores_df[categ]['avg_score']),\n",
    "                            'min':np.min(categ_praise_scores_df[categ]['avg_score'])}\n",
    "categ_stats_df = pd.DataFrame(categ_stats)\n",
    "categ_stats_df.transpose().sort_values(by='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>attendance</th>\n",
       "      <td>2.929640</td>\n",
       "      <td>55.00</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twitter</th>\n",
       "      <td>5.120920</td>\n",
       "      <td>47.75</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>share</th>\n",
       "      <td>5.177013</td>\n",
       "      <td>73.33</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discussion</th>\n",
       "      <td>6.421413</td>\n",
       "      <td>79.33</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IRL</th>\n",
       "      <td>8.790000</td>\n",
       "      <td>18.33</td>\n",
       "      <td>1.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>general</th>\n",
       "      <td>9.912228</td>\n",
       "      <td>58.25</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work</th>\n",
       "      <td>10.381128</td>\n",
       "      <td>84.67</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead</th>\n",
       "      <td>10.561823</td>\n",
       "      <td>125.67</td>\n",
       "      <td>0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hack</th>\n",
       "      <td>16.658664</td>\n",
       "      <td>125.67</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 mean     max   min\n",
       "attendance   2.929640   55.00  0.03\n",
       "twitter      5.120920   47.75  0.33\n",
       "share        5.177013   73.33  0.10\n",
       "discussion   6.421413   79.33  0.10\n",
       "IRL          8.790000   18.33  1.50\n",
       "general      9.912228   58.25  0.73\n",
       "work        10.381128   84.67  0.10\n",
       "lead        10.561823  125.67  0.13\n",
       "hack        16.658664  125.67  1.00"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top 3 highest scored praise in each category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "# attendance\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 55.0 | iviangita#3204 | for accepting the repsonisbility of acting Steward for the TEC and keeping tht WG afloat while we put out a call for new legal minds to join. | 2021-11-03\n",
       "| 25.33 | griff (💜, 💜)#8888 | for talking about Conviction Voting at an ETHCC venue, which lead Michael to meet Livia, and then Mt. Manu, who joined him to work in ArborVote DAO | 2021-07-30\n",
       "| 24.25 | Juankbell#7458 | for contributing to Graviton Training, you’ve been doing an amazing job and the attendance has been absolutely phenomenal | 2021-11-04\n",
       "# discussion\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 79.33 | Nuggan#5183 | for jumping in params, giving feedback and asking questions | 2021-07-15\n",
       "| 68.0 | sem(🌸,🐝)#0161 | for engaging in discussion on the TE Commons Forum (https://forum.tecommons.org) the past week. Thank you for helping our Token Engineering Commons community share and learn! | 2021-07-15\n",
       "| 51.67 | Alp#3768 | for a 2-hour onboarding session full of questions, insights, and specialized knowledge | 2021-07-28\n",
       "# work\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 84.67 | natesuits#4789 | for helping me write and rewrite the forum post on the power of defaults: https://forum.tecommons.org/t/the-power-of-defaults-in-the-commons-configuration-dashboard/511/2 | 2021-07-25\n",
       "| 80.33 | Nuggan#5183 | for helping me write and rewrite the forum post on the power of defaults: https://forum.tecommons.org/t/the-power-of-defaults-in-the-commons-configuration-dashboard/511/2 | 2021-07-25\n",
       "| 77.93 | divine_comedian#5493 | for their work building the Commons Configuration Dashboard | 2021-07-30\n",
       "# lead\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 125.67 | sem(🌸,🐝)#0161 | for developing and testing the smart contracts. They are actually hosting a demo of the augmented bonding curve and all the commons upgrade tooling. Much admiration and respect for that | 2021-11-28\n",
       "| 78.67 | Vyvy-vi#5040 | for being our bot & data guru that we can always reach out to for meaningful information and problem solving | 2022-01-10\n",
       "| 68.0 | mZ#3472 | for thought leadership in web3 and for keeping engineering ethics as our community’s North Star | 2021-12-25\n",
       "# share\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 73.33 | sem(🌸,🐝)#0161 | for the incredible work with the demos, making improvements, managing the tech team and for being such a good teacher across the space and sharing his knowledge | 2021-07-15\n",
       "| 68.0 | sem(🌸,🐝)#0161 | for engaging in discussion on the TE Commons Forum (https://forum.tecommons.org) the past week. Thank you for helping our Token Engineering Commons community share and learn! | 2021-07-15\n",
       "| 59.33 | sem(🌸,🐝)#0161 | for the Real Time Launch action! Great sharing the war room with you!!! | 2022-01-24\n",
       "# twitter\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 47.75 | iviangita#3204 | for finding out our twitter account as block and fix it super quick!!⚡ (We need to be carefull with bots) | 2021-09-08\n",
       "| 42.0 | iviangita#3204 | for all the incredible behind-the-scenes work and all the little things she’s constantly doing in the back office helping with Twitter, the board and work agreements | 2021-09-02\n",
       "| 33.33 | innov8tor3#3988 | for mentioning or retweeting TE Commons on the socials the past week! Thank you for helping us grow the Token Engineering Commons community and spreading the message! 🙏🏼☺️ | 2021-07-15\n",
       "# hack\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 125.67 | sem(🌸,🐝)#0161 | for developing and testing the smart contracts. They are actually hosting a demo of the augmented bonding curve and all the commons upgrade tooling. Much admiration and respect for that | 2021-11-28\n",
       "| 70.67 | liviade#1387 | for the long verification hack session for the final IH list | 2021-07-15\n",
       "| 63.67 | VitorNunes#0090 | for all the work on the CCD, the designs, comments, user testing and making things understandable for people | 2021-08-19\n",
       "# general\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 58.25 | akrtws (TE Academy)#4246 | for keeping projects, companies, and really smart people excited about supporting education for TEs! | 2021-10-28\n",
       "| 41.75 | natesuits#4789 | for their support in the creation of the Communitas WG 🏘️ | 2021-09-18\n",
       "| 39.25 | divine_comedian#5493 | for design, development, ideation and copywriting support on the CCD | 2021-09-01\n",
       "# IRL\n",
       "    | Avg. score | To | Reason | Date |\n",
       "     |:-----------|----|:-------|\n",
       "| 18.33 | chuygarcia.eth#6692 | for all the work in Comms, for leading Comms and helping organize the trip to Paris | 2021-07-15\n",
       "| 16.8 | jukren#8803 | for making the Paris trip possible ✈️ | 2021-08-05\n",
       "| 15.0 | mateodaza#3156 | for the epic road trip we had to Paris!!! | 2021-07-17\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Markdown as md\n",
    "\n",
    "mdtext = ''\n",
    "for categ in categ_praise_scores_df.keys():\n",
    "    categ_name = '# '+categ + '\\n'\n",
    "    toppraise = categ_praise_scores_df[categ].sort_values(by='avg_score',ascending=False).iloc[:3]\n",
    "    top3_table= (f\"\\\n",
    "    | Avg. score | To | Reason | Date |\\n \\\n",
    "    |:-----------|----|:-------|\\n\")\n",
    "    for kr,row in toppraise.iterrows():\n",
    "        to_user = row['receiver']\n",
    "        reason = row['praise']\n",
    "        score = row['avg_score']\n",
    "        date = row['date'][:10]\n",
    "                    \n",
    "        top3_table += (f\"| {score} | {to_user} | {reason} | {date}\\n\")\n",
    "        #print(f'Praise score average: {score}\\nFROM {from_user} TO {to_user},reason:\\n{reason}\\n')\n",
    "    mdtext += categ_name + top3_table    \n",
    "md(mdtext)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO\n",
    "- maybe further adjust keyword to make the top scores look normal\n",
    "- how to make keywords a manipulable setting in json?\n",
    "- incorporate this into cross-period analysis"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('rad_venv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6f5e358c8e33f9727a38b86c793cce5dae3c31b66c870660197da5de07d8c93c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}