arxiv2kindle_with_latex_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import needed libraries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For security reasons you need to create credentials.py file with dict like the following:  \n",
    "credentials = {  \n",
    "    'kindle_email': \"YOUR_EMAIL@kindle.com\",  \n",
    "    'your_gmail': \"YOUR_GMAIL@gmail.com\",  \n",
    "    'gmailpass': \"YOUR_PASS\",  \n",
    "               }  \n",
    "\n",
    "For creating gmailpass, please follow the [link](https://myaccount.google.com/apppasswords). Note that the account security settings will have to \"allow unsecure apps\" for permission to use the Gmail SMTP server with TLS.  \n",
    "\n",
    "**Do not forget to add this file to .gitignore**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(180000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 180 seconds\n"
     ]
    }
   ],
   "source": [
    "# for not to lose progress and for updating without reloading\n",
    "%autosave 180\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import requests\n",
    "import lxml.html as html\n",
    "import re\n",
    "import os, sys\n",
    "import wget\n",
    "import glob\n",
    "import tempfile\n",
    "import tarfile\n",
    "\n",
    "from credentials import credentials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parameters\n",
    "## Input parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the needed_article_url can be an arxiv URL or any string containing an arxiv ID.\n",
    "needed_article_url = \"https://arxiv.org/pdf/2107.12708.pdf\"\n",
    "\n",
    "# paper settings (decrease width/height to increase font)\n",
    "landscape = False # False=horizontal, True=vertical\n",
    "width = \"4.2in\"\n",
    "height = \"6.55in\"\n",
    "margin = \"0.1in\"\n",
    "\n",
    "show_generated_pdf = True\n",
    "send = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "kindle_email = credentials['kindle_email']\n",
    "your_gmail = credentials['your_gmail']\n",
    "gmailpass = credentials['gmailpass']\n",
    "\n",
    "# settings for latex geometry package:\n",
    "if not landscape:\n",
    "    width, height = height, width\n",
    "geom_settings = dict(paperwidth=width, paperheight=height, margin=margin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download archived arxiv archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arxiv_id: 2107.12708\n",
      "arxiv_title: QA Dataset Explosion: A Taxonomy of NLP Resources for Question Answering and Reading Comprehension\n"
     ]
    }
   ],
   "source": [
    "arxiv_id = re.match(r'((http|https)://.*?/)?(?P<id>\\d{4}\\.\\d{4,5}(v\\d{1,2})?)', needed_article_url).group('id')\n",
    "arxiv_abs = 'http://arxiv.org/abs/' + arxiv_id\n",
    "arxiv_pdf = 'http://arxiv.org/pdf/' + arxiv_id\n",
    "arxiv_pgtitle = html.fromstring(requests.get(arxiv_abs).text.encode('utf8')).xpath('/html/head/title/text()')[0]\n",
    "arxiv_title = re.sub(r'\\s+', ' ', re.sub(r'^\\[[^]]+\\]\\s*', '', arxiv_pgtitle), re.DOTALL)\n",
    "arxiv_title_scrubbed = re.sub('[^-_A-Za-z0-9]+', '_', arxiv_title, re.DOTALL)\n",
    "\n",
    "print('arxiv_id:', arxiv_id)\n",
    "print('arxiv_title:', arxiv_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100% [..........................................................................] 1819262 / 1819262"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\QA Dataset Explosion: A Taxonomy of NLP Resources for Question Answering and Reading Comprehension.tar.gz'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create temporary directory\n",
    "d = tempfile.mkdtemp(prefix='arxiv2kindle_')\n",
    "\n",
    "archive_url = 'http://arxiv.org/e-print/' + arxiv_id\n",
    "\n",
    "# download tar.gz file and add file extension\n",
    "tar_filename = wget.download(archive_url, out=os.path.join(d, ''.join([arxiv_title, '.tar.gz'])))\n",
    "tar_filename"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process the files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'found files with .tex extension'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\02_explosion.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\03_probing-vs-info-seeking.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\043_format_question.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\04_format.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\05_modality.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\06_amount.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\07_domain.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\08_discourse.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\09_languages.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\10_reasoning.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\11_discussion.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\12_conclusion.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\acm_template_text.tex',\n",
       " 'C:\\\\Users\\\\Oleg\\\\AppData\\\\Local\\\\Temp\\\\arxiv2kindle_72w6ntgm\\\\main.tex']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# extract file contents\n",
    "os.chdir(d)\n",
    "tf = tarfile.open(tar_filename)\n",
    "tf.extractall()\n",
    "\n",
    "# find tex files\n",
    "texfiles = glob.glob(os.path.join(d, '*.tex'))\n",
    "display('found files with .tex extension', texfiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "correct file: C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\main.tex\n"
     ]
    }
   ],
   "source": [
    "def reorder_list(ls):\n",
    "    # works also for 'domain'. may be fixed with regexp\n",
    "    main_files = [file for file in ls if 'main.tex' in file]\n",
    "    for file in main_files:\n",
    "        ls.remove(file)\n",
    "        ls.insert(0, file)\n",
    "\n",
    "    template_files = [file for file in ls if 'template' in file]\n",
    "    for file in template_files:\n",
    "        ls.remove(file)\n",
    "        ls.append(file)\n",
    "    \n",
    "    return ls\n",
    "\n",
    "# finding the main tex file\n",
    "def find_main_tex(texfiles):\n",
    "    texfiles = reorder_list(texfiles)\n",
    "    \n",
    "    for texfile in texfiles:\n",
    "        with open(texfile, 'r') as f:\n",
    "            src = f.readlines()\n",
    "        for line in src:\n",
    "            if line.startswith('\\documentclass'):\n",
    "                print('correct file: ' + texfile)\n",
    "                return texfile\n",
    "            else:\n",
    "                continue\n",
    "    print('correct file not found')\n",
    "    \n",
    "texfile = find_main_tex(texfiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open(texfile, 'r') as f:\n",
    "    src = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter comments/newlines for easier debugging:\n",
    "src = [line for line in src if line[0] != '%' and len(line.strip()) > 0]\n",
    "\n",
    "# strip column stuff and stuff in documentclass line:\n",
    "src[0] = re.sub(r'\\b\\d+pt\\b', '', src[0]) # strip font size, ex. \"11pt\"\n",
    "src[0] = re.sub(r'\\b\\w+column\\b', '', src[0]) # strip \n",
    "src[0] = re.sub(r'\\b\\w+paper\\b', '', src[0]) # strip paper size, ex. \"letterpaper\" or \"a4paper\"\n",
    "src[0] = re.sub(r'(?<=\\[),', '', src[0]) # remove extraneous starting commas\n",
    "src[0] = re.sub(r',(?=[\\],])', '', src[0]) # remove extraneous middle/ending commas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for the case if we use the package after it was initialized with other params\n",
    "src.insert(0, \"\\\\PassOptionsToPackage{\" + ','.join(k+'='+v for k,v in geom_settings.items()) +\"}{geometry}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find begin{document}:\n",
    "begindocs = [i for i, line in enumerate(src) if re.match(r'\\s*\\\\begin{document}', line)]\n",
    "\n",
    "try:\n",
    "    assert(len(begindocs) == 1)\n",
    "    src.insert(begindocs[0], '\\\\usepackage['+','.join(k+'='+v for k,v in geom_settings.items())+']{geometry}\\n')\n",
    "    src.insert(begindocs[0], '\\\\usepackage{times}\\n')\n",
    "    src.insert(begindocs[0], '\\\\pagestyle{empty}\\n')\n",
    "    if landscape:\n",
    "        src.insert(begindocs[0], '\\\\usepackage{pdflscape}\\n')\n",
    "except:\n",
    "    print('Beginning not found. Adding needd packages to the beginning of the file')\n",
    "    try:\n",
    "        if landscape:\n",
    "            src.insert(0, '\\\\usepackage{pdflscape}\\n')\n",
    "        src.insert(0, '\\\\pagestyle{empty}\\n')\n",
    "        src.insert(0, '\\\\usepackage{times}\\n')\n",
    "        src.insert(0, '\\\\usepackage['+','.join(k+'='+v for k,v in geom_settings.items())+']{geometry}\\n')\n",
    "    except:\n",
    "        print('assert occured')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shrink figures to be at most the size of the page:\n",
    "for i, line in enumerate(src):\n",
    "    m = re.search(r'\\\\includegraphics\\[width=([.\\d]+)\\\\(line|text)width\\]', line)\n",
    "    if m:\n",
    "        mul = m.group(1)\n",
    "        print(m)\n",
    "        src[i] = re.sub(r'\\\\includegraphics\\[width=([.\\d]+)\\\\(line|text)width\\]',\n",
    "                   r'\\\\includegraphics[width={mul}\\\\textwidth,height={mul}\\\\textheight,keepaspectratio]'.format(mul=mul),\n",
    "                   line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace phrases or words found in dictionary\n",
    "def normalize_by_dictionary(word, dictionary):\n",
    "    result = []\n",
    "    for word in word.split():\n",
    "        # if word is in uppercase\n",
    "        if word == word.upper():\n",
    "            if word.lower() in dictionary:\n",
    "                result.append(dictionary[word.lower()].upper())\n",
    "            else:\n",
    "                result.append(word)\n",
    "        else:\n",
    "            if word.lower() in dictionary:\n",
    "                result.append(dictionary[word.lower()])\n",
    "            else:\n",
    "                result.append(word)\n",
    "    \n",
    "    return \" \".join(result)\n",
    "\n",
    "\n",
    "latex_rubbish_mapping = {'пјЏ'.lower(): '/', 'In': 'in'}\n",
    "\n",
    "# replace this fucking latex shit\n",
    "for i in range(len(src)):\n",
    "    line = src[i].split()\n",
    "    for j, word in enumerate(line):\n",
    "            normalized_word = normalize_by_dictionary(word, latex_rubbish_mapping)\n",
    "            if normalized_word != word:\n",
    "                print(word, normalized_word)\n",
    "                print(i)\n",
    "            src[i] = src[i].replace(word, normalized_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bad_images []\n"
     ]
    }
   ],
   "source": [
    "# find non-compiling file extensions\n",
    "folder_files = os.listdir()\n",
    "bad_images = [file for file in folder_files if '.ps' in file or '.eps' in file]\n",
    "print('bad_images', bad_images)\n",
    "\n",
    "# replace strange formats with pdf images\n",
    "for file in bad_images:\n",
    "    print(file)\n",
    "    filename = file.split('.')[0]\n",
    "    ! ps2pdf {file} {''.join([filename,'.pdf'])}\n",
    "    ! rm {file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all files in article dir\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\02_explosion.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\03_probing-vs-info-seeking.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\043_format_question.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\04_format.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\05_modality.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\06_amount.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\07_domain.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\08_discourse.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\09_languages.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\10_reasoning.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\11_discussion.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\12_conclusion.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\ACM-Reference-Format.bbx\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\ACM-Reference-Format.bst\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\ACM-Reference-Format.cbx\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\ACM-Reference-Format.dbx\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\acmart.cls\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\acmart.pdf\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\acm_template_text.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\main.bbl\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\main.tex\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\QA Dataset Explosion\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\README\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\sample-base.bib\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\sample-franklin.png\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\figs\\qa_survey_diagrams.png\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\figs\\qa_survey_diagrams_v1.pdf\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\figs\\qa_survey_diagrams_v2.5.png\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\figs\\qa_survey_diagrams_v2.pdf\n",
      "C:\\Users\\Oleg\\AppData\\Local\\Temp\\arxiv2kindle_72w6ntgm\\figs\\qa_survey_diagrams_v3.pdf\n"
     ]
    }
   ],
   "source": [
    "print('all files in article dir')\n",
    "for path, subdirs, files in os.walk(os.getcwd()):\n",
    "    for name in files:\n",
    "        print(os.path.join(path, name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# src -> tex.bak and compile\n",
    "os.rename(texfile, texfile+'.bak')\n",
    "with open(texfile, 'w') as f:\n",
    "    f.writelines(src)\n",
    "\n",
    "# texout = !pdflatex {texfile} && pdflatex {texfile} && pdflatex {texfile}\n",
    "texout = !pdflatex {texfile}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Package natbib Warning: Citation `augenstein-etal-2019-multifc' on page 15 unde\",\n",
       " 'fined on input line 118.',\n",
       " '',\n",
       " '',\n",
       " \"Package natbib Warning: Citation `atanasova-etal-2020-generating-fact' on page \",\n",
       " '15 undefined on input line 118.',\n",
       " '',\n",
       " '',\n",
       " \"LaTeX Warning: Reference `sec:domains' on page 1 undefined on input line 120.\",\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `Yatskar_2019_Qualitative_Comparison_of_CoQA_S',\n",
       " \"QuAD_20_and_QuAC' on page 15 undefined on input line 120.\",\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `FischTalmorEtAl_2019_MRQA_2019_Shared_Task_Ev',\n",
       " \"aluating_Generalization_in_Reading_Comprehension' on page 15 undefined on input\",\n",
       " ' line 120.',\n",
       " '',\n",
       " '',\n",
       " \"LaTeX Warning: Reference `sec:languages-mono' on page 1 undefined on input line\",\n",
       " ' 121.',\n",
       " '',\n",
       " '',\n",
       " \"LaTeX Warning: Reference `sec:discussion-required-skills' on page 1 undefined o\",\n",
       " 'n input line 124.',\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `Linzen_2020_How_Can_We_Accelerate_Progress_To',\n",
       " \"wards_Human-like_Linguistic_Generalization' on page 15 undefined on input line \",\n",
       " '124.',\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `GardnerMerrillEtAl_2021_Competency_Problems_O',\n",
       " \"n_Finding_and_Removing_Artifacts_in_Language_Data' on page 15 undefined on inpu\",\n",
       " 't line 124.',\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `BenderFriedman_2018_Data_Statements_for_Natur',\n",
       " 'al_Language_Processing_Toward_Mitigating_System_Bias_and_Enabling_Better_Scienc',\n",
       " \"e' on page 15 undefined on input line 127.\",\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `GebruMorgensternEtAl_2020_Datasheets_for_Data',\n",
       " \"sets' on page 15 undefined on input line 127.\",\n",
       " '',\n",
       " '',\n",
       " 'Package natbib Warning: Citation `MitchellWuEtAl_2019_Model_Cards_for_Model_Rep',\n",
       " \"orting' on page 15 undefined on input line 127.\",\n",
       " '',\n",
       " '',\n",
       " \"Package natbib Warning: Citation `Rogers_2021_Changing_World_by_Changing_Data' \",\n",
       " 'on page 15 undefined on input line 127.',\n",
       " '',\n",
       " ') (12_conclusion.tex) (main.bbl [15.15]',\n",
       " 'Underfull \\\\vbox (badness 10000) has occurred while \\\\output is active [16.16]',\n",
       " 'Underfull \\\\vbox (badness 10000) has occurred while \\\\output is active [17.17]',\n",
       " '[18.18] [19.19]',\n",
       " 'Underfull \\\\vbox (badness 10000) has occurred while \\\\output is active [20.20]',\n",
       " '[21.21]) [22.22]',\n",
       " '',\n",
       " 'Package natbib Warning: There were undefined citations.',\n",
       " '',\n",
       " '',\n",
       " 'Class acmart Warning: Some images may lack descriptions.',\n",
       " '',\n",
       " '(main.aux',\n",
       " '',\n",
       " 'Package natbib Warning: Citation(s) may have changed.',\n",
       " '(natbib)                Rerun to get citations correct.',\n",
       " '',\n",
       " ')',\n",
       " '',\n",
       " 'LaTeX Warning: There were undefined references.',\n",
       " '',\n",
       " '',\n",
       " 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.',\n",
       " '',\n",
       " '',\n",
       " \"Package rerunfilecheck Warning: File `main.out' has changed.\",\n",
       " '(rerunfilecheck)                Rerun to get outlines right',\n",
       " \"(rerunfilecheck)                or use package `bookmark'.\",\n",
       " '',\n",
       " ' )',\n",
       " '(see the transcript file for additional information)pdfTeX warning (dest): name',\n",
       " '{Hfootnote.15} has been referenced but does not exist, replaced by a fixed one',\n",
       " '',\n",
       " 'pdfTeX warning (dest): name{Hfootnote.14} has been referenced but does not exis',\n",
       " 't, replaced by a fixed one',\n",
       " '',\n",
       " '{D:/Programs/Computer/MKTeX/fonts/enc/dvips/base/8r.enc} <C:\\\\Users\\\\Oleg\\\\AppData',\n",
       " '\\\\Local\\\\MiKTeX\\\\2.9\\\\fonts/pk/ljfour/jknappen/ec/dpi600\\\\tcrm0800.pk><D:/Programs/C',\n",
       " 'omputer/MKTeX/fonts/type1/public/newtx/txsys.pfb><D:/Programs/Computer/MKTeX/fo',\n",
       " 'nts/type1/urw/helvetic/uhvb8a.pfb><D:/Programs/Computer/MKTeX/fonts/type1/urw/h',\n",
       " 'elvetic/uhvr8a.pfb><D:/Programs/Computer/MKTeX/fonts/type1/urw/helvetic/uhvro8a',\n",
       " '.pfb><D:/Programs/Computer/MKTeX/fonts/type1/urw/times/utmb8a.pfb><D:/Programs/',\n",
       " 'Computer/MKTeX/fonts/type1/urw/times/utmr8a.pfb><D:/Programs/Computer/MKTeX/fon',\n",
       " 'ts/type1/urw/times/utmri8a.pfb>',\n",
       " 'Output written on main.pdf (22 pages, 523178 bytes).',\n",
       " 'Transcript written on main.log.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texout[-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdffilename = texfile[:-4] + '.pdf'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "landscape False width 6.55in height 4.2in margin 0.1in\n"
     ]
    }
   ],
   "source": [
    "if show_generated_pdf:\n",
    "    # for displaying pdf\n",
    "    from wand.image import Image as WImage\n",
    "    \n",
    "    print('landscape', landscape, 'width', width, 'height', height, 'margin', margin)\n",
    "    #! {pdffilename}\n",
    "    # load first page just for check\n",
    "    i = 0\n",
    "    filename, file_extension = os.path.splitext(pdffilename)\n",
    "    while(1):\n",
    "        try:\n",
    "            filename_to_open = ''.join([filename, file_extension + '[{}]'.format(i)])\n",
    "            #print('filename_to_open', filename_to_open)\n",
    "            img = WImage(filename=filename_to_open)\n",
    "            display(img)\n",
    "            i += 1\n",
    "        except:\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sending message from google mail to kindle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import smtplib\n",
    "\n",
    "from email.mime.application import MIMEApplication\n",
    "from email.mime.multipart import MIMEMultipart\n",
    "\n",
    "msg = MIMEMultipart()\n",
    "pdf_part = MIMEApplication(open(texfile[:-4]+'.pdf', 'rb').read(), _subtype='pdf')\n",
    "pdf_part.add_header('Content-Disposition', 'attachment', filename=arxiv_title_scrubbed+\".pdf\")\n",
    "msg.attach(pdf_part)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if send:\n",
    "    with smtplib.SMTP_SSL('smtp.gmail.com') as server:\n",
    "        server.set_debuglevel(0)\n",
    "        server.ehlo()\n",
    "        #server.starttls()  \n",
    "        server.login(your_gmail, gmailpass)\n",
    "        server.sendmail(your_gmail, kindle_email, msg.as_string())\n",
    "        server.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "349.091px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}