extracttext.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca6b9663",
   "metadata": {},
   "source": [
    "Building the LegalQA dataset by extracting text from pdf files downloaded from: https://www.gov.br/receitafederal/pt-br/centrais-de-conteudo/publicacoes/perguntas-e-respostas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6884fa92",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PyPDF2 import PdfReader\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import re\n",
    "from re import search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3740de15",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.width', None)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fd774498",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create pandas dataframe to store the QAs extracted from pdf files\n",
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8213dce4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./rawdata/ITR2022.pdf has 81 pages\n",
      "we have a total of 226 questions and answers for the ITR 2022 pdf file\n"
     ]
    }
   ],
   "source": [
    "# Reading ITR 2022 pdf file\n",
    "\n",
    "filename = './rawdata/ITR2022.pdf'\n",
    "skipto = 12\n",
    "qa = []\n",
    "    \n",
    "# read file\n",
    "reader = PdfReader(filename)    \n",
    "# get total number of pages\n",
    "num_pages = len(reader.pages)\n",
    "print('File %s has %d pages' % (filename, num_pages))\n",
    "\n",
    "question = ''\n",
    "answer = ''\n",
    "skipline = False\n",
    "stillquestion = False\n",
    "\n",
    "qnum = \"|\".join(\"^%03d —\" % i for i in range(227))\n",
    "\n",
    "for idx in range(skipto, num_pages):\n",
    "    page = reader.pages[idx]\n",
    "    pagetext = page.extract_text()\n",
    "        \n",
    "    lines = pagetext.split('\\n')\n",
    "    lineidx = 0\n",
    "    while lineidx < len(lines):\n",
    "        line = lines[lineidx]\n",
    "        if search(qnum, line):\n",
    "            if question:\n",
    "                qa.append({'question': question, 'answer': answer})\n",
    "            question = line.split('—')[1]\n",
    "            answer = ''\n",
    "            if '?' not in line:\n",
    "                stillquestion = True            \n",
    "        elif 'Retorno ao sumário' in line:\n",
    "            lineidx += 1\n",
    "            if lineidx >= len(lines):\n",
    "                break\n",
    "            line = lines[lineidx]\n",
    "            if search(qnum, line):\n",
    "                continue\n",
    "        elif '?' in line:\n",
    "            question = question + ' ' + line\n",
    "            stillquestion = False\n",
    "        else:\n",
    "            if stillquestion:\n",
    "                question = question + ' ' + line\n",
    "            else:\n",
    "                answer = answer + ' ' + line\n",
    "        lineidx += 1\n",
    "qa.append({'question': question, 'answer': answer})\n",
    "              \n",
    "print('we have a total of %d questions and answers for the ITR 2022 pdf file' % len(qa))    \n",
    "df_temp = pd.DataFrame.from_dict(qa)\n",
    "\n",
    "df = pd.concat([df, df_temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c1907d96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(226, 2)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "07ea1d60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./rawdata/Dirf2023.pdf has 45 pages\n",
      "we have a total of 77 questions and answers for the DIRF 2023 pdf file\n"
     ]
    }
   ],
   "source": [
    "# Reading DIRF 2023 pdf file\n",
    "\n",
    "filename = './rawdata/Dirf2023.pdf'\n",
    "skipto = 7\n",
    "qa = []\n",
    "\n",
    "titles_to_skip = {'1 INFORMAÇÕES GERAIS': \"^(1.1|1.2|1.3) \", \n",
    "                  '2 PRAZO DE ENTREGA': \"^(2.1|2.2|2.3) \", \n",
    "                  '3 DECLARANTES': \"^(3.1|3.2|3.3|3.4) \", \n",
    "                  '4 RENDIMENTOS': \"^(4.1|4.2|4.3) \", \n",
    "                  '5 RENDIMENTOS ISENTOS': \"^(5.1|5.2|5.3|5.4|5.5) \", \n",
    "                  '6 REMESSA PARA O EXTERIOR': \"^(6.1|6.2|6.3) \", \n",
    "                  '7 PREVIDÊNCIA': \"^(7.1|7.2|7.3|7.4) \", \n",
    "                  '8 PLANO PRIVADO DE ASSISTÊNCIA À SAÚDE – COLETIVO EMPRESARIAL': \"^(8.1|8.2|8.3|8.4) \", \n",
    "                  '9 PREENCHIMENTO': \"^(9.1|9.2|9.3|9.4|9.5|9.6|9.7|9.8) \",\n",
    "                  '10 RENDIMENTOS RECEBIDOS ACUMULADAMENTE': \"^(10.1|10.2|10.3|10.4|10.5|10.6) \", \n",
    "                  '11 RENDIMENTOS PAGOS ÀS ENTIDADES IMUNES/ISENTAS – IN RFB 1.234/2012': \"^(11.1|11.2) \",\n",
    "                  '12 COMPROVANTE DE RENDIMENTOS': \"^(12.1|12.2|12.3|12.4|12.5|12.6|12.7) \", \n",
    "                  '13 PROGRAMA GERADOR DA DECLARAÇÃO': \"^(13.1|13.2|13.3|13.4|13.5) \", \n",
    "                  '14 MULTA POR ATRASO NA ENTREGA': \"^(14.1|14.2|14.3) \", \n",
    "                  '15 RETIFICAÇÃO DE DECLARAÇÃO': \"^(15.1|15.2|15.3|15.4|15.5|15.6) \", \n",
    "                  '16 LEIAUTE': \"^(16.1|16.2|16.3) \",\n",
    "                  '17 IMPORTAÇÃO': \"^(17.1|17.2|17.3) \", \n",
    "                  '18 TRANSMISSÃO': \"^(18.1|18.2|18.3|18.4|18.5) \"}\n",
    "\n",
    "    \n",
    "# read file\n",
    "reader = PdfReader(filename)    \n",
    "# get total number of pages\n",
    "num_pages = len(reader.pages)\n",
    "print('File %s has %d pages' % (filename, num_pages))\n",
    "\n",
    "question = ''\n",
    "answer = ''\n",
    "skipline = False\n",
    "stillquestion = False\n",
    "\n",
    "for idx in range(skipto, num_pages):\n",
    "    page = reader.pages[idx]\n",
    "    pagetext = page.extract_text()\n",
    "    lines = pagetext.split('\\n')     \n",
    "    for line in lines:\n",
    "        if line in titles_to_skip.keys():\n",
    "            qnum = titles_to_skip[line]\n",
    "        else:\n",
    "            if search(qnum, line):\n",
    "                if question:\n",
    "                    qa.append({'question': question, 'answer': answer})    \n",
    "                question = [s.strip() for s in re.split(qnum, line)][2]\n",
    "                answer = ''\n",
    "                if '?' not in line:\n",
    "                    stillquestion = True\n",
    "            elif '?' in line:\n",
    "                question = question + ' ' + line\n",
    "                stillquestion = False\n",
    "            else:\n",
    "                if stillquestion:\n",
    "                    question = question + ' ' + line\n",
    "                else:\n",
    "                    answer = answer + ' ' + line\n",
    "qa.append({'question': question, 'answer': answer})  \n",
    "\n",
    "print('we have a total of %d questions and answers for the DIRF 2023 pdf file' % len(qa))    \n",
    "df_temp = pd.DataFrame.from_dict(qa)\n",
    "\n",
    "df = pd.concat([df, df_temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4e2bb4ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(303, 2)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "21550910",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./rawdata/dctfweb.pdf has 19 pages\n",
      "we have a total of 61 questions and answers for the DCTFWeb 2022 pdf file\n"
     ]
    }
   ],
   "source": [
    "# Reading DCTFWeb 2022 pdf file\n",
    "\n",
    "filename = './rawdata/dctfweb.pdf'\n",
    "skipto = 2\n",
    "qa = []\n",
    "\n",
    "titles_to_skip = {'1 – PAGAMENTO: DARF, SALDO A PAGAR, ABATIMENTO, AJUSTE NO SISTAD  ': \"^(1.1|1.2|1.3|1.4|1.5|1.6|1.7|1.8|1.9|1.10|1.11|1.12|1.13|    1.14|1.15|1.16|1.17|    1.18|1.19|1.20|1.21|    1.22|1.23|1.24|1.25) \",\n",
    "                  '2 – INTEGRAÇÃO DA DCTFWEB COM ESOCIAL E EFD -REINF, SUSPENSÃO  ': \"^(2.1|2.2|    2.3|2.4|2.5|2.6|2.7|2.8|2.9|2.10) \",\n",
    "                  '3 – COMPENSAÇÃO E RESTITUIÇÃO  ': \"^(3.1|3.2|3.3|3.4|3.5|3.6|3.7|3.8|3.9|3.10|    3.11|3.12|    3.13|3.14) \", \n",
    "                  '4 – IRRF E FGTS  ': \"^(4.1|4.2|4.3) \", \n",
    "                  '    5 – OUTROS ASSUN TOS RELACIONADOS À DCTFWEB  ': \"^(5.1|5.2|5.3|5.4|5.5|5.6|    5.7|5.8|5.9) \"}\n",
    "    \n",
    "# read file\n",
    "reader = PdfReader(filename)    \n",
    "# get total number of pages\n",
    "num_pages = len(reader.pages)\n",
    "print('File %s has %d pages' % (filename, num_pages))\n",
    "\n",
    "question = ''\n",
    "answer = ''\n",
    "skipline = False\n",
    "stillquestion = False\n",
    "\n",
    "for idx in range(skipto, num_pages):\n",
    "    page = reader.pages[idx]\n",
    "    pagetext = page.extract_text()\n",
    "    lines = pagetext.split('\\n')   \n",
    "    \n",
    "    if idx == 2:\n",
    "        lines = lines[35:]\n",
    "    \n",
    "    lineidx = 0\n",
    "    while lineidx < len(lines):\n",
    "        line = lines[lineidx]\n",
    "        if 'Perguntas e Respostas da DCTFWeb – março /2022' in line:\n",
    "            lineidx += 2\n",
    "            continue\n",
    "        if line in titles_to_skip.keys():\n",
    "            qnum = titles_to_skip[line]\n",
    "        else:                \n",
    "            if search(qnum, line):\n",
    "                if question:\n",
    "                    qa.append({'question': question, 'answer': answer})\n",
    "                question = [s.strip() for s in re.split(qnum, line)][2]\n",
    "                answer = ''\n",
    "                if '?' not in line:\n",
    "                    stillquestion = True            \n",
    "            elif '?' in line:\n",
    "                question = question + ' ' + line\n",
    "                stillquestion = False\n",
    "            elif 'previdenciários pelo sistema SisFIES.' in line:\n",
    "                question = question + ' ' + line\n",
    "                stillquestion = False\n",
    "            else:\n",
    "                if stillquestion:\n",
    "                    question = question + ' ' + line\n",
    "                else:\n",
    "                    answer = answer + ' ' + line\n",
    "        lineidx += 1\n",
    "qa.append({'question': question, 'answer': answer})  \n",
    "\n",
    "print('we have a total of %d questions and answers for the DCTFWeb 2022 pdf file' % len(qa))    \n",
    "df_temp = pd.DataFrame.from_dict(qa)\n",
    "\n",
    "df = pd.concat([df, df_temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c134a4ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(364, 2)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4f001695",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./rawdata/derex.pdf has 8 pages\n",
      "we have a total of 17 questions and answers for the DEREX pdf file\n"
     ]
    }
   ],
   "source": [
    "# Reading DEREX pdf file\n",
    "# QUESTÕES RELACIONADAS À DECLARAÇÃO SOBRE A UTILIZAÇÃO DE RECURSOS EM MOEDA ESTRANGEIRA DECORRENTES DO RECEBIMENTO DE EXPORTAÇÕES (DEREX)\n",
    "\n",
    "filename = './rawdata/derex.pdf'\n",
    "skipto = 0\n",
    "qa = []\n",
    "    \n",
    "# read file\n",
    "reader = PdfReader(filename)    \n",
    "# get total number of pages\n",
    "num_pages = len(reader.pages)\n",
    "print('File %s has %d pages' % (filename, num_pages))\n",
    "\n",
    "question = ''\n",
    "answer = ''\n",
    "skipline = False\n",
    "stillquestion = True\n",
    "\n",
    "qnum = \"|\".join(\"^%1d. [A-Z]\" % i for i in range(18))\n",
    "list_nums = \"|\".join(\"^%1d. \" % i for i in range(18))\n",
    "\n",
    "for idx in range(skipto, num_pages):\n",
    "    page = reader.pages[idx]\n",
    "    pagetext = page.extract_text()\n",
    "    \n",
    "    pagetext = pagetext[3:]\n",
    "    lines = pagetext.split('\\n')\n",
    "    if idx == 0:\n",
    "        lines = lines[13:]  \n",
    "        \n",
    "    for line in lines:\n",
    "        if search(qnum, line):\n",
    "            if question:\n",
    "                qa.append({'question': question, 'answer': answer})\n",
    "            question = [s.strip() for s in re.split(list_nums, line)][1]\n",
    "            answer = ''\n",
    "            stillquestion = True\n",
    "        elif search('^R.:', line):\n",
    "            line = line.split('R.:')[1]\n",
    "            answer = answer + ' ' + line\n",
    "            stillquestion = False\n",
    "        else:\n",
    "            if stillquestion:\n",
    "                question = question + ' ' + line\n",
    "            else:\n",
    "                answer = answer + ' ' + line\n",
    "qa.append({'question': question, 'answer': answer})\n",
    "\n",
    "print('we have a total of %d questions and answers for the DEREX pdf file' % len(qa))    \n",
    "df_temp = pd.DataFrame.from_dict(qa)\n",
    "\n",
    "df = pd.concat([df, df_temp])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "7bb3ec5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(381, 2)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "337508c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ./rawdata/salex.pdf has 4 pages\n",
      "we have a total of 15 questions and answers for the salex pdf file\n"
     ]
    }
   ],
   "source": [
    "# Reading salex pdf file\n",
    "# QUESTÕES RELACIONADAS À COMPROVAÇÃO DO INGRESSO DOS RECURSOS PROVENIENTES DAS EXPORTAÇÕES (Lei nº 11.371/2006)\n",
    "\n",
    "filename = './rawdata/salex.pdf'\n",
    "skipto = 0\n",
    "qa = []\n",
    "    \n",
    "# read file\n",
    "reader = PdfReader(filename)    \n",
    "# get total number of pages\n",
    "num_pages = len(reader.pages)\n",
    "print('File %s has %d pages' % (filename, num_pages))\n",
    "\n",
    "question = ''\n",
    "answer = ''\n",
    "skipline = False\n",
    "stillquestion = True\n",
    "\n",
    "qnum = \"|\".join(\"^%1d. [A-Z]\" % i for i in range(15))\n",
    "list_nums = \"|\".join(\"^%1d. \" % i for i in range(15))\n",
    "\n",
    "for idx in range(skipto, num_pages):\n",
    "    page = reader.pages[idx]\n",
    "    pagetext = page.extract_text()\n",
    "    \n",
    "#     pagetext = pagetext[3:]\n",
    "    lines = pagetext.split('\\n')\n",
    "    if idx == 0:\n",
    "        lines = lines[10:]     \n",
    "    else:\n",
    "        lines = lines[1:]\n",
    "        \n",
    "    for line in lines:\n",
    "        if search(qnum, line):\n",
    "            if question:\n",
    "                qa.append({'question': question, 'answer': answer})\n",
    "            question = [s.strip() for s in re.split(list_nums, line)][1]\n",
    "            answer = ''\n",
    "            stillquestion = True\n",
    "        elif search('^R.:', line):\n",
    "            line = line.split('R.:')[1]\n",
    "            answer = answer + ' ' + line\n",
    "            stillquestion = False\n",
    "        else:\n",
    "            if stillquestion:\n",
    "                question = question + ' ' + line\n",
    "            else:\n",
    "                answer = answer + ' ' + line              \n",
    "qa.append({'question': question, 'answer': answer})\n",
    "\n",
    "print('we have a total of %d questions and answers for the salex pdf file' % len(qa))    \n",
    "df_temp = pd.DataFrame.from_dict(qa)\n",
    "\n",
    "df = pd.concat([df, df_temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "40877565",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(396, 2)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5984157b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving QA data\n",
    "df.to_csv('LegalQA_dataset.csv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e8e691",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}