Traducao_TED_PT_ENG.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mKw2I57kIcOO"
      },
      "source": [
        "# Modelo de tradução utilizando TED Talks\n",
        "Traduzindo de inglês para português brasileiro, e vice-versa."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7QN3muszIxT8"
      },
      "source": [
        "## Bibliotecas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "78Ii2OB-JTXu",
        "outputId": "f54f6ca7-5c3b-4f51-f67f-1b3b7b67ad5f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     C:\\Users\\User\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "nltk.download('punkt')\n",
        "\n",
        "import torch\n",
        "from torch import optim\n",
        "from torch.utils.data import DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QqopyKbUBTGg",
        "outputId": "0274d891-d154-4d21-f004-5c3607914cd6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: sentencepiece in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (0.1.96)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python39\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (4.17.0)\n",
            "Requirement already satisfied: requests in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: numpy>=1.17 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (1.22.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (2022.3.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (6.0)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python39\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: sacremoses in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (0.0.47)\n",
            "Requirement already satisfied: filelock in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,>=0.11.1 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from transformers) (0.11.6)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: colorama in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from requests->transformers) (3.3)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from requests->transformers) (1.26.8)\n",
            "Requirement already satisfied: click in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from sacremoses->transformers) (8.0.4)\n",
            "Requirement already satisfied: joblib in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from sacremoses->transformers) (1.16.0)\n",
            "Requirement already satisfied: translate-toolkit in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (3.6.0)\n",
            "Requirement already satisfied: lxml>=4.6.3 in c:\\users\\user\\appdata\\local\\programs\\python\\python39\\lib\\site-packages (from translate-toolkit) (4.8.0)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: You are using pip version 21.2.4; however, version 22.0.4 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python39\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        }
      ],
      "source": [
        "!pip3 install sentencepiece\n",
        "!pip3 install transformers\n",
        "!pip3 install translate-toolkit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "u8DRkbdxX0DS"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
        "from translate.storage.tmx import tmxfile"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "debugging = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12SFsRXEJIV_"
      },
      "source": [
        "## Córpus\n",
        "Vamos utilizar um córpus de legendas de TED talks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NN01B86eI-hB",
        "outputId": "a87f6251-3cee-4c4e-fc32-80898ee23801"
      },
      "outputs": [],
      "source": [
        "# ler córpus\n",
        "with open(\"Dados/en-pt_br.tmx\", 'rb') as fin:\n",
        "    f_en2pt_br = tmxfile(fin, 'en', 'pt')\n",
        "    f_pt_br2en = tmxfile(fin, 'pt', 'en')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7LoeCUQJWtI"
      },
      "source": [
        "Separando em conjuntos de treino e teste"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "jc28PjxBJOwx"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total de frases no arquivo: 406821\n"
          ]
        }
      ],
      "source": [
        "prefixo_en2pt_br = '>>pt_br<<'\n",
        "prefixo_pt_br2en = '>>en<<'\n",
        "\n",
        "# formatar as traduções corretamente \n",
        "data_en2pt_br = [\n",
        "                    { 'src': prefixo_en2pt_br + ' ' + w.source, 'trg': w.target } \n",
        "                    for w in f_en2pt_br.unit_iter()\n",
        "                ]\n",
        "\n",
        "data_pt_br2en = [\n",
        "                    { 'src': prefixo_pt_br2en + ' ' + w.target, 'trg': w.source } \n",
        "                    for w in f_pt_br2en.unit_iter()\n",
        "                ]\n",
        "\n",
        "print(\"Total de frases no arquivo: \" + str(len(data_en2pt_br)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "# separar em conjuntos de treino e teste\n",
        "size_en2pt_br = int(len(data_en2pt_br) * 0.2)\n",
        "\n",
        "if debugging:\n",
        "    treino_en2pt_br = data_en2pt_br[size_en2pt_br:][:10000]\n",
        "    teste_en2pt_br = data_en2pt_br[:size_en2pt_br][:1000]\n",
        "\n",
        "    size_pt_br2en = size_en2pt_br\n",
        "    treino_pt_br2en = data_pt_br2en[size_pt_br2en:][:10000]\n",
        "    teste_pt_br2en = data_pt_br2en[:size_pt_br2en][:1000]\n",
        "else:\n",
        "    treino_en2pt_br = data_en2pt_br[size_en2pt_br:]\n",
        "    teste_en2pt_br = data_en2pt_br[:size_en2pt_br]\n",
        "\n",
        "    size_pt_br2en = size_en2pt_br\n",
        "    treino_pt_br2en = data_pt_br2en[size_pt_br2en:]\n",
        "    teste_pt_br2en = data_pt_br2en[:size_pt_br2en]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rL5S2CTGJg6m",
        "outputId": "7cfcdb64-1e92-41cb-a923-99b6af85de25"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'src': \">>pt_br<< And as long as we've looked for explanations, we've wound up with something that gets closer and closer to science, which is hypotheses as to why we get sick, and as long as we've had hypotheses about why we get sick, we've tried to treat it as well. \",\n",
              " 'trg': 'E à medida que procuramos explicações, vamos chegando a conclusões que se aproximam cada vez mais da ciência, que é a hipótese sobre porque adoecemos, e à medida que temos hipóteses sobre porque adoecemos, também procuramos nos tratar. '}"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "treino_en2pt_br[10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SlkhbYLWevpO",
        "outputId": "41e5f2c3-42d0-4a6e-bcc9-396a2532a16d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'src': '>>en<< Essa é a verdade. ', 'trg': \"That's the truth. \"}"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "treino_pt_br2en[4234]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xEIDFHIjJwK5"
      },
      "source": [
        "## Treinamento\n",
        "Definindo parâmetros do modelo e treinamento"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "eZulD4cPJl_Q"
      },
      "outputs": [],
      "source": [
        "learning_rate = 1e-5 \n",
        "epochs = 2\n",
        "batch_size = 16\n",
        "batch_status = 32\n",
        "early_stop = 5\n",
        "token_max_length = 128\n",
        "num_beams = 4\n",
        "write_path_en2pt_br = 'model.pt'\n",
        "write_path_pt_br2en = 'model.en'\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aNKiTB-GKDdp"
      },
      "source": [
        "Separando dados em batches ( lotes )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "8Xz7yuBjKCuY"
      },
      "outputs": [],
      "source": [
        "train_data_en2pt_br = DataLoader(treino_en2pt_br, batch_size = batch_size)\n",
        "dev_data_en2pt_br = DataLoader(teste_en2pt_br, batch_size = batch_size)\n",
        "\n",
        "train_data_pt_br2en = DataLoader(treino_pt_br2en, batch_size = batch_size)\n",
        "dev_data_pt_br2en = DataLoader(teste_pt_br2en, batch_size = batch_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zXrOnb8hKgF6"
      },
      "source": [
        "Método de avaliação"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "-l_Cd8L1KfVC"
      },
      "outputs": [],
      "source": [
        "def evaluate(tokenizer, model, dev_data, batch_status, device):\n",
        "\n",
        "    model.eval()\n",
        "    \n",
        "    y_real = []\n",
        "    y_pred = []\n",
        "    \n",
        "    for batch_idx, inp in enumerate(dev_data):\n",
        "        y_real.extend(inp['trg'])\n",
        "        \n",
        "        # tokenização\n",
        "        model_inputs = tokenizer(\n",
        "            inp['src'], \n",
        "            truncation = True, \n",
        "            padding = True, \n",
        "            max_length = token_max_length, \n",
        "            return_tensors = \"pt\"\n",
        "        ).to(device)\n",
        "        \n",
        "        # tradução\n",
        "        generated_ids = model.generate(**model_inputs, num_beams = num_beams)\n",
        "        \n",
        "        # pós-processamento da tradução\n",
        "        output = tokenizer.batch_decode(generated_ids, skip_special_tokens = True)\n",
        "        y_pred.extend(output)\n",
        "    \n",
        "        # imprime resultados\n",
        "        if (batch_idx + 1) % batch_status == 0:\n",
        "            print(\n",
        "                'Evaluation: [{}/{} ({:.0f}%)]'.format(batch_idx + 1,\n",
        "                len(dev_data), \n",
        "                100. * batch_idx / len(dev_data))\n",
        "            )\n",
        "\n",
        "    # cálculo BLUE score\n",
        "    hyps, refs = [], []\n",
        "    \n",
        "    for i, snt_pred in enumerate(y_pred):\n",
        "        hyps.append(nltk.word_tokenize(snt_pred))\n",
        "        refs.append([nltk.word_tokenize(y_real[i])])\n",
        "    \n",
        "    bleu = corpus_bleu(refs, hyps)\n",
        "\n",
        "    return bleu"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tH2wZ22zLFHY"
      },
      "source": [
        "Método do treinamento"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "ntr4XkyFLEJa"
      },
      "outputs": [],
      "source": [
        "def train(tokenizer, model, train_data, dev_data, optimizer, num_epochs, \n",
        "    batch_status, device, write_path, early_stop = 5):\n",
        "    \n",
        "    max_bleu = evaluate(tokenizer, model, dev_data, batch_status, device)\n",
        "    print('BLEU inicial:', max_bleu)\n",
        "    \n",
        "    model.train()\n",
        "    repeat = 0\n",
        "    \n",
        "    for epoch in range(num_epochs):\n",
        "        losses = []\n",
        "\n",
        "        for batch_idx, inp in enumerate(train_data):\n",
        "            # inicializa zerando o gradiente\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # tokenização\n",
        "            model_inputs = tokenizer(\n",
        "                inp['src'], \n",
        "                truncation = True,\n",
        "                padding = True, \n",
        "                max_length = token_max_length, \n",
        "                return_tensors = \"pt\"\n",
        "            ).to(device)\n",
        "            \n",
        "            with tokenizer.as_target_tokenizer():\n",
        "                labels = tokenizer(\n",
        "                    inp['trg'], \n",
        "                    truncation = True, \n",
        "                    padding = True, \n",
        "                    max_length = token_max_length, \n",
        "                    return_tensors = \"pt\"\n",
        "                ).input_ids.to(device)\n",
        "            \n",
        "            # tradução\n",
        "            output = model(**model_inputs, labels=labels) # forward pass\n",
        "\n",
        "            # cálculo perda\n",
        "            loss = output.loss\n",
        "            losses.append(float(loss))\n",
        "\n",
        "            # backpropagation\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # imprime resultados\n",
        "            if (batch_idx + 1) % batch_status == 0:\n",
        "                print(\n",
        "                    'Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}\\tTotal Loss: {:.6f}'.format(\n",
        "                        epoch, batch_idx + 1, len(train_data), 100. * batch_idx / len(train_data), \n",
        "                        float(loss), round(sum(losses) / len(losses), 5)\n",
        "                    )\n",
        "                )\n",
        "\n",
        "        bleu = evaluate(tokenizer, model, dev_data, batch_status, device)\n",
        "        print('BLEU:', bleu)\n",
        "        \n",
        "        if bleu > max_bleu:\n",
        "            max_bleu = bleu\n",
        "            repeat = 0\n",
        "\n",
        "            print('Saving best model...')\n",
        "            torch.save(model, write_path)\n",
        "        else:\n",
        "            repeat += 1\n",
        "\n",
        "        if repeat == early_stop:\n",
        "            break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5xjqXQTnL8Ie"
      },
      "source": [
        "Inicializando o modelo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "0bwoZUFzL1hb"
      },
      "outputs": [],
      "source": [
        "model = AutoModelForSeq2SeqLM.from_pretrained(\"Helsinki-NLP/opus-mt-en-ROMANCE\").to(device)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"Helsinki-NLP/opus-mt-en-ROMANCE\")\n",
        "optimizer = optim.AdamW(model.parameters(), lr = learning_rate)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hU8U5fT8MCOC"
      },
      "source": [
        "Treinando inglês -> português brasileiro"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-TSBLPyTMCyI",
        "outputId": "b32a5b56-f776-449b-dd38-de5c3c3350d3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluation: [32/63 (49%)]\n",
            "BLEU inicial: 0.4362939916105423\n",
            "Train Epoch: 0 [32/625 (5%)]\tLoss: 0.577310\tTotal Loss: 0.917880\n",
            "Train Epoch: 0 [64/625 (10%)]\tLoss: 0.476916\tTotal Loss: 0.751300\n",
            "Train Epoch: 0 [96/625 (15%)]\tLoss: 0.802500\tTotal Loss: 0.677270\n",
            "Train Epoch: 0 [128/625 (20%)]\tLoss: 0.444657\tTotal Loss: 0.620940\n",
            "Train Epoch: 0 [160/625 (25%)]\tLoss: 0.508787\tTotal Loss: 0.596420\n",
            "Train Epoch: 0 [192/625 (31%)]\tLoss: 0.323029\tTotal Loss: 0.566130\n",
            "Train Epoch: 0 [224/625 (36%)]\tLoss: 0.430378\tTotal Loss: 0.552680\n",
            "Train Epoch: 0 [256/625 (41%)]\tLoss: 0.400554\tTotal Loss: 0.531840\n",
            "Train Epoch: 0 [288/625 (46%)]\tLoss: 0.869685\tTotal Loss: 0.539020\n",
            "Train Epoch: 0 [320/625 (51%)]\tLoss: 0.361840\tTotal Loss: 0.524600\n",
            "Train Epoch: 0 [352/625 (56%)]\tLoss: 0.449872\tTotal Loss: 0.515380\n",
            "Train Epoch: 0 [384/625 (61%)]\tLoss: 0.716345\tTotal Loss: 0.514920\n",
            "Train Epoch: 0 [416/625 (66%)]\tLoss: 0.369736\tTotal Loss: 0.505060\n",
            "Train Epoch: 0 [448/625 (72%)]\tLoss: 0.517114\tTotal Loss: 0.506690\n",
            "Train Epoch: 0 [480/625 (77%)]\tLoss: 0.413180\tTotal Loss: 0.503460\n",
            "Train Epoch: 0 [512/625 (82%)]\tLoss: 0.336314\tTotal Loss: 0.498340\n",
            "Train Epoch: 0 [544/625 (87%)]\tLoss: 0.699533\tTotal Loss: 0.498090\n",
            "Train Epoch: 0 [576/625 (92%)]\tLoss: 0.244417\tTotal Loss: 0.498890\n",
            "Train Epoch: 0 [608/625 (97%)]\tLoss: 0.433315\tTotal Loss: 0.495190\n",
            "Evaluation: [32/63 (49%)]\n",
            "BLEU: 0.43564703158911067\n",
            "Train Epoch: 1 [32/625 (5%)]\tLoss: 0.420925\tTotal Loss: 0.395650\n",
            "Train Epoch: 1 [64/625 (10%)]\tLoss: 0.374578\tTotal Loss: 0.431490\n",
            "Train Epoch: 1 [96/625 (15%)]\tLoss: 0.701509\tTotal Loss: 0.432300\n",
            "Train Epoch: 1 [128/625 (20%)]\tLoss: 0.371585\tTotal Loss: 0.417680\n",
            "Train Epoch: 1 [160/625 (25%)]\tLoss: 0.426353\tTotal Loss: 0.415970\n",
            "Train Epoch: 1 [192/625 (31%)]\tLoss: 0.264726\tTotal Loss: 0.403860\n",
            "Train Epoch: 1 [224/625 (36%)]\tLoss: 0.367941\tTotal Loss: 0.401620\n",
            "Train Epoch: 1 [256/625 (41%)]\tLoss: 0.306234\tTotal Loss: 0.391210\n",
            "Train Epoch: 1 [288/625 (46%)]\tLoss: 0.779365\tTotal Loss: 0.404730\n",
            "Train Epoch: 1 [320/625 (51%)]\tLoss: 0.295638\tTotal Loss: 0.397340\n",
            "Train Epoch: 1 [352/625 (56%)]\tLoss: 0.393875\tTotal Loss: 0.393370\n",
            "Train Epoch: 1 [384/625 (61%)]\tLoss: 0.605799\tTotal Loss: 0.396400\n",
            "Train Epoch: 1 [416/625 (66%)]\tLoss: 0.323518\tTotal Loss: 0.391010\n",
            "Train Epoch: 1 [448/625 (72%)]\tLoss: 0.449785\tTotal Loss: 0.394330\n",
            "Train Epoch: 1 [480/625 (77%)]\tLoss: 0.355469\tTotal Loss: 0.393330\n",
            "Train Epoch: 1 [512/625 (82%)]\tLoss: 0.275417\tTotal Loss: 0.390970\n",
            "Train Epoch: 1 [544/625 (87%)]\tLoss: 0.587643\tTotal Loss: 0.392560\n",
            "Train Epoch: 1 [576/625 (92%)]\tLoss: 0.214912\tTotal Loss: 0.395040\n",
            "Train Epoch: 1 [608/625 (97%)]\tLoss: 0.373571\tTotal Loss: 0.393130\n",
            "Evaluation: [32/63 (49%)]\n",
            "BLEU: 0.4305956938049274\n"
          ]
        }
      ],
      "source": [
        "train(\n",
        "    tokenizer, model, train_data_en2pt_br, dev_data_en2pt_br, optimizer, epochs, \n",
        "    batch_status, device, write_path_en2pt_br, early_stop\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uPna-r8qfdrP"
      },
      "source": [
        "Treinando português brasileiro -> inglês"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zphyZ6affgTk",
        "outputId": "0976f02a-6700-4054-d2cb-771b228d7f17"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluation: [32/63 (49%)]\n",
            "BLEU inicial: 0.00901090337512849\n",
            "Train Epoch: 0 [32/625 (5%)]\tLoss: 1.320086\tTotal Loss: 1.744320\n",
            "Train Epoch: 0 [64/625 (10%)]\tLoss: 1.885685\tTotal Loss: 1.729560\n",
            "Train Epoch: 0 [96/625 (15%)]\tLoss: 1.952566\tTotal Loss: 1.682780\n",
            "Train Epoch: 0 [128/625 (20%)]\tLoss: 1.113552\tTotal Loss: 1.622230\n",
            "Train Epoch: 0 [160/625 (25%)]\tLoss: 1.319821\tTotal Loss: 1.573780\n",
            "Train Epoch: 0 [192/625 (31%)]\tLoss: 1.122351\tTotal Loss: 1.532320\n",
            "Train Epoch: 0 [224/625 (36%)]\tLoss: 1.033884\tTotal Loss: 1.489770\n",
            "Train Epoch: 0 [256/625 (41%)]\tLoss: 1.224562\tTotal Loss: 1.456490\n",
            "Train Epoch: 0 [288/625 (46%)]\tLoss: 1.254629\tTotal Loss: 1.448780\n",
            "Train Epoch: 0 [320/625 (51%)]\tLoss: 1.205522\tTotal Loss: 1.426740\n",
            "Train Epoch: 0 [352/625 (56%)]\tLoss: 1.350949\tTotal Loss: 1.400580\n",
            "Train Epoch: 0 [384/625 (61%)]\tLoss: 1.577502\tTotal Loss: 1.392220\n",
            "Train Epoch: 0 [416/625 (66%)]\tLoss: 1.011594\tTotal Loss: 1.366800\n",
            "Train Epoch: 0 [448/625 (72%)]\tLoss: 1.146112\tTotal Loss: 1.361850\n",
            "Train Epoch: 0 [480/625 (77%)]\tLoss: 0.908751\tTotal Loss: 1.347700\n",
            "Train Epoch: 0 [512/625 (82%)]\tLoss: 0.905041\tTotal Loss: 1.331390\n",
            "Train Epoch: 0 [544/625 (87%)]\tLoss: 1.532053\tTotal Loss: 1.316280\n",
            "Train Epoch: 0 [576/625 (92%)]\tLoss: 0.970179\tTotal Loss: 1.305600\n",
            "Train Epoch: 0 [608/625 (97%)]\tLoss: 0.991235\tTotal Loss: 1.291410\n",
            "Evaluation: [32/63 (49%)]\n",
            "BLEU: 0.1424645501827359\n",
            "Saving best model...\n",
            "Train Epoch: 1 [32/625 (5%)]\tLoss: 0.782068\tTotal Loss: 0.867120\n",
            "Train Epoch: 1 [64/625 (10%)]\tLoss: 1.223467\tTotal Loss: 0.953610\n",
            "Train Epoch: 1 [96/625 (15%)]\tLoss: 1.277266\tTotal Loss: 0.971780\n",
            "Train Epoch: 1 [128/625 (20%)]\tLoss: 0.739013\tTotal Loss: 0.964890\n",
            "Train Epoch: 1 [160/625 (25%)]\tLoss: 0.926103\tTotal Loss: 0.957130\n",
            "Train Epoch: 1 [192/625 (31%)]\tLoss: 0.665606\tTotal Loss: 0.943260\n",
            "Train Epoch: 1 [224/625 (36%)]\tLoss: 0.749537\tTotal Loss: 0.926550\n",
            "Train Epoch: 1 [256/625 (41%)]\tLoss: 0.774580\tTotal Loss: 0.915430\n",
            "Train Epoch: 1 [288/625 (46%)]\tLoss: 0.920531\tTotal Loss: 0.925790\n",
            "Train Epoch: 1 [320/625 (51%)]\tLoss: 0.898335\tTotal Loss: 0.918250\n",
            "Train Epoch: 1 [352/625 (56%)]\tLoss: 0.857436\tTotal Loss: 0.907570\n",
            "Train Epoch: 1 [384/625 (61%)]\tLoss: 1.248049\tTotal Loss: 0.910490\n",
            "Train Epoch: 1 [416/625 (66%)]\tLoss: 0.703451\tTotal Loss: 0.899330\n",
            "Train Epoch: 1 [448/625 (72%)]\tLoss: 0.838361\tTotal Loss: 0.903670\n",
            "Train Epoch: 1 [480/625 (77%)]\tLoss: 0.621707\tTotal Loss: 0.898490\n",
            "Train Epoch: 1 [512/625 (82%)]\tLoss: 0.654384\tTotal Loss: 0.891200\n",
            "Train Epoch: 1 [544/625 (87%)]\tLoss: 1.227236\tTotal Loss: 0.885830\n",
            "Train Epoch: 1 [576/625 (92%)]\tLoss: 0.730086\tTotal Loss: 0.884410\n",
            "Train Epoch: 1 [608/625 (97%)]\tLoss: 0.767613\tTotal Loss: 0.878060\n",
            "Evaluation: [32/63 (49%)]\n",
            "BLEU: 0.1854083253944498\n",
            "Saving best model...\n"
          ]
        }
      ],
      "source": [
        "train(\n",
        "    tokenizer, model, train_data_pt_br2en, dev_data_pt_br2en, optimizer, epochs, \n",
        "    batch_status, device, write_path_pt_br2en, early_stop\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0LLDucgM0lt"
      },
      "source": [
        "## Experimentos"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WQ5mhZVcgBv7"
      },
      "source": [
        "Inglês -> Português brasileiro"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "35zFc-xxMUSp",
        "outputId": "38f56fda-3c2a-4129-cbc7-bd70cd8d34f7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Por favor, não falle comigo agora.',\n",
              " 'Quem é um bom tradutor?',\n",
              " 'Espero que você seja capaz de traduzir uma frase grande, porque as pessoas hoje adoram SMS. E eu quero apresentar isso ao meu professor e colegas, então você tem que trabalhar!',\n",
              " 'Eu realmente não quero estudar hoje, mas eu tenho que fazer isso porque eu quero me formar, arrumar um emprego e ter muito dinheiro.']"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# sentenças a serem traduzidas\n",
        "batch_input_str = (\n",
        "    (\">>pt_br<< Please, don't fail me now.\"), \n",
        "    (\">>pt_br<< Who is a good translator? You are!\"), \n",
        "    (\">>pt_br<< I hope you are able to translate a big sentence, because people nowadays love texting. And I want to present this to my teacher and colleagues, so you have to work!\"),\n",
        "    (\">>pt_br<< I really don't want to study tonight but I have to do it because I want to graduate and get a job and have a lot of money.\")\n",
        ")\n",
        "\n",
        "# tokenizando as sentenças\n",
        "encoded = tokenizer(batch_input_str, return_tensors = 'pt', padding = True).to(device)\n",
        "\n",
        "# traduzindo\n",
        "translated = model.generate(**encoded)\n",
        "\n",
        "# preparando a saída\n",
        "tokenizer.batch_decode(translated, skip_special_tokens = True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0-nbA-GTgCPj"
      },
      "source": [
        "Português brasileiro -> Inglês"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x61pMDuIgJ8g",
        "outputId": "43e81d65-ade9-4e2d-f3ae-2abe32561da0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[\"What is what it's going to work?\",\n",
              " \"It's going to trade that I knowledge!\",\n",
              " 'I think I need to learn you wonder by most time, yes?',\n",
              " \"I know that I should be more creative in my trials, but I don't believe that I can't trade brazilish to English, it's a BLEU's down.\"]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "batch_input_str = (\n",
        "    (\">>en<< Será que isso vai funcionar?\"),\n",
        "    (\">>en<< Teste número 2. Você consegue traduzir isso que eu sei!\"),\n",
        "    (\">>en<< Acho que eu preciso deixar você rodando por mais tempo, né?\"),\n",
        "    (\">>en<< Eu sei que eu deveria ser mais criativo nos meus testes, mas não acredito que consegui traduzir de português brasileiro para inglês, mesmo com um BLEU tão baixo.\")\n",
        ")\n",
        "\n",
        "# tokenizando as sentenças\n",
        "encoded = tokenizer(batch_input_str, return_tensors = 'pt', padding = True).to(device)\n",
        "\n",
        "# traduzindo\n",
        "translated = model.generate(**encoded)\n",
        "\n",
        "# preparando a saída\n",
        "tokenizer.batch_decode(translated, skip_special_tokens = True)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Traducao TED PT ENG",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}