transfer_learning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DlAfI8mCWAf3"
      },
      "source": [
        "<center>\n",
        "<h2> </h2>\n",
        "<h2>ALTeGraD 2024<br>Lab Session 2: Transfer learning for NLP</h2> 15 / 10 / 2024<br> Dr. G. Shang, Y.Zhang<br><br>\n",
        "\n",
        "\n",
        "<b>Student name:</b> Rachida Saroui\n",
        "\n",
        "</center>\n",
        "\n",
        "<br><br>\n",
        "In this lab we will:\n",
        "* Implement and pretrain a language model with transformer architecture.\n",
        "* Use the pretrained model (transfer learning) to perform a sentiment analysis task which consists of classifying some books reviews into positive and negative ones.\n",
        "* Compare the performance of the pretrained model to a model trained from scratch.\n",
        " <br>\n",
        "\n",
        "<b>The deadline for this lab is October 22, 2024 11:59 PM.</b> More details about the submission and the architecture for this lab can be found in the handout PDF."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "IqukuIe0Rb_c"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5FF6fjkqgN39"
      },
      "source": [
        "### The Model"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerModel(nn.Module):\n",
        "    def __init__(self, ntoken, nhead, nhid, nlayers, dropout=0.5):\n",
        "        super(TransformerModel, self).__init__()\n",
        "        \"\"\"\n",
        "        ntokens: the size of vocabulary\n",
        "        nhid: the hidden dimension of the model.\n",
        "        We assume that embedding_dim = nhid\n",
        "        nlayers: the number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
        "        nhead: the number of heads in the multiheadattention models\n",
        "        dropout: the dropout value\n",
        "         \"\"\"\n",
        "        self.model_type = \"Transformer\"\n",
        "        self.encoder = nn.Embedding(\n",
        "            num_embeddings=ntoken, embedding_dim=nhid\n",
        "        )  # fill me, nhid = the dim_embed\n",
        "        self.pos_encoder = PositionalEncoding(\n",
        "            nhid\n",
        "        )  # fill me, the PositionalEncoding class is implemented in the next cell\n",
        "        encoder_layers = nn.TransformerEncoderLayer(\n",
        "            d_model=nhid, nhead=nhead, dim_feedforward=nhid\n",
        "        )  # fill me we assume nhid = d_model = dim_feedforward\n",
        "        self.transformer_encoder = nn.TransformerEncoder(\n",
        "            encoder_layers, num_layers=nlayers\n",
        "        )  # fill me\n",
        "        self.nhid = nhid\n",
        "        self.init_weights()\n",
        "\n",
        "    def generate_square_subsequent_mask(self, sz):\n",
        "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
        "        mask = (\n",
        "            mask.float()\n",
        "            .masked_fill(mask == 0, float(\"-inf\"))\n",
        "            .masked_fill(mask == 1, float(0.0))\n",
        "        )\n",
        "        return mask\n",
        "\n",
        "    def init_weights(self):\n",
        "        initrange = 0.1\n",
        "        self.encoder.weight.data.uniform_(-initrange, initrange)\n",
        "\n",
        "    def forward(self, src, src_mask):\n",
        "        src = self.encoder(src) * math.sqrt(self.nhid)\n",
        "        src = self.pos_encoder(src)  # fill me\n",
        "        output = self.transformer_encoder(src, src_mask)  # fill me\n",
        "        return output\n",
        "\n",
        "\n",
        "class ClassificationHead(nn.Module):\n",
        "    def __init__(self, nhid, nclasses):\n",
        "        super(ClassificationHead, self).__init__()\n",
        "        self.decoder = nn.Linear(nhid, nclasses)  # fill me\n",
        "        self.init_weights()\n",
        "\n",
        "    def init_weights(self):\n",
        "        initrange = 0.1\n",
        "        self.decoder.bias.data.zero_()\n",
        "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
        "\n",
        "    def forward(self, src):\n",
        "        output = self.decoder(src)\n",
        "        return output\n",
        "\n",
        "\n",
        "class Model(nn.Module):\n",
        "    def __init__(self, ntoken, nhead, nhid, nlayers, nclasses, dropout=0.5):\n",
        "        super(Model, self).__init__()\n",
        "        self.base = TransformerModel(ntoken, nhead, nhid, nlayers, dropout)\n",
        "        self.classifier = ClassificationHead(nhid, nclasses)\n",
        "\n",
        "    def forward(self, src, src_mask):\n",
        "        # base model\n",
        "        x = self.base(src, src_mask)\n",
        "        # classifier model\n",
        "        output = self.classifier(x)\n",
        "        return output"
      ],
      "metadata": {
        "id": "8YUQ4P0FgYWl"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "kt2QQohaFZry"
      },
      "outputs": [],
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, nhid, dropout=0.1, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "\n",
        "        pe = torch.zeros(max_len, nhid)\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(\n",
        "            torch.arange(0, nhid, 2).float() * (-math.log(10000.0) / nhid)\n",
        "        )\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer(\"pe\", pe)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + self.pe[: x.size(0), :]\n",
        "        return self.dropout(x)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SfEYHJx2JW6l"
      },
      "source": [
        "Let's verify if our model works, by applying one inference step"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "rhb2gkUhJMR0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4cea3054-97ba-442c-87a9-56320f2711fd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/transformer.py:307: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
            "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([1, 6, 100])\n"
          ]
        }
      ],
      "source": [
        "ntokens = 100 # the size of vocabulary\n",
        "nhid = 200  # hidden dimension\n",
        "nlayers = 4  # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
        "nhead = 2  # the number of heads in the multiheadattention models\n",
        "dropout = 0  # the dropout value\n",
        "\n",
        "model = Model(ntokens, nhead, nhid, nlayers, ntokens, dropout).to(device)\n",
        "dummy_input = torch.tensor([[2, 6, 2, 5, 43, 21]]).to(device)\n",
        "src_mask = model.base.generate_square_subsequent_mask(1).to(device)\n",
        "out = model.forward(dummy_input, src_mask)\n",
        "\n",
        "print(out.shape) # is it the right shape?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i74NN897Fcit"
      },
      "source": [
        "## Vocabulary and Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "5qjd26ghWuff",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f426890f-442c-4eef-8544-e68a17aba8f4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-10-22 19:27:52--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/dict.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 577587 (564K) [text/plain]\n",
            "Saving to: ‘dict.txt’\n",
            "\n",
            "dict.txt            100%[===================>] 564.05K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2024-10-22 19:27:53 (15.1 MB/s) - ‘dict.txt’ saved [577587/577587]\n",
            "\n",
            "▁d 1\n",
            "es 1\n",
            "▁l 1\n",
            "en 1\n",
            "on 1\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/dict.txt\n",
        "!head -5 dict.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "vFdH_-JeFbGA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "374afbf4-d4f3-4809-b225-41e361e5e7c4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "▁trop\n"
          ]
        }
      ],
      "source": [
        "path_vocab = \"dict.txt\"\n",
        "token2ind = {\"<sos>\": 0, \"<pad>\": 1, \"<eos>\": 2, \"<oov>\": 3} # the 4 first indices are reserved to special tokens\n",
        "with open(path_vocab, \"r\") as f:\n",
        "    for idx, line in enumerate(f):\n",
        "        word = line.split()[0].strip()\n",
        "        token2ind[word] = idx + 4 #fill me\n",
        "\n",
        "ind2token = {idx: token for token, idx in token2ind.items()} #fill me\n",
        "\n",
        "print(ind2token[1111])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XOExGODajN8p"
      },
      "source": [
        "### Data Loader\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy\n",
        "import torch\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "\n",
        "\n",
        "class Dataset(Dataset):\n",
        "    def __init__(\n",
        "        self,\n",
        "        path_documents,\n",
        "        path_labels=None,\n",
        "        token2ind={},\n",
        "        max_len=512,\n",
        "        task=\"language_modeling\",\n",
        "    ):\n",
        "        self.task = task\n",
        "        self.max_len = max_len\n",
        "        self.token2ind = token2ind\n",
        "        self.documents = []\n",
        "        self.labels = []\n",
        "        with open(path_documents, \"r\") as f1:\n",
        "            for line in f1:\n",
        "                self.documents.append(line.strip())\n",
        "        if task == \"classification\":\n",
        "            with open(path_labels, \"r\") as f1:\n",
        "                for line in f1:\n",
        "                    self.labels.append(int(line.strip()))\n",
        "            assert len(self.labels) == len(self.documents)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.documents)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        sequence = self.documents[index].split()\n",
        "        if len(sequence) > self.max_len - 1:\n",
        "            sequence = sequence[: self.max_len - 1]\n",
        "        source_sequence = [self.token2ind[\"<sos>\"]] + [\n",
        "            self.token2ind.get(word, self.token2ind[\"<oov>\"]) for word in sequence\n",
        "        ]  # fill me (constract the input sequence using token2ind, sequence and special tokens)\n",
        "        if self.task == \"language_modeling\":\n",
        "            target = source_sequence[1:]\n",
        "            target.append(self.token2ind[\"<eos>\"])\n",
        "        elif self.task == \"classification\":\n",
        "            target = [self.labels[index]]\n",
        "        sample = {\n",
        "            \"source_sequence\": torch.tensor(source_sequence),\n",
        "            \"target\": torch.tensor(target),\n",
        "        }\n",
        "        return sample\n",
        "\n",
        "\n",
        "def MyCollator(batch):\n",
        "    source_sequences = pad_sequence(\n",
        "        # we use padding to match the length of the sequences in the same batch\n",
        "        [sample[\"source_sequence\"] for sample in batch],\n",
        "        padding_value=token2ind[\"<pad>\"],\n",
        "    )\n",
        "    target = pad_sequence(\n",
        "        [sample[\"target\"] for sample in batch], padding_value=token2ind[\"<pad>\"]\n",
        "    )\n",
        "    return source_sequences, target.reshape(-1)\n",
        "\n",
        "\n",
        "def get_loader(\n",
        "    path_documents,\n",
        "    path_labels=None,\n",
        "    token2ind={},\n",
        "    max_len=512,\n",
        "    batch_size=32,\n",
        "    task=\"language_modeling\",\n",
        "):\n",
        "    dataset = Dataset(\n",
        "        path_documents,\n",
        "        path_labels=path_labels,\n",
        "        token2ind=token2ind,\n",
        "        max_len=512,\n",
        "        task=task,\n",
        "    )\n",
        "    data_loader = DataLoader(\n",
        "        dataset=dataset,\n",
        "        batch_size=batch_size,\n",
        "        shuffle=True,\n",
        "        collate_fn=MyCollator,\n",
        "        pin_memory=True,\n",
        "        drop_last=True,\n",
        "    )\n",
        "    return data_loader\n"
      ],
      "metadata": {
        "id": "gNr-p1cugBAu"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uTns4lHrjUTa"
      },
      "source": [
        "## The Training"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def train(\n",
        "    path_data_train,\n",
        "    path_labels_train=None,\n",
        "    path_data_valid=None,\n",
        "    save_interval=-1,\n",
        "    log_interval=5,\n",
        "    task=\"language_modeling\",\n",
        "    batch_size=32,\n",
        "):\n",
        "    model.train()\n",
        "    total_loss = 0.0\n",
        "    ntokens = len(token2ind)\n",
        "    data_loader = get_loader(\n",
        "        path_data_train,\n",
        "        path_labels_train,\n",
        "        token2ind,\n",
        "        task=task,\n",
        "        batch_size=batch_size,\n",
        "    )\n",
        "\n",
        "    losses = []\n",
        "    for idx, data in enumerate(data_loader):  # step 1\n",
        "        optimizer.zero_grad()\n",
        "        src_mask = model.base.generate_square_subsequent_mask(data[0].size(0)).to(\n",
        "            device\n",
        "        )\n",
        "        input = data[0].to(device)\n",
        "        output = model(input, src_mask)  # step 2\n",
        "        if task == \"classification\":\n",
        "            # last vector only\n",
        "            output = output[-1, :]  # fill me\n",
        "        output = output.view(-1, output.shape[-1])\n",
        "        target = data[1]  # fill me\n",
        "        target = target.to(device)\n",
        "        loss = criterion(output, target)  # fill me, Cross entropy check next cells\n",
        "        # fill me step 3\n",
        "        loss.backward()\n",
        "\n",
        "        torch.nn.utils.clip_grad_norm_(\n",
        "            model.parameters(), 0.5\n",
        "        )  # prevent exploding gradient\n",
        "        # fill me step 4\n",
        "        optimizer.step()\n",
        "\n",
        "        total_loss += loss.item()\n",
        "        if idx % log_interval == 0 and idx > 0:\n",
        "            cur_loss = total_loss / log_interval\n",
        "            print(\n",
        "                \"| epoch {:3d} | {:5d}/{:5d} steps | \"\n",
        "                \"loss {:5.5f} | ppl {:8.3f}\".format(\n",
        "                    epoch,\n",
        "                    idx,\n",
        "                    len(data_loader),\n",
        "                    cur_loss,\n",
        "                    math.exp(cur_loss),\n",
        "                )\n",
        "            )\n",
        "            losses.append(cur_loss)\n",
        "            total_loss = 0\n",
        "    return losses"
      ],
      "metadata": {
        "id": "HvKFzQ8FfY0W"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "pgf6BDB9jUr6"
      },
      "outputs": [],
      "source": [
        "ntokens =len(token2ind) #fill me # the size of vocabulary\n",
        "nhid = 200  # the dimension of the feedforward network model in nn.TransformerEncoder\n",
        "nlayers = 4  # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
        "nhead = 2  # the number of heads in the multiheadattention models\n",
        "dropout = 0  # the dropout value\n",
        "\n",
        "nclasses = 2 # for classification task only\n",
        "\n",
        "model = Model(ntokens, nhead, nhid, nlayers, ntokens, dropout).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "u-OLy4KIkDwf"
      },
      "outputs": [],
      "source": [
        "# optimization paramerters\n",
        "\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=token2ind['<pad>'])\n",
        "lr = 0.0003  # learning rate\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Bwh3n9xZQy4e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "94953a00-c4c1-4241-d40f-142a4894b43e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-10-22 19:28:16--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/pretraining_subset.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10146460 (9.7M) [text/plain]\n",
            "Saving to: ‘pretraining_subset.txt’\n",
            "\n",
            "pretraining_subset. 100%[===================>]   9.68M  --.-KB/s    in 0.08s   \n",
            "\n",
            "2024-10-22 19:28:17 (126 MB/s) - ‘pretraining_subset.txt’ saved [10146460/10146460]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/pretraining_subset.txt\n",
        "path_data_train = \"pretraining_subset.txt\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "0m11g4ScjZaR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7e56beb-b10a-459a-82a2-c7431e808021"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "| epoch   1 |   500/ 3125 steps | loss 7.22934 | ppl 1379.318\n",
            "| epoch   1 |  1000/ 3125 steps | loss 6.56601 | ppl  710.532\n",
            "| epoch   1 |  1500/ 3125 steps | loss 6.29263 | ppl  540.572\n",
            "| epoch   1 |  2000/ 3125 steps | loss 6.11371 | ppl  452.011\n",
            "| epoch   1 |  2500/ 3125 steps | loss 6.04023 | ppl  419.990\n",
            "| epoch   1 |  3000/ 3125 steps | loss 5.92588 | ppl  374.610\n",
            "| epoch   2 |   500/ 3125 steps | loss 5.64558 | ppl  283.037\n",
            "| epoch   2 |  1000/ 3125 steps | loss 5.62259 | ppl  276.604\n",
            "| epoch   2 |  1500/ 3125 steps | loss 5.59374 | ppl  268.740\n",
            "| epoch   2 |  2000/ 3125 steps | loss 5.53924 | ppl  254.486\n",
            "| epoch   2 |  2500/ 3125 steps | loss 5.52228 | ppl  250.205\n",
            "| epoch   2 |  3000/ 3125 steps | loss 5.47615 | ppl  238.925\n"
          ]
        }
      ],
      "source": [
        "#pretraining on a tiny subset\n",
        "log_interval = 500\n",
        "epochs = 2\n",
        "for epoch in range(1, epochs + 1): #5\n",
        "    train(\n",
        "        path_data_train,\n",
        "        save_interval=-1,\n",
        "        task=\"language_modeling\", # fill me\n",
        "        #task=\"classification\",\n",
        "        batch_size=16,\n",
        "        log_interval=log_interval,\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MeOM1dOvkO4e"
      },
      "source": [
        "## Text Generation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "-BcBC6FSkMH3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54d8af92-be40-4b31-884f-eb56c809c9a3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-10-22 19:35:44--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/pretrained_model_4layers.pt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 88093955 (84M) [application/octet-stream]\n",
            "Saving to: ‘pretrained_model_4layers.pt’\n",
            "\n",
            "pretrained_model_4l 100%[===================>]  84.01M   264MB/s    in 0.3s    \n",
            "\n",
            "2024-10-22 19:35:44 (264 MB/s) - ‘pretrained_model_4layers.pt’ saved [88093955/88093955]\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/transformer.py:307: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
            "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n",
            "<ipython-input-15-76a40ce8b80e>:6: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  checkpoint = torch.load('pretrained_model_4layers.pt')\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/pretrained_model_4layers.pt\n",
        "\n",
        "model = Model(ntokens, nhead, nhid, nlayers, ntokens).to(device)\n",
        "\n",
        "#load the checkpoint\n",
        "checkpoint = torch.load('pretrained_model_4layers.pt')\n",
        "#load state dict\n",
        "model.load_state_dict(checkpoint['model_state_dict'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "tBRRVsWqlIoQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "926f34ce-7dee-4da9-ee64-7bbe16803b8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-10-22 19:35:49--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/sentencepiece.french.model\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1115362 (1.1M) [application/octet-stream]\n",
            "Saving to: ‘sentencepiece.french.model’\n",
            "\n",
            "\r          sentencep   0%[                    ]       0  --.-KB/s               \rsentencepiece.frenc 100%[===================>]   1.06M  --.-KB/s    in 0.04s   \n",
            "\n",
            "2024-10-22 19:35:50 (25.6 MB/s) - ‘sentencepiece.french.model’ saved [1115362/1115362]\n",
            "\n",
            "['▁Bonjour', '▁les', '▁amis', '!']\n",
            "Bonjour les amis!\n"
          ]
        }
      ],
      "source": [
        "# !pip install sentencepiece   # uncomment this if you are using google colab\n",
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/sentencepiece.french.model\n",
        "\n",
        "import sentencepiece as spm\n",
        "\n",
        "s = spm.SentencePieceProcessor(model_file='sentencepiece.french.model') #load sentencepiece model\n",
        "\n",
        "#examples\n",
        "encoded = s.encode_as_pieces(\"Bonjour les amis!\")\n",
        "decoded = s.decode_pieces(encoded)\n",
        "print(encoded)\n",
        "print(decoded)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "TtLlV05pkQI3"
      },
      "outputs": [],
      "source": [
        "def infer_next_token(sent):\n",
        "    model.eval()\n",
        "    sent_pieces = s.encode_as_pieces(sent)\n",
        "    source = [token2ind['<sos>']] + [token2ind[el] for el in sent_pieces] # list of tokens\n",
        "    source = torch.tensor(source).to(device)\n",
        "    source = source.reshape(-1, 1)\n",
        "    src_mask = model.base.generate_square_subsequent_mask(source.size(0)).to(device)\n",
        "    out = model(source, src_mask)\n",
        "    next_token_ind =torch.argmax(out, axis=-1)[-1, 0]  #fill me\n",
        "    return next_token_ind, out\n",
        "\n",
        "def infer_next_tokens(sent, max_len=50):\n",
        "    s = spm.SentencePieceProcessor(model_file=\"sentencepiece.french.model\")\n",
        "    sent_pieces = s.encode_as_pieces(sent)\n",
        "    while len(sent_pieces) < max_len:\n",
        "        new_token = infer_next_token(s.decode_pieces(sent_pieces))[0].item()\n",
        "        if new_token == token2ind[\"<eos>\"]:\n",
        "            break\n",
        "        sent_pieces += ind2token[new_token]\n",
        "    return s.decode_pieces(sent_pieces)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "f83Nn5nSly4v",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "cb7d9c35-4abf-4347-d844-84081712c679"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Bonjour les gens qui ont été très accueillants et sympathiques'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "sent = \"Bonjour les\"\n",
        "infer_next_tokens(sent)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lp7mjVzomoZ3"
      },
      "source": [
        "### Supervised task"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "0K1BZsblmEmx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4669c5a-937e-45f8-817e-f7f3f4eaba25"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-10-22 19:36:00--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/train.review.spm\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1495960 (1.4M) [text/plain]\n",
            "Saving to: ‘train.review.spm’\n",
            "\n",
            "train.review.spm    100%[===================>]   1.43M  --.-KB/s    in 0.04s   \n",
            "\n",
            "2024-10-22 19:36:00 (35.0 MB/s) - ‘train.review.spm’ saved [1495960/1495960]\n",
            "\n",
            "--2024-10-22 19:36:00--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/train.label\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3200 (3.1K) [text/plain]\n",
            "Saving to: ‘train.label’\n",
            "\n",
            "train.label         100%[===================>]   3.12K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-10-22 19:36:00 (50.5 MB/s) - ‘train.label’ saved [3200/3200]\n",
            "\n",
            "--2024-10-22 19:36:00--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/test.review.spm\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1864544 (1.8M) [text/plain]\n",
            "Saving to: ‘test.review.spm’\n",
            "\n",
            "test.review.spm     100%[===================>]   1.78M  --.-KB/s    in 0.05s   \n",
            "\n",
            "2024-10-22 19:36:01 (33.1 MB/s) - ‘test.review.spm’ saved [1864544/1864544]\n",
            "\n",
            "--2024-10-22 19:36:01--  https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/test.label\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4000 (3.9K) [text/plain]\n",
            "Saving to: ‘test.label’\n",
            "\n",
            "test.label          100%[===================>]   3.91K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-10-22 19:36:01 (58.8 MB/s) - ‘test.label’ saved [4000/4000]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/train.review.spm\n",
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/train.label\n",
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/test.review.spm\n",
        "!wget https://raw.githubusercontent.com/moussaKam/transfer_learning_transformers/main/cls-books/test.label\n",
        "\n",
        "path_data_train = \"train.review.spm\"\n",
        "path_labels_train = \"train.label\"\n",
        "\n",
        "path_data_valid = \"test.review.spm\"\n",
        "path_labels_valid = \"test.label\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "_MLfvjiom2SL"
      },
      "outputs": [],
      "source": [
        "# a function to evaluate the validation accuracy of the model.\n",
        "def evaluate_accuracy(data_loader):\n",
        "    correct_samples = 0\n",
        "    total_samples = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for idx, data in enumerate(data_loader):  # step 1\n",
        "            src_mask = model.base.generate_square_subsequent_mask(data[0].size(0)).to(\n",
        "                device\n",
        "            )\n",
        "            input = data[0].to(device)\n",
        "            outputs = model(input, src_mask)  # step 2\n",
        "            predictions = torch.argmax(outputs[-1, :], axis=-1)\n",
        "            targets = data[1].to(device)\n",
        "            total_samples += targets.shape[0]\n",
        "            correct_samples += torch.sum(targets == predictions)\n",
        "    accuracy = correct_samples / total_samples\n",
        "    print(f\"Accuracy = {accuracy}\")\n",
        "    return accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "qzmx7T7xoa6v"
      },
      "outputs": [],
      "source": [
        "#save the base model to be loaded later in the fine-tuning phase\n",
        "torch.save({\"model_state_dict\": model.base.state_dict(),}, \"pretrained_model_4layers_no_class_head.pt\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "i-xclMCpnVpw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7ebadca-bf9a-40f5-fef1-02c98f9df3ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=====Trainig FROM SCRATCH======\n",
            "| epoch   1 |    50/  200 steps | loss 0.79098 | ppl    2.206\n",
            "| epoch   1 |   100/  200 steps | loss 0.72358 | ppl    2.062\n",
            "| epoch   1 |   150/  200 steps | loss 0.69674 | ppl    2.007\n",
            "Accuracy = 0.5100000500679016\n",
            "| epoch   2 |    50/  200 steps | loss 0.72484 | ppl    2.064\n",
            "| epoch   2 |   100/  200 steps | loss 0.69810 | ppl    2.010\n",
            "| epoch   2 |   150/  200 steps | loss 0.64816 | ppl    1.912\n",
            "Accuracy = 0.6665000319480896\n",
            "| epoch   3 |    50/  200 steps | loss 0.56914 | ppl    1.767\n",
            "| epoch   3 |   100/  200 steps | loss 0.49637 | ppl    1.643\n",
            "| epoch   3 |   150/  200 steps | loss 0.47815 | ppl    1.613\n",
            "Accuracy = 0.7490000128746033\n",
            "| epoch   4 |    50/  200 steps | loss 0.24863 | ppl    1.282\n",
            "| epoch   4 |   100/  200 steps | loss 0.27545 | ppl    1.317\n",
            "| epoch   4 |   150/  200 steps | loss 0.39427 | ppl    1.483\n",
            "Accuracy = 0.7705000638961792\n",
            "| epoch   5 |    50/  200 steps | loss 0.13701 | ppl    1.147\n",
            "| epoch   5 |   100/  200 steps | loss 0.20282 | ppl    1.225\n",
            "| epoch   5 |   150/  200 steps | loss 0.10675 | ppl    1.113\n",
            "Accuracy = 0.7560000419616699\n",
            "| epoch   6 |    50/  200 steps | loss 0.09399 | ppl    1.099\n",
            "| epoch   6 |   100/  200 steps | loss 0.06530 | ppl    1.067\n",
            "| epoch   6 |   150/  200 steps | loss 0.03335 | ppl    1.034\n",
            "Accuracy = 0.7505000233650208\n",
            "| epoch   7 |    50/  200 steps | loss 0.03215 | ppl    1.033\n",
            "| epoch   7 |   100/  200 steps | loss 0.00191 | ppl    1.002\n",
            "| epoch   7 |   150/  200 steps | loss 0.07425 | ppl    1.077\n",
            "Accuracy = 0.7590000629425049\n",
            "| epoch   8 |    50/  200 steps | loss 0.00364 | ppl    1.004\n",
            "| epoch   8 |   100/  200 steps | loss 0.02487 | ppl    1.025\n",
            "| epoch   8 |   150/  200 steps | loss 0.01004 | ppl    1.010\n",
            "Accuracy = 0.7475000619888306\n",
            "| epoch   9 |    50/  200 steps | loss 0.01043 | ppl    1.010\n",
            "| epoch   9 |   100/  200 steps | loss 0.02853 | ppl    1.029\n",
            "| epoch   9 |   150/  200 steps | loss 0.00056 | ppl    1.001\n",
            "Accuracy = 0.7585000395774841\n",
            "| epoch  10 |    50/  200 steps | loss 0.00183 | ppl    1.002\n",
            "| epoch  10 |   100/  200 steps | loss 0.01456 | ppl    1.015\n",
            "| epoch  10 |   150/  200 steps | loss 0.00014 | ppl    1.000\n",
            "Accuracy = 0.7565000653266907\n",
            "| epoch  11 |    50/  200 steps | loss 0.01491 | ppl    1.015\n",
            "| epoch  11 |   100/  200 steps | loss 0.00449 | ppl    1.004\n",
            "| epoch  11 |   150/  200 steps | loss 0.02067 | ppl    1.021\n",
            "Accuracy = 0.7515000104904175\n",
            "| epoch  12 |    50/  200 steps | loss 0.00196 | ppl    1.002\n",
            "| epoch  12 |   100/  200 steps | loss 0.00910 | ppl    1.009\n",
            "| epoch  12 |   150/  200 steps | loss 0.01443 | ppl    1.015\n",
            "Accuracy = 0.76500004529953\n",
            "| epoch  13 |    50/  200 steps | loss 0.00001 | ppl    1.000\n",
            "| epoch  13 |   100/  200 steps | loss 0.00646 | ppl    1.006\n",
            "| epoch  13 |   150/  200 steps | loss 0.00002 | ppl    1.000\n",
            "Accuracy = 0.7345000505447388\n",
            "| epoch  14 |    50/  200 steps | loss 0.00003 | ppl    1.000\n",
            "| epoch  14 |   100/  200 steps | loss 0.00977 | ppl    1.010\n",
            "| epoch  14 |   150/  200 steps | loss 0.02036 | ppl    1.021\n",
            "Accuracy = 0.7270000576972961\n",
            "| epoch  15 |    50/  200 steps | loss 0.00260 | ppl    1.003\n",
            "| epoch  15 |   100/  200 steps | loss 0.00001 | ppl    1.000\n",
            "| epoch  15 |   150/  200 steps | loss 0.02838 | ppl    1.029\n",
            "Accuracy = 0.7085000276565552\n",
            "\n",
            "=====PRETRAINED MODEL======\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-22-f09076582a3e>:14: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  checkpoint = torch.load(\"pretrained_model_4layers_no_class_head.pt\")\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "| epoch   1 |    50/  200 steps | loss 0.88626 | ppl    2.426\n",
            "| epoch   1 |   100/  200 steps | loss 0.69311 | ppl    2.000\n",
            "| epoch   1 |   150/  200 steps | loss 0.63456 | ppl    1.886\n",
            "Accuracy = 0.768500030040741\n",
            "| epoch   2 |    50/  200 steps | loss 0.52715 | ppl    1.694\n",
            "| epoch   2 |   100/  200 steps | loss 0.43736 | ppl    1.549\n",
            "| epoch   2 |   150/  200 steps | loss 0.48474 | ppl    1.624\n",
            "Accuracy = 0.7050000429153442\n",
            "| epoch   3 |    50/  200 steps | loss 0.42475 | ppl    1.529\n",
            "| epoch   3 |   100/  200 steps | loss 0.44863 | ppl    1.566\n",
            "| epoch   3 |   150/  200 steps | loss 0.33750 | ppl    1.401\n",
            "Accuracy = 0.8020000457763672\n",
            "| epoch   4 |    50/  200 steps | loss 0.28339 | ppl    1.328\n",
            "| epoch   4 |   100/  200 steps | loss 0.35900 | ppl    1.432\n",
            "| epoch   4 |   150/  200 steps | loss 0.32037 | ppl    1.378\n",
            "Accuracy = 0.8045000433921814\n",
            "| epoch   5 |    50/  200 steps | loss 0.25845 | ppl    1.295\n",
            "| epoch   5 |   100/  200 steps | loss 0.22377 | ppl    1.251\n",
            "| epoch   5 |   150/  200 steps | loss 0.35254 | ppl    1.423\n",
            "Accuracy = 0.8135000467300415\n",
            "| epoch   6 |    50/  200 steps | loss 0.16875 | ppl    1.184\n",
            "| epoch   6 |   100/  200 steps | loss 0.18892 | ppl    1.208\n",
            "| epoch   6 |   150/  200 steps | loss 0.20965 | ppl    1.233\n",
            "Accuracy = 0.8030000329017639\n",
            "| epoch   7 |    50/  200 steps | loss 0.15292 | ppl    1.165\n",
            "| epoch   7 |   100/  200 steps | loss 0.13633 | ppl    1.146\n",
            "| epoch   7 |   150/  200 steps | loss 0.19942 | ppl    1.221\n",
            "Accuracy = 0.8015000224113464\n",
            "| epoch   8 |    50/  200 steps | loss 0.05512 | ppl    1.057\n",
            "| epoch   8 |   100/  200 steps | loss 0.14917 | ppl    1.161\n",
            "| epoch   8 |   150/  200 steps | loss 0.15861 | ppl    1.172\n",
            "Accuracy = 0.8075000643730164\n",
            "| epoch   9 |    50/  200 steps | loss 0.06563 | ppl    1.068\n",
            "| epoch   9 |   100/  200 steps | loss 0.17510 | ppl    1.191\n",
            "| epoch   9 |   150/  200 steps | loss 0.04138 | ppl    1.042\n",
            "Accuracy = 0.796500027179718\n",
            "| epoch  10 |    50/  200 steps | loss 0.05312 | ppl    1.055\n",
            "| epoch  10 |   100/  200 steps | loss 0.10170 | ppl    1.107\n",
            "| epoch  10 |   150/  200 steps | loss 0.03531 | ppl    1.036\n",
            "Accuracy = 0.8030000329017639\n",
            "| epoch  11 |    50/  200 steps | loss 0.03446 | ppl    1.035\n",
            "| epoch  11 |   100/  200 steps | loss 0.03606 | ppl    1.037\n",
            "| epoch  11 |   150/  200 steps | loss 0.05724 | ppl    1.059\n",
            "Accuracy = 0.7930000424385071\n",
            "| epoch  12 |    50/  200 steps | loss 0.05137 | ppl    1.053\n",
            "| epoch  12 |   100/  200 steps | loss 0.03410 | ppl    1.035\n",
            "| epoch  12 |   150/  200 steps | loss 0.00426 | ppl    1.004\n",
            "Accuracy = 0.7990000247955322\n",
            "| epoch  13 |    50/  200 steps | loss 0.02307 | ppl    1.023\n",
            "| epoch  13 |   100/  200 steps | loss 0.03584 | ppl    1.036\n",
            "| epoch  13 |   150/  200 steps | loss 0.07184 | ppl    1.074\n",
            "Accuracy = 0.7950000166893005\n",
            "| epoch  14 |    50/  200 steps | loss 0.06521 | ppl    1.067\n",
            "| epoch  14 |   100/  200 steps | loss 0.01259 | ppl    1.013\n",
            "| epoch  14 |   150/  200 steps | loss 0.00893 | ppl    1.009\n",
            "Accuracy = 0.8005000352859497\n",
            "| epoch  15 |    50/  200 steps | loss 0.00053 | ppl    1.001\n",
            "| epoch  15 |   100/  200 steps | loss 0.06551 | ppl    1.068\n",
            "| epoch  15 |   150/  200 steps | loss 0.00032 | ppl    1.000\n",
            "Accuracy = 0.7990000247955322\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from_scratch_settings = [True, False]\n",
        "\n",
        "from_scratch_valid_acc = []\n",
        "pretrained_valid_acc = []\n",
        "lr = 0.0001\n",
        "\n",
        "for from_scratch in from_scratch_settings:\n",
        "    model = Model(ntokens, nhead, nhid, nlayers, 2, dropout).to(device)\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "    if not from_scratch:\n",
        "        print(\"=====PRETRAINED MODEL======\")\n",
        "        #load checkpoint\n",
        "        checkpoint = torch.load(\"pretrained_model_4layers_no_class_head.pt\")\n",
        "        #load state dict\n",
        "        model.base.load_state_dict(checkpoint['model_state_dict'])\n",
        "    else:\n",
        "        print(\"=====Trainig FROM SCRATCH======\")\n",
        "    epochs = 15\n",
        "    for epoch in range(1, epochs + 1):\n",
        "        train(\n",
        "            path_data_train,\n",
        "            path_labels_train,\n",
        "            save_interval=-1,\n",
        "            task='classification',\n",
        "            batch_size=8,\n",
        "            log_interval=50,\n",
        "        )\n",
        "        acc = evaluate_accuracy(\n",
        "            get_loader(\n",
        "                path_data_valid,\n",
        "                path_labels_valid,\n",
        "                token2ind=token2ind,\n",
        "                batch_size=20,\n",
        "                task='classification',\n",
        "            )\n",
        "        )\n",
        "        if from_scratch:\n",
        "            from_scratch_valid_acc.append(acc)\n",
        "        else:\n",
        "            pretrained_valid_acc.append(acc)\n",
        "    print()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "RCpBIdTHojm6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "outputId": "939cbeec-05f7-432c-d08a-5085e4d1bf96"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACOqklEQVR4nOzdd1zU9R/A8ddx7K1sFAUX7oWKmmWm5U4tt4UrLcu0aNcvR0Mzy0wbtiwrV47UcqSiWY7cW8QtKktAtqy77++Pk5MTUA4ODuT9fDzuwfG9732/7+8Bd28+4/1RKYqiIIQQQghRhViYOwAhhBBCiPImCZAQQgghqhxJgIQQQghR5UgCJIQQQogqRxIgIYQQQlQ5kgAJIYQQosqRBEgIIYQQVY4kQEIIIYSociQBEkIIIUSVIwmQqDRUKhXTpk0z+nmXLl1CpVLx008/mTwmcf+48/frp59+QqVScenSpXs+19/fn1GjRpk0nlGjRuHv72/SY1YWDz/8ME2bNjV3GGanUqmYOHGiucO4b0kCJIyS96GgUqnYuXNngccVRcHPzw+VSkWfPn3MEKFpbNiwAZVKha+vL1qt1tzhiHwmTZqESqXi3LlzRe7zzjvvoFKpOHbsWDlGZryoqCimTZvGkSNHzB2KgevXrzN58mQaNmyInZ0dnp6etGvXjjfeeIO0tDSjj7d7926mTZtGUlKS6YMthbz3ssJuzz33nLnDE2XM0twBiMrJ1taWJUuW0KlTJ4PtO3bs4OrVq9jY2JgpMtNYvHgx/v7+XLp0iW3bttGtWzdzhyRuGTFiBPPnz2fJkiVMmTKl0H2WLl1Ks2bNaN68eYnP8/TTTzN06NAy/V2Oiopi+vTp+Pv707JlS4PHvvvuO7Mk34mJibRp04aUlBTGjBlDw4YNSUhI4NixY3z99ddMmDABR0dHo465e/dupk+fzqhRo3B1dS2bwEvo0UcfJSQkpMD2Bg0amCEaUZ4kARIl0qtXL1asWMG8efOwtLz9a7RkyRKCgoKIj483Y3Slk56eztq1a5k5cyY//vgjixcvrrAJUHp6Og4ODuYOo1wFBwdTr149li5dWmgCtGfPHi5evMhHH31UqvOo1WrUanWpjlEaVlZWZjnvDz/8QGRkJLt27aJjx44Gj6WkpGBtbW2WuMpKgwYNeOqpp8wdhjAD6QITJTJs2DASEhLYsmWLflt2djYrV65k+PDhhT4nPT2dV155BT8/P2xsbAgMDOSTTz5BURSD/bKysnj55Zfx8PDAycmJxx9/nKtXrxZ6zGvXrjFmzBi8vLywsbGhSZMmLFy4sFTX9vvvv3Pz5k0GDRrE0KFDWb16NZmZmQX2y8zMZNq0aTRo0ABbW1t8fHx44oknOH/+vH4frVbL559/TrNmzbC1tcXDw4MePXpw4MAB4O7jk+4ckzJt2jRUKhWnTp1i+PDhVKtWTd8Cd+zYMUaNGkWdOnWwtbXF29ubMWPGkJCQUOhrNnbsWHx9fbGxsSEgIIAJEyaQnZ3NhQsXUKlUfPbZZwWet3v3blQqFUuXLr3r6xcXF8fYsWPx8vLC1taWFi1asGjRIoN98q77k08+4dtvv6Vu3brY2NjQtm1b9u/ff9fjg64V6PTp0xw6dKjAY0uWLEGlUjFs2DCys7OZMmUKQUFBuLi44ODgwIMPPsj27dvveY7CxgApisIHH3xAzZo1sbe3p0uXLpw8ebLAcxMTE3n11Vdp1qwZjo6OODs707NnT44eParf5++//6Zt27YAjB49Wt/1kve7UNgYoOL+DeWNHVmzZg1NmzbV/21s2rTpntd9/vx51Go17du3L/CYs7Mztra2Btv27t1Ljx49cHFxwd7ens6dO7Nr1y7949OmTeO1114DICAgQH+dxRlbdfDgQTp27IidnR0BAQEsWLBA/1haWhoODg5Mnjy5wPOuXr2KWq1m5syZ9zxHceSNSbpbPHmK8/sP935vyK8kP0dxb5IAiRLx9/enQ4cOBh+GGzduJDk5maFDhxbYX1EUHn/8cT777DN69OjBnDlzCAwM5LXXXiM0NNRg32eeeYa5c+fy2GOP8dFHH2FlZUXv3r0LHDM2Npb27duzdetWJk6cyOeff069evUYO3Ysc+fOLfG1LV68mC5duuDt7c3QoUNJTU3ljz/+MNhHo9HQp08fpk+fTlBQEJ9++imTJ08mOTmZEydO6PcbO3YsL730En5+fsyaNYs333wTW1tb/vvvvxLHN2jQIDIyMpgxYwbjxo0DYMuWLVy4cIHRo0czf/58hg4dyrJly+jVq5fBh2NUVBTt2rVj2bJlDBkyhHnz5vH000+zY8cOMjIyqFOnDg888ACLFy8u9HVxcnKiX79+RcZ28+ZNHn74YX755RdGjBjB7NmzcXFxYdSoUXz++ecF9l+yZAmzZ8/m2Wef5YMPPuDSpUs88cQT5OTk3PU1GDFihP75+Wk0Gn777TcefPBBatWqRUpKCt9//z0PP/wws2bNYtq0aVy/fp3u3buXaNzNlClTePfdd2nRogWzZ8+mTp06PPbYY6Snpxvsd+HCBdasWUOfPn2YM2cOr732GsePH6dz585ERUUB0KhRI9577z0Axo8fzy+//MIvv/zCQw89VOi5jfkbAti5cyfPP/88Q4cO5eOPPyYzM5Mnn3yy0KQ4v9q1a6PRaPjll1/u+Xps27aNhx56iJSUFKZOncqMGTNISkrikUceYd++fQA88cQTDBs2DIDPPvtMf50eHh53PfaNGzfo1asXQUFBfPzxx9SsWZMJEybo/8FxdHRkwIABLF++HI1GY/DcpUuXoiiK/vfkbjIzM4mPjy9wy87ONioeMO73v7jvDSX9OYpiUIQwwo8//qgAyv79+5UvvvhCcXJyUjIyMhRFUZRBgwYpXbp0URRFUWrXrq307t1b/7w1a9YogPLBBx8YHG/gwIGKSqVSzp07pyiKohw5ckQBlOeff95gv+HDhyuAMnXqVP22sWPHKj4+Pkp8fLzBvkOHDlVcXFz0cV28eFEBlB9//PGe1xcbG6tYWloq3333nX5bx44dlX79+hnst3DhQgVQ5syZU+AYWq1WURRF2bZtmwIokyZNKnKfu8V25/VOnTpVAZRhw4YV2DfvWvNbunSpAij//POPfltISIhiYWGh7N+/v8iYvvnmGwVQwsPD9Y9lZ2cr7u7uysiRIws8L7+5c+cqgPLrr78aPLdDhw6Ko6OjkpKSYnDdbm5uSmJion7ftWvXKoDyxx9/3PU8iqIobdu2VWrWrKloNBr9tk2bNimA8s033yiKoii5ublKVlaWwfNu3LiheHl5KWPGjDHYfufrnfe7fvHiRUVRFCUuLk6xtrZWevfurX+tFEVR3n77bQUweG0yMzMN4sq7ZhsbG+W9997Tb9u/f3+RP/+RI0cqtWvX1n9f3L+hvGuxtrY22Hb06FEFUObPn1/gXPnFxMQoHh4eCqA0bNhQee6555QlS5YoSUlJBvtptVqlfv36Svfu3Q1ej4yMDCUgIEB59NFH9dtmz55t8FreS+fOnRVA+fTTT/XbsrKylJYtWyqenp5Kdna2oiiK8tdffymAsnHjRoPnN2/eXOncufM9zwMUeVu6dKnR8RT397847w158ZX05yjuTVqARIkNHjyYmzdv8ueff5Kamsqff/5ZZPfXhg0bUKvVTJo0yWD7K6+8gqIobNy4Ub8fUGC/l156yeB7RVFYtWoVffv2RVEUg//cunfvTnJycqHdI/eybNkyLCwsePLJJ/Xbhg0bxsaNG7lx44Z+26pVq3B3d+fFF18scAyVSqXfR6VSMXXq1CL3KYnCZqfY2dnp7+f9R5vXhZH3Omi1WtasWUPfvn1p06ZNkTENHjwYW1tbg1agv/76i/j4+HuOldiwYQPe3t76//hBN5Zl0qRJpKWlsWPHDoP9hwwZQrVq1fTfP/jgg4CuBeVennrqKa5evco///yj37ZkyRKsra0ZNGgQoBvHkzdmRavVkpiYSG5uLm3atDH692Pr1q1kZ2fz4osvGvz87vzdBLCxscHCQvf2qtFoSEhIwNHRkcDAwBL9XkLx/4bydOvWjbp16+q/b968Oc7Ozvd8bb28vDh69CjPPfccN27cYMGCBQwfPhxPT0/ef/99fYvikSNHOHv2LMOHDychIUH/95eenk7Xrl35559/SjWI29LSkmeffVb/vbW1Nc8++yxxcXEcPHhQf42+vr4Gv6snTpzg2LFjxR7X069fP7Zs2VLg1qVLF6PjKe7vvzHvDSX9OYp7kwRIlJiHhwfdunVjyZIlrF69Go1Gw8CBAwvd9/Lly/j6+uLk5GSwvVGjRvrH875aWFgY/MEDBAYGGnx//fp1kpKS+Pbbb/Hw8DC4jR49GtD1xRvr119/pV27diQkJHDu3DnOnTtHq1atyM7OZsWKFfr9zp8/T2BgoMEA8DudP38eX19fqlevbnQcdxMQEFBgW2JiIpMnT8bLyws7Ozs8PDz0+yUnJwO61ywlJeWe9VVcXV3p27evQffS4sWLqVGjBo888shdn3v58mXq16+v//DPc+fPOU+tWrUMvs9LhvInm0UZOnQoarVaH2dmZia///47PXv2NEiqFi1aRPPmzbG1tcXNzQ0PDw/Wr1+vf12KKy/2+vXrG2z38PAwOB/okq3PPvuM+vXrY2Njg7u7Ox4eHhw7dszo8+Y/f3H+hvLc+dqC7vUtzmvr4+PD119/TXR0NBEREcybNw8PDw+mTJnCDz/8AMDZs2cBGDlyZIG/we+//56srKwSXyuAr69vgQH+eTOz8sYPWVhYMGLECNasWUNGRgag+121tbXVJ8H3UrNmTbp161bg5uXlZXQ8xf39N+a9oTQ/R3F3MgtMlMrw4cMZN24cMTEx9OzZs9ymuOb9Z/nUU08xcuTIQvcxdgr02bNn9QNw7/yQA90b6/jx442M9O6Kagm6c0xDfvlbe/IMHjyY3bt389prr9GyZUscHR3RarX06NGjRP+Fh4SEsGLFCnbv3k2zZs1Yt24dzz//fIE39tIqapaVcseg3sJ4enry6KOPsmrVKr788kv++OMPUlNTDcZ9/Prrr4waNYr+/fvz2muv4enpqR8cm3+wuqnNmDGDd999lzFjxvD+++9TvXp1LCwseOmll8ptantpXts8KpWKBg0a0KBBA3r37k39+vVZvHgxzzzzjP46Zs+eXWAKfx5jp8uXREhICLNnz2bNmjUMGzaMJUuW0KdPH1xcXMr83OXBFD9HUThJgESpDBgwgGeffZb//vuP5cuXF7lf7dq12bp1K6mpqQb/wZ4+fVr/eN5XrVarb2HJExERYXC8vBliGo3GZFPUFy9ejJWVFb/88kuBN52dO3cyb948IiMjqVWrFnXr1mXv3r3k5OQUOV25bt26/PXXXyQmJhb5n15ey8GdBeLu/G/+bm7cuEFYWBjTp083mBae9x96Hg8PD5ydnQ0GaRelR48eeHh4sHjxYoKDg8nIyODpp5++5/Nq167NsWPH0Gq1BsnSnT9nUxkxYgSbNm1i48aNLFmyBGdnZ/r27at/fOXKldSpU4fVq1cbJJuFdT3cS17sZ8+epU6dOvrt169fL/Df+MqVK+nSpYu+tSRPUlIS7u7u+u+N6Qot7t9QWalTpw7VqlUjOjoaQN9K6+zsfM+/wZJ0+UZFRRUo83DmzBkAg9lxTZs2pVWrVixevJiaNWsSGRnJ/PnzjT6fKeIp7u9/cd4bRNmTLjBRKo6Ojnz99ddMmzbN4IPnTr169UKj0fDFF18YbP/ss89QqVT07NkTQP913rx5BvvdOatLrVbz5JNPsmrVqkI/0K9fv270tSxevJgHH3yQIUOGMHDgQINb3jTevFlvTz75JPHx8QWuB27/Z/bkk0+iKArTp08vch9nZ2fc3d0NxrEAfPXVV8WOOy9Zu/M/wjtfMwsLC/r3788ff/xR6FTb/M+3tLRk2LBh/Pbbb/z000/FLirYq1cvYmJiDJLh3Nxc5s+fj6OjI507dy72dRVH//79sbe356uvvmLjxo088cQTBtO0C3tt9u7dy549e4w+V7du3bCysmL+/PkGxytsxqFarS7w81ixYgXXrl0z2Jb3YVqcCsnF/Rsqrb179xaY1Qawb98+EhIS9P+YBAUFUbduXT755JNCq0Pn/xs05jrz5Obm8s033+i/z87O5ptvvsHDw4OgoCCDfZ9++mk2b97M3LlzcXNzM9lrYWw8xf39L857gyh70gIkSq2oLqj8+vbtS5cuXXjnnXe4dOkSLVq0YPPmzaxdu5aXXnpJ/99ky5YtGTZsGF999RXJycl07NiRsLCwQpc9+Oijj9i+fTvBwcGMGzeOxo0bk5iYyKFDh9i6dSuJiYnFvoa9e/dy7ty5ItfdqVGjBq1bt2bx4sW88cYbhISE8PPPPxMaGsq+fft48MEHSU9PZ+vWrTz//PP069ePLl268PTTTzNv3jzOnj2r7476999/6dKli/5czzzzDB999BHPPPMMbdq04Z9//tH/Z1kczs7OPPTQQ3z88cfk5ORQo0YNNm/ezMWLFwvsO2PGDDZv3kznzp0ZP348jRo1Ijo6mhUrVrBz506DLsyQkBDmzZvH9u3bmTVrVrFiGT9+PN988w2jRo3i4MGD+Pv7s3LlSnbt2sXcuXMLjF8pLUdHR/r3768fB3TntOc+ffqwevVqBgwYQO/evbl48SILFiygcePGRi/p4OHhwauvvsrMmTPp06cPvXr14vDhw2zcuNGgVSfvvO+99x6jR4+mY8eOHD9+nMWLFxu0HIGuJcDV1ZUFCxbg5OSEg4MDwcHBhY7zKu7fUGn98ssvLF68mAEDBhAUFIS1tTXh4eEsXLgQW1tb3n77bUCXUH///ff07NmTJk2aMHr0aGrUqMG1a9fYvn07zs7O+vIReQnCO++8w9ChQ7GysqJv3753LeLp6+vLrFmzuHTpEg0aNGD58uUcOXKEb7/9tkCr6/Dhw3n99df5/fffmTBhglFFJM+cOcOvv/5aYLuXlxePPvqoUfEU9/e/uO8NooyV97QzUbnlnwZ/N3dOg1cURUlNTVVefvllxdfXV7GyslLq16+vzJ4922Dap6Ioys2bN5VJkyYpbm5uioODg9K3b1/lypUrBaYpK4pu2voLL7yg+Pn5KVZWVoq3t7fStWtX5dtvv9XvU5xp8C+++KICKOfPny9yn2nTpimAcvToUUVRdNN933nnHSUgIEB/7oEDBxocIzc3V5k9e7bSsGFDxdraWvHw8FB69uypHDx4UL9PRkaGMnbsWMXFxUVxcnJSBg8erMTFxRU5Df769esFYrt69aoyYMAAxdXVVXFxcVEGDRqkREVFFfqaXb58WQkJCVE8PDwUGxsbpU6dOsoLL7xQYLq4oihKkyZNFAsLC+Xq1atFvi53io2NVUaPHq24u7sr1tbWSrNmzQq89nk/k9mzZxd4fmEx38369esVQPHx8Skw9Vyr1SozZsxQateurdjY2CitWrVS/vzzzwJTzAs7753T4BVFUTQajTJ9+nTFx8dHsbOzUx5++GHlxIkTSu3atQtMg3/llVf0+z3wwAPKnj17lM6dOxeYnr127VqlcePGiqWlpcHvaWExFvdvCFBeeOGFAq/VnXEW5tixY8prr72mtG7dWqlevbpiaWmp+Pj4KIMGDVIOHTpUYP/Dhw8rTzzxhOLm5qbY2NgotWvXVgYPHqyEhYUZ7Pf+++8rNWrUUCwsLO45Jb5z585KkyZNlAMHDigdOnRQbG1tldq1aytffPFFkc/p1auXAii7d+++6/Xlx12mwef/ORkTT3F+/xWleO8Npfk5intTKYq0twkhCteqVSuqV69OWFiYuUMR4q4GDBjA8ePH77pIbkk9/PDDxMfHF2v8nKg8ZAyQEKJQBw4c4MiRI4UuFClERRIdHc369euLNVBfiDwyBkgIYeDEiRMcPHiQTz/9FB8fH4YMGWLukIQo1MWLF9m1axfff/89VlZWBoUKhbgXaQESQhhYuXIlo0ePJicnh6VLlxZY/FKIimLHjh08/fTTXLx4kUWLFuHt7W3ukEQlImOAhBBCCFHlSAuQEEIIIaocSYCEEEIIUeXIIOhCaLVaoqKicHJyKtWq3UIIIYQoP4qikJqaiq+v7z3XLpQEqBBRUVH4+fmZOwwhhBBClMCVK1eoWbPmXfeRBKgQeeXKr1y5grOzs5mjEUIIIURxpKSk4OfnV6xldyQBKkRet5ezs7MkQEIIIUQlU5zhKzIIWgghhBBVjiRAQgghhKhyJAESQgghRJUjCZAQQgghqhxJgIQQQghR5UgCJIQQQogqRxIgIYQQQlQ5kgAJIYQQosqRBEgIIYQQVY4kQEIIIYSociQBEkIIIUSVIwmQEEIIIaocSYCEKE+KApkp5o5CCCGqPFkNXoiylpsNl/6FiA1wegOkRoFvK2g+FJo+CY4e5o5QCCGqHEmAhCgLmclwdgucXg/ntkLWHa0+UYd1t7/ehrqPQIuhENgLrO3NE68QQlQxkgAJYSrJVyFioy7pubQTtDm3H3PwhMCe0LAPeDXWtQQdWwbXDsK5LbqbtSM0ehyaD4aAh8BCbb5rEUKI+5xKURTF3EFUNCkpKbi4uJCcnIyzs7O5wxEVlaJA7MlbXVt/QvRRw8fdA6FhLwjsDTWCwKKQIXfx5+DYct0t6fLt7U4+0GwgNB8C3s3K9jqEEOI+YczntyRAhZAESBRJkwuRe3StPBHrISky34Mq8Au+nfS41yv+cRUFruzVJUInVkNm0u3HPJvoWoWaDQKXGqa6EiGEuO9IAlRKkgAJA1lpcD5M12119i+4eeP2Y5a2UKeLLulp0AMcPUt/vtws3fihY8vhzCbQZN96QAUBD+pahRo9DrbyuymEEPlJAlRKkgAJUmPhzEZd0nPhb9Bk3X7MrrpuPE9gL6jbBawdyi6Omzfg1Fo49htc3nV7u6Wt7vwthuoGUautyi6G+4UmB+LCdYPPrex1P0MbR3NHJYQwIUmASkkSoCrq+hldt9bp9XD1AJDvT6NaADTsrUs6/IJBbYb5Azcuw/EVupah+DO3t9u76abTNx8KNVqDSlX+sVU0Wi0knIOoQ3DtkO5rzHHIzby9j7UjNO4PrUZArQ7yuglxH5AEqJQkAaoitBq4uv/WeJ4Nug/M/Hxb3x7P49mo4nxAKgpEH9G1Ch1fAenXbz/mVk/XRdZsEFQPMFuI5UpRdGOx9MnOYYg6AtmpBfe1cQHfFroZe4kXbm+vFgAtR0DLYeBSs9xCF0KYliRApSQJ0H1KkwPp8brk4fR63fia/MmDhRXU6axr5QnsCc6+Zgu12DS5ui66Y8sg/E/IvXn7Mb9gXTLUZADYVzdbiCaXGmvYshN1GDISCu5naQc+LXStYr6tdcUnq9fRzcZTFIj8D478CifXQHbarSepoM7D0OopXYuflV05XpgQorQkASolSYAqCUXRjZFJj9clMvpbfOH388+symPjAg0e0yU99bpV7oHFWam6JOjYcri4AxStbruFFTTorptJVr87WNmaN05j3LyhS3D0LTuHIeVawf0srMCrya1kp5Uu4fFoWLyuyux0OLUOjizWVezOY+MCTZ/QJUM1gipOC6AQokiSAJWSJEBmlJ1RSBJTRFKTEQ/aXOOOr1KDqx/Uv5X0+He6PwcQp0TDiZW6ZCjm+O3tti7QuB+41dcNALZ2AhunW/cddffzvlrZle+Hfna6rpZS/pad/N1UeipdcpM/2fFqYprELvEiHF0KR5ZCcr4SB+6BurFCzYeAk3fpz2MuaXFwZZ+u5pR/J/BuLomduK9UqgToyy+/ZPbs2cTExNCiRQvmz59Pu3btitx/7ty5fP3110RGRuLu7s7AgQOZOXMmtra2JT7mnSQBKkPXDum6HopKbHLSjT+mrQs4eNy6uee7X8j3tq6FFyS8n8We0iVCx1cU3npSFJXFrQQpX3JUIFHKu5//MUewcb59P2///JWtc7Mg9oRhy87107dbrfKrFmCY7Pi0KPvZW1qtrjXoyGJd61Be16JKrWspbDUCGvQES+uyjaM0NLm61/jqfl2NqbzEJz+3eroB9E2fBI9A88QphAlVmgRo+fLlhISEsGDBAoKDg5k7dy4rVqwgIiICT8+C9VSWLFnCmDFjWLhwIR07duTMmTOMGjWKoUOHMmfOnBIdszCSAJWRjESY08hwJk5h1Da6ejoGyUsR9+3dwNKmfOKv7LRauLwTIjbpxsxkpeoGCmel6cbAZKXd2paGwQw4U7Gy1yVD1vaQfM1wqZA8Tr63kp2Wt8ftmHv8UmYynPwdDi+Gq/tub7errutWbDlcl5SZW3q8Lsm5ug+u7Ne1ouVk3LGTSjeg37mGLsHL/7fo1VTX5dfkiaozgF7cdypNAhQcHEzbtm354osvANBqtfj5+fHiiy/y5ptvFth/4sSJhIeHExYWpt/2yiuvsHfvXnbu3FmiYxZGEqAycuw3WD0OHL113TBFJTY2TtIsb05are6DU58Updy+n50vScrKS55SDR/Lvy0rtfBEB3QJRP6WnRqtK3730vUzulaho8sgLeb2dq9mulahZoPBwa3s49DkQtyp28nO1X2FdxfauEDNNuDXDmq21d23ddE9lpWqW7vuxCo4F2b4c6oRpGsVajKgckwGEPeWlaZbuifmmO6WEqWrYVag9fYeLbpWDhW6Fb1SJEDZ2dnY29uzcuVK+vfvr98+cuRIkpKSWLt2bYHnLFmyhOeff57NmzfTrl07Lly4QO/evXn66ad5++23S3RMgKysLLKybhe6S0lJwc/PTxIgU1sxGk6uhk6h0G2quaMR5SU3q2Ci5OQNrrUrb6KryYUL2+Hwr7oSCnnVuvMGnLd6Cuo9arp6URmJt7qybrXwXD1YeHexeyD4tdXNAKzZDtwbFO/DKiNRt57diVVw8Z98XZEqqN1R1zLUqB84epjmeswtN0vXnWmOel7lIS0Ooo/dTnZijkPCeUzTsqvSJU75u8H1XeT5tznlS6iK2uZk8jGYxiRAZvvpx8fHo9Fo8PLyMtju5eXF6dOnC33O8OHDiY+Pp1OnTiiKQm5uLs899xxvv/12iY8JMHPmTKZPn17KKxJ3pcnR/ZcJuiUjRNVhaaO7lUfLSHlRW0L9R3W3jEQ4vlLXMhR9RJdInP4THDyhxRBo+RR4Niz+sbUa3XioK/tuJzx31qgC3YeJvnWnHdQMArtqJbse++rQOkR3S4vTVR8/sUq37t3lXbrbhtd1ZSKaPgkN+4Cda8nOVd40ObqWD33phCO61jOVClxr6caYVQ/QlUjIu1/Nv3KUQNBq4cZF3eSBmOO3k5202ML3d/LRLa7s3Vx3jbmZt1ptU/O18qbma8m9Y5uiBRTddn3piFLoMBG6f1j645RQpUp///77b2bMmMFXX31FcHAw586dY/Lkybz//vu8++67JT7uW2+9RWhoqP77vBYgYUKReyArWTdmp2Ybc0cjhOnYV4fg8bpbzAk4skQ36Dw9DnbP1918W+u6yJo+WTBJuXlD16JzZe/t1p3Ciji61b/dleXXTjcTLv/AclNx9IR243S35Ku68U8nVukGqp/fprv98ZIu+WvyRMVaUkSrgfizhkUxY44bLmWTR0HXbZh4Ac4Xciwnn3xJkf+tr3V0CVJJE83SyMmE6+G3Wnbykp0TRUwcUYF7/dvJTt7X0rTgKQrk3MzX1Z1asGu8uIlUVpruZ2LjVPJ4TMBsCZC7uztqtZrYWMNMNTY2Fm/vwscBvPvuuzz99NM888wzADRr1oz09HTGjx/PO++8U6JjAtjY2GBjIwNpy1TEJt3X+t3L5k1biIrAuyn0mAHdpsHZzbpk6Oxft6b1H4JNb0OjPrqlN6KP6MbvxEcUPI61o25MVM12t5MecwwGd6kJHV/U3RLO67qwT6zWtaBEbNDdLO0gsIcuuav3aPnVmVIUuHHJsGUn+kjhLRO2LoZjzXxa6lqAEi/qEqAbF3X3b1yExEu6f9ZSo3W3/Gvw6Y/nequlKF9SlNd65Ohd+jEyGYm6GXz5k53rEaBoCu5raasrA6FPdpqDV2PTr1GoUukmMFjbm2bRZ01O4bM+y5HZEiBra2uCgoIICwvTj9fRarWEhYUxceLEQp+TkZGBxR2/WGq17sNUUZQSHVOUA0XRLSwKujdKIe53lta6RKdRH12X0rHfdF1kcad0rSknVhnuX73OrWTn1vgdz8YV7x8Ft7rw0Gu6W1y4LhE6sVKXQJz8XXezdtJdc9MndRW1TTm+IyX6dn2ovNadm4kF97Oy183Ky0t28iqAFzbezKUmBDxouE1RdAmIQVKU9/WCrnspM+l2+YY7Wdrm60q746trLcPXRFEg+Youycmf7CRfKfw1sKsOPs0Nkx23epVzLFMFqL9m1lctNDSUkSNH0qZNG9q1a8fcuXNJT09n9OjRAISEhFCjRg1mzpwJQN++fZkzZw6tWrXSd4G9++679O3bV58I3euYwgwSzuneOCysdCuXC1GVOHpCx4nQ4QXdB+aRJbq/B58Wt1t3HNzNHaVxPBvBI+9Al7d1409OrNIlRClXdYUkjy7VdRM17qdLhmo/YFxCl5GYL9k5rLufGl1wPwsrXatb/mTHPbB0CYFKpRuv5lBEd312uq7lKX9SlHc/6YpuXM31cN2twLHVuqSreoCuuy7meOEV6kE3ScCnuWEXlrNv5Z04UAGZNQEaMmQI169fZ8qUKcTExNCyZUs2bdqkH8QcGRlp0OLzv//9D5VKxf/+9z+uXbuGh4cHffv25cMPPyz2MYUZRNxq/fHvZPY+XyHMRqXSfUjXaG3uSExHpbpVs6kldJuum6l2YpWuNSg9Dg7+pLs5ekHj/rpkqGZbwy6irFRdEqVv2TmkSzAKnMtCN+7JtzXUaKVLdryaln8dMGsHXZeTV5OCj2lydK03+qTokmErUu5NXTHK/AUpLSzBo1G+lp1muuuqLIPMKzGzV4KuiKQOkIn92EvXj97zYwh+1tzRCCHKmiZXV3TzxGrdjLL8rRwufrpZZJnJumTnegSFTs+uXidfy05rXWJQUQZbl4SiQGrM7WQIdK1XHg2lmKsJVYo6QBWZJEAmlJEIs+vpBu9NPgbVaps7IiFEecrNhgt/61qGTv9Z+CBl5xq6Fp28ZMe3pXlmWolKr1LUARJVxLmtuuTHs7EkP0JURZbW0OAx3S3nJpzdAufDdN1iecudOMkQBVH+JAESZevMrenvUvxQCGFlB40f192EMLOKu6CHqPw0OXB2q+6+JEBCCCEqEEmARNmR6s9CCCEqKEmARNmR6s9CCCEqKEmARNnJG/8j1Z+FEEJUMJIAibIRfxYSz0v1ZyGEEBWSJECibEj1ZyGEEBWYJECibOi7v3qaNw4hhBCiEJIACdPLSITI/3T3Zfq7EEKICkgSIGF658Kk+rMQQogKTRIgYXpnbo3/adDdvHEIIYQQRZAESJiWQfVnGf8jhBCiYpIESJiWVH8WQghRCUgCJExLqj8LIYSoBCQBEqYl1Z+FEEJUApIACdPJX/25ThdzRyOEEEIUSRIgYTr5qz/bOps3FiGEEOIuJAESpiPVn4UQQlQSkgAJ05Dqz0IIISoRSYCEaUj1ZyGEEJWIJEDCNKT6sxBCiEpEEiBRelL9WQghRCUjCZAovUpU/Xn/pUR+3nOJlMwcc4cihBDCjCzNHYC4D5z5S/e1Ald/ztVo+WzrGb76+zyKAp/8FcHYTnUY3ckfZ1src4cnhBCinEkLUHnKzYLDiyH+nLkjMa28+j8VtPpzVNJNhn77H19u1yU/3s62pGTm8tnWM3T6aBufbz1L8k1pERJCiKpEEqDy9GcorH0eds01dySmU8GrP285FUvPz//lwOUbONlY8sXwVux+8xG+GN6K+p6OtxOhWduYu/WMJEJCCFFFSAJUnlqH6L4eWw6pMeaNxVQqaPXnrFwN0/84ybifD5B8M4fmNV1YP+lB+jT3xcJCRZ/mvvz10kN8MbwVDbwcSc3MZe7Ws3SatY3PtkgiJIQQ9ztJgMpTrWDwCwZNNuz9xtzRmEbe+J8KVP35Unw6T369mx93XQLgmU4BrHyuI7Xc7A32y0uENk1+iC+HtybQy4nUzFw+DztLp4+2MWfLGZIzJBESQoj7kUpRFMXcQVQ0KSkpuLi4kJycjLOziVs1wv+E5SPAxgVCT4KNk2mPX55u3oCP6+oKIE4+CtX8zR0Ra49c453fT5CWlUs1eys+GdSCro28ivVcrVZh08kYPt96lojYVACcbCwZ/YA/YzvVwcVeBksLUdll52rZHhGHv5sDgd6V+P1XFMqYz2+ZBVbeAnuBWz1IOAeHfoYOL5g7opI7u1WX/Hg0MnvyczNb1+W1bP8VANr5V+fzYS3xcbEr9jEsLFT0auZDjybe/HUyhs/DznI6JpV5287x465LjHrAn7GdAnC1ty6ryxBClJH0rFyW7ovkh50XiU7OxM5KzS9j29HGv7q5QxNmIi1AhSjTFiCAgz/BH5PBuSZMPgLqStqysHIMnFgFnV6GbtPMFsaZ2FQmLjnEmdg0VCp4sUs9JnWtj6W6dD28Wq3C5lMxzN2qS4QAHPUtQpIICVEZxKdlsWj3JX7ec1k/ts9KrSJHo+BkY8niccE0r+lq3iCFyRjz+S0JUCHKPAHKyYS5zSA9Dp74DpoPNv05ypomR9f9lZUMYzbrxjeVM0VRWL7/CtP+OElmjhYPJxs+H9KSjvXcTXqeohKhUR11iVA1B0mEhKhoIhMy+Pbf86w4cJWsXC0AAe4OjH+oDr2a+jD+lwPsvZiIq70Vy8d3kO6w+4QkQKVU5gkQwD+zYdsH4NUUntsJKlXZnKesXPwXFvXRVX9+9Wy5F0BMzczh7d9P8MfRKAAerO/OZ0Na4u5oU2bn1CVCsXwedpbw6BQAHKzVjHrAn2c61ZFESIgK4MS1ZBbsOM+G49Fob326tfBzZULnOjza2Bu1he69Ni0rl6e+38uRK0m4O9qw4rkOBLg7mDFyYQqSAJVSuSRAGYnwWVPISYenVkO9rmVznrLy1zuw5wtoMRwGfF2upz5+NZmJSw9xOSEDtYWKVx8L5NmH6mBhUT5JpFarsCU8ls+3nuVUvkRoZEd/nnmwDtUlERKiXCmKwq5zCXzzz3n+PRuv3965gQfPda5L+zrVURXyT2ZyRg5Dv/uP8OgUfF1s+e25DtSsZl9gP1F5SAJUSuWSAAFsfBP2fg11HoaQtWV3nrIwr7WuAOKgRdCkf7mcUlEUftx1iZkbw8nRKNRwtWPesFYE1a5WLucvLJ4tp2KZe0ciFNLRn3GSCAlR5jRahY0novlmxwWOX0sGQG2hok9zH559qC6Nfe/9/h2flsWQb/Zw/no6td3sWfFsBzydbcs6dFFGJAEqpXJLgJIi4fOWuplU43eAb8uyO5cpxZ+FL9roqj+/fqFcCiAmZWTz6opjbA2PBaB7Ey8+frJFhZiarigKW8PjmLv1DCejdImQ/a0WIUmEdBRF4UJ8OttPx7HnfALeLrYMbVuLZjVdzB1auUu+mcOl+HScbC1xtbfG2day1AP2q5rMHA0rDl7lu38uEJmYAYCtlQVD29ZibKcA/Kob14oTk5zJoG92cyXxJvU9HVn+bAf5u62kJAEqpXJLgABWjoUTK6HpQBj4Q9mey1R2zYMt7+qWvghZU+an238pkclLDxOVnIm12oL/9WnE0+1rF9qkbU55idDnYWc4ce12IhTSwZ9xDwbgVobjkyqizBwNey8msv10HNtOx+k/qPJr4uvM0Ha16NfS975elFajVdh1Lp4VB6/y18kYsm8Nys3jZGOJi70VLnZWuNpb4WpnjfOt+y52Vrjeuu9sp3vMxV63zd5aXeH+DspSckYOv/x3iZ92XyI+LRsAV3srRnbwZ2RH/1IlLVcSMxi0YA8xKZk08XVmybj2uNjdv7+T9ytJgEqpXBOg6KPwzUOgUuumxLvWKtvzmcKPveHyTuj5MQQ/W2an0WoVvt5xnjlbzqDRKgS4OzB/WCua1qjYrQaKohAWHsfnYWf1zfL21mqe7lCb8Q/Wua8Toejkm2w7Hcf209fZdS6emzka/WNWahXBAW48WN+dk1EpbDoRQ7ZGlwjYWanp3dyHYe1q0bqW633zoX4xPp1VB6+y6tBVopMz9dvdHa3JytGSmpVbquNbqVW42Fnpb6721rjaWRkmT4UkVC52VlhVolanqKSbLNx5kaX7IknP1v1O1XC1Y9yDAQxu64e9tWlK2p2/nsaQb/YQn5ZN61qu/DI2GAcbKZdXmUgCVErlmgAB/NwPLvwNwROg50dlf77SKKfqz3GpmYQuP8rOc7oBjQNa1eD9/k1xrERvRoqisO10HHO33k6E7KzUjAiuxYMNPGji61yms9bKQ65Gy+ErSfpWnrwyAXm8nG3oEuhJl4aePFDP3eDnl5iezepDV1m2/wrn4tL02xt4OTK0bS2eaF2jUtZaSsvKZcOxaFYcvML+Szf0251tLenXsgaD2tSkWQ0XVCoVuRotKZm5JGVkk3Qzh+SbOSRn6L4mZeSQdDPbcNut7ck3s8nRlPyt20IFtd0caODlSKCXEw28nQj0csLf3aFCJUZnY1NZsOMCa49cI/fWlK6G3k4817kuvZv7lEms4dEpDP32P5Jv5tChjhs/jm6LrVX5znIVJScJUCmVewJ0Lgx+fQKs7OHlk2BfgSuTHlsBq5/RVX9+4b8yOcW/Z6/z8vIjxKdlY2el5r1+TRgYVLPStgooisL2CF0idOxqssFjnk42NPF1prGvM419XGji60yt6vblNqOtJG6kZ7PjzHW2nY5jx5nrBgvHqlTQys9Vn/Q08XW+589NURQOXr7B0n1XWH88iswcXauQtaUFPZt6M7RtrSJn8VQUWq3C3ouJrDx4lQ3Ho/UtXxYqeLC+B4Pa1KRbIy+TfZAqisLNHM2tZChH/zX5VsKkS57uSKhuZpOUkUNqZtGtTlZqFXU9HGng5USgt5Puq5cTNavZlevv5P5LiXyz4zxbw+P029rXqc5znevSuYFHmf8uHLmSxFPf7yUtK5dHGnqy4KkgrC0rTmIoiiYJUCmVewKkKLDgQYg9Do/8Dx56rezPWVJlWP05V6Pls61n+Orv8yiK7j+9L4a3op7n/VGgLC8RWn3oGqeiU7gYn05hf30O1moa+TjrE6Mmvi7U93LExtI8/4UqisKp6BR9K8+RK0n6+ioALnZWPNTAg0caetC5gWepxmEk38xh3ZFrLN13RT+zDqCOuwND2vrxZFDNCtVqdvVGBqsOXmPloStcSbyp317H3YEng2ryZOuaeLtUrBlFGq1CQloWZ+PSiIhJ5UxsKhGxqZyJSdV3L93JzkpNA687EiNvJzydbEyWjGi1CmGn41iw4zwHL+tazlQq6N7Ym2c716FVrfKd7bn3QgIjf9xHZo6W3s18+HxoSxmsXglIAlRK5Z4AARz7DVaPAwcPeOkEWFWsN01AV/15dl3INH3152tJN5m89DAHbr3xjQiuxbt9Gt/XTc/pWbmcjknlVHQKp6KSORWVwumYVH3V2vwsLVTU83S81VKkS4oa+ziX2Sy49Kxcdp6L5+8I3XiemJRMg8cbejvRpaEnjzT0pJWfq8k/GBRF4fi1ZJbui2TdkSj9B7OVWsWjjb0Y2rYWneq5m6Wl7Ga2hk0no1l58Cq7zyfok1hHG0v6NPdhUJuatK5VrUK3WBVGq1WISr7JmdhUTsfoEqKI2DTOx6Xpx2rdycXO6lYX2q2utFuJkTFdl9m5WtYeuca3/1zg7K2uUGu1BU+0rsG4h+pQ18PRJNdXEjvOXGfcogNka7Q82bomswc2r9Cts6ISJkBffvkls2fPJiYmhhYtWjB//nzatWtX6L4PP/wwO3bsKLC9V69erF+/HoBRo0axaNEig8e7d+/Opk2bihWPWRIgTY5uSnzKVegzF9qMLp/zGqOMqj9vORXLqyuOknwzBycbSz56sjm9m/uY5NiVTa5Gy4X4dE5FpXAyKplT0SmcjEohKSOn0P1ruNoZtBQ19nXG18W2RB++l+LTdQOYI+LYeyHR4EPPzkrNA/Xc6NLQk4cDPanhWvxFZksrLSuXP49GsXT/FY5eSdJvr1nNjqFt/RjUxg+vMq7boigKhyKTWHnwCn8ejTYYvNyxrhsDg2rSo6m3yQbjViS5Gi2XEjJ0LUX5WowuxacbtATm5+lkY9CF1sDbifqejgYDitOyclm6V7c4aV6C7WRjyYj2tRnzgH+FqcXz18kYnl98CI1W4an2tXi/X9NKl9xWJZUqAVq+fDkhISEsWLCA4OBg5s6dy4oVK4iIiMDT07PA/omJiWRnZ+u/T0hIoEWLFnz//feMGjUK0CVAsbGx/Pjjj/r9bGxsqFateE2oZkmAAPZ8CX+9rVst/oX9YFHBmltNXP05K1fDRxtP8+OuSwC0qOnC/GGtqeUmlVjzUxSF6OTMW0lRCqeikzkZlcLVGzcL3d/V3orGPrdaimroxhbV9XAo0EqTnatl38VEfdJzMT7d4PFa1e15pKEnDwd60L6OW4VojTsVlcKy/ZH8fviafiyL2kJFl0BPhrXz4+FAT/1SB6YQm5LJqkNXWXnwKheu3359alazY+CtLi5ja87cLzJzNJy/nnYrMUrTJ0jXkgr/vQTwq25HoJcTHk42rD8WTcqtn6Gnkw1jOgUwPLhWhSyHsPbINV5afgRFgfEP1eGtng0lCaqgKlUCFBwcTNu2bfniiy8A0Gq1+Pn58eKLL/Lmm2/e8/lz585lypQpREdH4+CgW8dl1KhRJCUlsWbNmhLFZLYEKCsVPmui62Iashga9Sm/cxeHCas/X4pPZ+LSQ/p6OeMeDOC17g1loKERkm/mEH6rhSivxehcXJp+tkx+1pYWNPR2orGPM7XdHDhy5QY7z8YbjPmwtFDR1r86jzTUDWCu6+FQYd/kb2Zr2HA8mmX7Iw1mWvm42DKojR+D29Qs8ZIGWbkatp6KY8XBK/xz5rq+lcPOSk3PZt4MDKpJ+wA36QopQmpmDmfj0m51oaXqE6T4tKwC+9bxcODZh+rQv1UNs41xK65l+yJ5c/VxAF7u1oDJ3eqbOSJRmEqTAGVnZ2Nvb8/KlSvp37+/fvvIkSNJSkpi7dp7Lw/RrFkzOnTowLfffqvfNmrUKNasWYO1tTXVqlXjkUce4YMPPsDNza3QY2RlZZGVdfuPMyUlBT8/v/JPgAC2Toedc8AvGMZuLt9z340Jqz+vPXKNd34/QVpWLtXsrfh0cAseaehlwmCrrqxcDWdj0zgVlXKr+yyZ8OhU0oqoN+PuaEOXQA8eaehJp/ruOFXA/77v5VxcKkv3XWH1oavcuNVVqFLBQ/U9GNbOj66NvO45XVpRFE5cS2HFwSusPRJlMLOtTe1qDGpTk17NfCrl61NRJKRlcSZW11IUmZhBW//qPNbYq1Ilkgt3XuS9P08B8E6vRox7qI6ZIxJ3qjQJUFRUFDVq1GD37t106NBBv/31119nx44d7N27967P37dvH8HBwezdu9dgzNCyZcuwt7cnICCA8+fP8/bbb+Po6MiePXtQqwv+lzFt2jSmT59eYLtZEqDUGJjbDDTZMOYvqNW+fM9flN3zYfP/Sl39efofJ/VdXu0CqvP50Jb4uJTfeJKqSKtViEzMuDXYOoUL8WkEejnTpaEHTX1dKtUH0N1k5Wr462Qsy/ZFsvt8gn67u6MNg9rUZGhbP2q7Ga72HZ+WxZrD11h58KpBDSNvZ1ueDKrBk61rUseMg3BFxfPl9nPM/isCgA/6N+Wp9rXNHJHIr8okQM8++yx79uzh2LFjd93vwoUL1K1bl61bt9K1a8FV1ytUCxDAuhfh0M8Q2BuGLSn/8xfGBNWfT8ek0GPuv6hU8OIj9Zn0SD2ZVirKxKX4dJYfuMKKA1cNul461nVjaLta2FpasOLgVbafjtN3GVpbWvBYYy8GtfGjUz13k44lEveXjzed5qu/z6NSwaeDWvBE65rmDkncYkwCZNYpC+7u7qjVamJjYw22x8bG4u3tfdfnpqens2zZMt577717nqdOnTq4u7tz7ty5QhMgGxsbbGwqTm0ROtxKgCI2wPUz4NHAvPHcvAGRe3T3G3Qv8WHWHokCdHU9Qh818zWJ+5q/uwNv9GhI6KMNCAuPZem+K/xz9jq7zycYtA6BbvD9wDZ+PN7ct0Isrisqvte6B5KRreGn3Zd4dcXRW+PDqubM1crMrP9+W1tbExQURFhYmH6bVqslLCzMoEWoMCtWrCArK4unnnrqnue5evUqCQkJ+PhUkl9QjwYQ2AtQYM98c0cDZ7fqlr7waFTipS+0WoV1txKgfi19TRicEEWzUlvQo6kPi8a049/XuzCpa31quNrh5WzDuAcD+Oulh1g7sRNPt68tyY8oNpVKxZQ+jRncpiZaBSYtO8z203H3fqKoUMze/xAaGsp3333HokWLCA8PZ8KECaSnpzN6tK4OTkhICG+99VaB5/3www/079+/wMDmtLQ0XnvtNf777z8uXbpEWFgY/fr1o169enTvXvLWi3LXcZLu69FlkBp7933L2plb9ZMCe5T4EAcjb3At6SZONpZ0aViwvIEQZa1mNXtCH23ArjcfYe/b3Xind2MCve+PKuOi/FlYqJj5RHP6NPchR6Pw3K8H2X0+3txhCSOYPQEaMmQIn3zyCVOmTKFly5YcOXKETZs24eWlmxUUGRlJdHS0wXMiIiLYuXMnY8eOLXA8tVrNsWPHePzxx2nQoAFjx44lKCiIf//9t2J1c91LrfZQs51uMPS+b8wXhyYHzm3R3W/Qs8SHWXvkGgDdm3pXiHoyQghRWmoLFZ8NaUm3Rl5k5Wp5ZtEB/TIeouIzex2gishsdYDuFP4HLH8KbF10i6TamOG/VRNUf87RaGn34VZuZOTw69hgOtV3L4NAhRDCPDJzNIz7+QD/no3HydaSpePa07SGi7nDqpKM+fw2ewuQuIvAXlD91tpbh34xTwx53V/1Hyvx0hf/nr3OjYwc3B1t6FC38FpMQghRWdlaqfnm6SDa+lcjNTOXkIX7OBeXeu8nCrOSBKgis1BDxxd19//7StcdVd4iNuq+Nij5+J+82V99W/jI1GIhxH3J3tqSH0a1pXlNFxLTsxn+3V4uJ6Tf+4nCbCQBquhaDNOtEJ98BU6uKd9zx5/TLX1hYQV1HynRITKyc9l8UjeIu3/LGqaMTgghKhRnWysWjW5HoJcTcalZDP9uL1F3WRtNmJckQBWdlS20u1V4cNfnUJ5Dts7cav3x71TipS+2nIrlZo4Gfzd7mteUPnEhxP2tmoM1vzzTjgB3B64l3WTE93uJS800d1iiEJIAVQZtx4KVPcQehwvby++8EXnT30sz+0vX/fV4yxoVdmFNIYQwJU8nWxY/E0wNVzsuxqfz9Pf7uJGebe6wxB0kAaoM7KtD6xDd/V3zyuecJqj+nJiezT9nrgNS/FAIUbX4utqxZFwwnk42RMSmMvLHfaRmmmEcpyiSJECVRfvnQaXWtQBFHy3785mg+vP649HkahWa1XChriwoKYSoYmq7ObD4mWCqO1hz7GoyY37aT0Z2rrnDErdIAlRZVKsNTfrr7u8uh+UxTFD9ed2t4ofS+iOEqKrqeznx85h2ONlasv/SDZ795SCZORpzhyWQBKhyyVse48RqSIosu/OYoPrz1RsZ7L90A5UK+jSXBEgIUXU1reHCT6PbYW+t5t+z8UxccpgcjdbcYVV5kgBVJr4tIaCzrmvqv6/L7jyR/+mKL9q7Qc02JTrEH0d1y5e0D3DD28XWlNEJIUSlE1S7Gt+PbIONpQVbw2MZ9/MBNh6PlsHRZmRp7gCEkR6YBBd3wMFF0Pl1sKtm+nOYoPpz3tpf/VtJ648QQgB0rOvOgqeCGP/LAf6OuM7fEddRqaCRtzMd67rRsZ4bbf2r42RrZe5QqwRJgCqbul3BqynEnoD9P8BDr5r+HHkJUAmrP5+OSeF0TCrWagt6NPExYWBCCFG5dWnoycrnOvL74WvsOZ9ARGwqp6JTOBWdwvc7L6K2UNG8posuIarrTlDtarKAdBmRBKiyUal0Y4F+Hw97v4EOE3XFEk0l/hwknCtV9ed1t2r/PBzogYu9/CcjhBD5tfBzpYWfKwDXU7P470ICu88nsOd8PJcSMjgcmcThyCS+3H4ea7UFrWu70rGuOx3rutG8pivWljJ6xRQkAaqMmj4BYe9BylU4tgyCRpnu2KWs/qzVKvrih/1bydIXQghxNx5ONvRt4UvfFrrhAteSbrLnfAK7z8ez+1wCMSmZ/Hchkf8uJDJnC9hbq2nrX13fQtTY11nWWCwhSYAqI7UVtJ8Am9+B3V9AqxCwMNF/BKWs/nwo8gbXkm7iaGPJIw09TROTEEJUETVc7RgYVJOBQTVRFIVLCRm6ZOh8AnvOJ5CYns2OM9fZcavIrLOtJe3ruN0aQ+ROfU9HqbpfTJIAVVZBI2HHx5BwVtdq07B36Y9pgurPea0/3Zt4S7+1EEKUgkqlIsDdgQB3B0YE10arVTgTl8ruc7ous70XEkjJzGXzqVg2n9ItOu3uaE2HW91lHeq4UdvNXhKiIkgCVFnZOEHbMbDzM93yGKZIgM6Flar6c45Gy/rjuunvUvxQCCFMy8JCRUNvZxp6OzOmUwC5Gi0no1LYfavLbP+lROLTsvnjaBR/HNX9M+rrYqtPiDrWc8PHxc7MV1FxSAJUmQU/B3u+hCv/QeReqBVcuuNF3Br/U8LqzzvPxpOYno27ow0d67qVLhYhhBB3Zam20A+onvBwXbJyNRy9kqzvMjsceYOo5ExWHbrKqkNXAQhwd6BDXTc6N/Dg0UZeWFTh8UOSAFVmTt7QfAgc/gV2z4Nai0t+LBNUf86r/dOnuQ+WapmlIIQQ5cnGUk27gOq0C6jOS93gZraGA5cTb7UQJXD8ahIX49O5GJ/Okr2RtKldjZlPNKO+l5O5QzcLSYAqu44v6hKg0+sh/iy41y/ZcUpZ/TkjO1ffBy3dX0IIYX521moerO/Bg/U9AEjJzGHfhUR2notnxYErHLh8g17z/uX5h+vxfJe62FhWrXGb8m96ZecReKvFRindIqmlrP685VQsGdkaarvZ0/JWfQshhBAVh7OtFd0aezHt8SZsDu1M14ae5GgUPg87S6/P/2X/pURzh1iuJAG6Hzxwa5HUo8sgLa5kxyhl9ee84of9WvjKjAMhhKjgarja8f3INnwxvBXujjacv57OoAV7ePv346Rk5pg7vHIhCdD9oFYHqNkWNFm66tDGKmX15xu36lIAPC7dX0IIUSmoVCr6NPclLLQzQ9v6AbBkbyTdPt3BphPRZo6u7EkCdD/IWx4DYP/3kJVm3PNLWf15w4locrUKTXydqedZNQfTCSFEZeVib8VHTzZn6bj21HF3IC41i+d+PcT4nw8Qk5xp7vDKjNEJ0IULF8oiDlFaDXtD9TqQmaQbFG2MiNJ1f609fGvpi5ay9IUQQlRWHeq6sWHyg0zsUg9LCxWbT8XSbc4OftlzCa1WMXd4Jmd0AlSvXj26dOnCr7/+Smbm/ZsZVjoWat2MMIA9X4Emt3jPy1/9uQT1f64l3WTfpURUKujTQlZ+F0KIyszWSs2r3QP5c1InWvq5kpaVy7trTzLomz2ciU01d3gmZXQCdOjQIZo3b05oaCje3t48++yz7Nu3ryxiE8ZqMQzs3SE5Ek6tKd5zSln9Oa/aaHBAdakwKoQQ94mG3s6smtCR6Y83wcFazcHLN+g971/mbI4gK1dj7vBMwugEqGXLlnz++edERUWxcOFCoqOj6dSpE02bNmXOnDlcv369LOIUxWFlB8HP6u7vmgtKMZosS1n9ec1hXfHDftL9JYQQ9xW1hYqRHf3ZEtqZbo10U+bnbTtHz8//Zd/Fyj9lvsSDoC0tLXniiSdYsWIFs2bN4ty5c7z66qv4+fkREhJCdPT9P4K8Qmr7DFjZQ8xxuPD33fctZfXniJhUTsekYqVW0aupdH8JIcT9yNfVju9C2vDl8Na4O9pw4Xo6g7/Zw1urj5N8s/JOmS9xAnTgwAGef/55fHx8mDNnDq+++irnz59ny5YtREVF0a9fP1PGKYrLvjq0elp3f/e8u+97ZW+pqj+vO6pr/Xk40BMXeyujny+EEKJyUKlU9G7uQ1hoZ4a1002ZX7ovkkfn7GDj8WiU4vQ4VDBGJ0Bz5syhWbNmdOzYkaioKH7++WcuX77MBx98QEBAAA8++CA//fQThw4dKot4RXF0eB5UFnB+G0QfK3q/vO6vElR/VhSFtXnFD6X2jxBCVAku9lbMfKI5y8bfnjI/YfEhxv18kOjkm+YOzyhGJ0Bff/01w4cP5/Lly6xZs4Y+ffpgYWF4GE9PT3744QeTBSmMVM0fGvfX3b/b8hilqP58KPIGV2/cxMFaTdeGXkY/XwghROXVvo5uyvyLj+imzG8Nj+XROf/wcyWaMq9SKmO7VRlLSUnBxcWF5ORknJ2NLwxYIUQdhm8fBpUaJh8FVz/Dx+PPwRdBuurPr18wugDilLUn+HnPZZ5oXYM5g1uaLGwhhBCVS0RMKm+uPsbhyCQAWtdy5aMnm9PADKvMG/P5bXQL0I8//siKFSsKbF+xYgWLFi0y9nCirPi2goCHdFPc//u64OOlqP6co9Gy/phukLvM/hJCiKot0NuJlc915L1+TXC0seRQZJJ+ynxmTsWdMm90AjRz5kzc3d0LbPf09GTGjBkmCUqYSMfJuq8Hf9IVPMzvzF+6ryXo/tp5Lp6E9GzcHa15oK5b6WIUQghR6aktVIR08GdL6EN0a+SlnzLfa96/7L2QYO7wCmV0AhQZGUlAQECB7bVr1yYyMtIkQQkTqdcVPJtATjocWHh7+80bcHm37n4J6v/krfzep7kvlmpZTk4IIYSOj4sd34UE8dWI1ng46abMD/n2P95afazCTZk3+tPL09OTY8cKziw6evQobm7SGlChqFTwwK1FUvd+Azm3li4pRfXnm9ka/joZA8jK70IIIQpSqVT0aubD1tDODGtXC4Cl+67Qbc4O1h+rOFPmjU6Ahg0bxqRJk9i+fTsajQaNRsO2bduYPHkyQ4cOLYsYRWk0fRKca0BaLBxbrttWiurPW8NjycjW4FfdjlZ+rqaLUwghxH3Fxc6KmU80Y/n49tTxcOB6ahYvLDnEuJ8PEJVk/inzRidA77//PsHBwXTt2hU7Ozvs7Ox47LHHeOSRR2QMUEWktoL2E3T3d8+H3Ox81Z+NT4DWHrm19EWLGqhUKlNFKYQQ4j4VXMeNDZMeZNIj9bBSq9gaHsejc3bw63+XzRqX0QmQtbU1y5cv5/Tp0yxevJjVq1dz/vx5Fi5ciLW1dVnEKEqr9UiwcYGEsxA2PV/157ZGHeZGejZ/R+jWeuvfSrq/hBBCFI+tlZrQxwJZP+lBWtdyJT1bQ1xqllljsizpExs0aECDBg1MGYsoK7bO0Ga0boHUPV/otpWg+vPGEzHkahUa+zhTz7P86zsIIYSo3Bp46abMrz58jb4tzLuGZIkSoKtXr7Ju3ToiIyPJzs42eGzOnDkmCUyYWPBzsOdL0N4ahV+C7q81ed1fMvhZCCFECVlYqBgYVNPcYRifAIWFhfH4449Tp04dTp8+TdOmTbl06RKKotC6deuyiFGYgrMPtBgCh3/VVX+u+4hRT49Kusm+i4moVNC3hSRAQgghKjejxwC99dZbvPrqqxw/fhxbW1tWrVrFlStX6Ny5M4MGDSqLGIWpdAoFu+rQYqjR1Z//OKqr/dPOvzq+rnZlEZ0QQghRboxOgMLDwwkJCQHA0tKSmzdv4ujoyHvvvcesWbNKFMSXX36Jv78/tra2BAcHs2/fviL3ffjhh1GpVAVuvXv31u+jKApTpkzBx8cHOzs7unXrxtmzZ0sU233FrS68dh76fWH0U2+v/C5LXwghhKj8jE6AHBwc9ON+fHx8OH/+vP6x+Ph4owNYvnw5oaGhTJ06lUOHDtGiRQu6d+9OXFxcofuvXr2a6Oho/e3EiROo1WqD1qePP/6YefPmsWDBAvbu3YuDgwPdu3cnMzPT6PjuOxbGV24+G5vKqegUrNQqejXzLoOghBBCiPJl9Kdh+/bt2blzJwC9evXilVde4cMPP2TMmDG0b9/e6ADmzJnDuHHjGD16NI0bN2bBggXY29uzcOHCQvevXr063t7e+tuWLVuwt7fXJ0CKojB37lz+97//0a9fP5o3b87PP/9MVFQUa9asMTo+cbv1p3MDT1ztpdSBEEKIys/oBGjOnDkEBwcDMH36dLp27cry5cvx9/fnhx9+MOpY2dnZHDx4kG7dut0OyMKCbt26sWfPnmId44cffmDo0KE4ODgAcPHiRWJiYgyO6eLiQnBwcLGPKW5TFIW1R2X2lxBCiPuLUbPANBoNV69epXnz5oCuO2zBggUlPnl8fDwajQYvLy+D7V5eXpw+ffqez9+3bx8nTpwwSLxiYmL0x7jzmHmP3SkrK4usrNsFmVJSUop9Dfe7Q5FJXEm8ib21mm6NvO79BCGEEKISMKoFSK1W89hjj3Hjxo2yiscoP/zwA82aNaNdu3alOs7MmTNxcXHR3/z8/EwUYeW37lbtn+5NvLGzNq5wohBCCFFRGd0F1rRpUy5cuGCSk7u7u6NWq4mNjTXYHhsbi7f33Qfbpqens2zZMsaOHWuwPe95xhzzrbfeIjk5WX+7cuWKsZdyX8rVaPnzWDQg3V9CCCHuL0YnQB988AGvvvoqf/75J9HR0aSkpBjcjGFtbU1QUBBhYWH6bVqtlrCwMDp06HDX565YsYKsrCyeeuopg+0BAQF4e3sbHDMlJYW9e/cWeUwbGxucnZ0NbgJ2nU8gIT0bNwdrHqjnbu5whBBCCJMxuhJ0r169AHj88ccNVgNXFAWVSoVGozHqeKGhoYwcOZI2bdrQrl075s6dS3p6OqNHjwYgJCSEGjVqMHPmTIPn/fDDD/Tv3x83NzeD7SqVipdeeokPPviA+vXrExAQwLvvvouvry/9+/c39nKrtLWHdd1fvZv7YKU2fvq8EEIIUVEZnQBt377dpAEMGTKE69evM2XKFGJiYmjZsiWbNm3SD2KOjIzE4o7aNREREezcuZPNmzcXeszXX3+d9PR0xo8fT1JSEp06dWLTpk3Y2tqaNPb72c1sDX+d1A0al+4vIYQQ9xuVoiiKuYOoaFJSUnBxcSE5ObnKdof9eSyKiUsOU7OaHf++3sWgtU8IIYSoiIz5/Da6Beiff/656+MPPfSQsYcUFdCaw3lLX/hK8iOEEOK+Y3QC9PDDDxfYlv8D0tgxQKLiScrIZscZ3VIksvaXEEKI+5HRI1tv3LhhcIuLi2PTpk20bdu2yDE5onLZeCKGHI1CIx9nGng5mTscIYQQwuSMbgFycXEpsO3RRx/F2tqa0NBQDh48aJLAhPmsPSJLXwghhLi/mWxus5eXFxEREaY6nDCT6OSb7L2YCEDfFpIACSGEuD8Z3QJ07Ngxg+8VRSE6OpqPPvqIli1bmiouYSZ/HI1CUaCdf3VquNqZOxwhhBCiTBidALVs2RKVSsWds+fbt2/PwoULTRaYMI+1R27N/molrT9CCCHuX0YnQBcvXjT43sLCAg8PDykyeB84F5fKyagULC1U9GrqY+5whBBCiDJjdAJUu3btsohDVAB5rT+dG3hQzcHazNEIIYQQZcfoQdCTJk1i3rx5BbZ/8cUXvPTSS6aISZiBoij6BOhxmf0lhBDiPmd0ArRq1SoeeOCBAts7duzIypUrTRKUKH9HriQRmZiBvbWaRxt7mTscIYQQokwZnQAlJCQUWgvI2dmZ+Ph4kwQlyl9e689jjb2wtza6Z1QIIYSoVIxOgOrVq8emTZsKbN+4cSN16tQxSVCifOVqtPx5LG/tL1n6QgghxP3P6H/1Q0NDmThxItevX+eRRx4BICwsjE8//ZS5c+eaOj5RDnafTyA+LZvqDtZ0qu9u7nCEEEKIMmd0AjRmzBiysrL48MMPef/99wHw9/fn66+/JiQkxOQBirK35tbSF72b+WClNllxcCGEEKLCKtFgjwkTJjBhwgSuX7+OnZ0djo6Opo5LlJPMHA1/nYgBZO0vIYQQVUeJCiHm5uZSv359PDw89NvPnj2LlZUV/v7+poxPlLGw8DjSszXUcLWjda1q5g5HCCGEKBdG93eMGjWK3bt3F9i+d+9eRo0aZYqYRDlak2/ldwsLlZmjEUIIIcqH0QnQ4cOHC60D1L59e44cOWKKmEQ5Sc7I4e+IOEBmfwkhhKhajE6AVCoVqampBbYnJyej0WhMEpQoHxtPRJOjUWjo7USgt5O5wxFCCCHKjdEJ0EMPPcTMmTMNkh2NRsPMmTPp1KmTSYMTZUuWvhBCCFFVGT0IetasWTz00EMEBgby4IMPAvDvv/+SkpLCtm3bTB6gKBsxyZn8dzEBgMdbSAIkhBCiajG6Bahx48YcO3aMwYMHExcXR2pqKiEhIZw+fZqmTZuWRYyiDPxxNApFgbb+1ahZzd7c4QghhBDlqkR1gHx9fZkxY4bBtqSkJL744gsmTpxoksBE2Vp7VDf763EZ/CyEEKIKKnXZ37CwMIYPH46Pjw9Tp041RUyijJ2LS+PEtRQsLVT0buZj7nCEEEKIcleiBOjKlSu89957BAQE8NhjjwHw+++/ExMTY9LgRNlYd6v2z0MNPKjuYG3maIQQQojyV+wEKCcnhxUrVtC9e3cCAwM5cuQIs2fPxsLCgv/973/06NEDKyursoxVmICiKKw9mrfyuwx+FkIIUTUVewxQjRo1aNiwIU899RTLli2jWjXdsgnDhg0rs+CE6R29mszlhAzsrNR0a+Rl7nCEEEIIsyh2C1Bubi4qlQqVSoVarS7LmEQZWnNY1/31WBMvHGxKNAZeCCGEqPSKnQBFRUUxfvx4li5dire3N08++SS///47KpWsH1VZ5Gq0/HksGpDuLyGEEFVbsRMgW1tbRowYwbZt2zh+/DiNGjVi0qRJ5Obm8uGHH7JlyxZZCqOCOxSZRHxaFq72VjxY38Pc4QghhBBmU6JZYHXr1uWDDz7g8uXLrF+/nqysLPr06YOXl4wpqciOX0sGoJ1/dazUpa6AIIQQQlRapRoEYmFhQc+ePenZsyfXr1/nl19+MVVcogyER6cA0NjX2cyRCCGEEOZlsmYADw8PQkNDTXU4UQZORekSoEY+kgAJIYSo2qQfpIrIztVyLi4NgMaSAAkhhKjiJAGqIs5fTyNbo8XJ1pKa1ezMHY4QQghhVpIAVRH5u7+kdIEQQoiqThKgKkI/AFq6v4QQQgjjZ4FpNBp++uknwsLCiIuLQ6vVGjy+bds2kwUnTOeUJEBCCCGEntEJ0OTJk/npp5/o3bs3TZs2le6USkBRFJkCL4QQQuRjdAK0bNkyfvvtN3r16lUW8YgyEJOSyY2MHCwtVNTzdDR3OEIIIYTZGT0GyNramnr16pVFLKKM5LX+1PVwxNZKFrIVQgghjE6AXnnlFT7//HMURSmLeEQZyJsBJt1fQgghhI7RXWA7d+5k+/btbNy4kSZNmmBlZWXw+OrVq00WnDCNvAHQjXyczByJEEIIUTEYnQC5uroyYMCAsohFlJHw6FQAGvu4mDkSIYQQomIwOgH68ccfTRrAl19+yezZs4mJiaFFixbMnz+fdu3aFbl/UlIS77zzDqtXryYxMZHatWszd+5c/aDsadOmMX36dIPnBAYGcvr0aZPGXVmkZ+VyKSEdkBYgIYQQIk+JV4O/fv06ERERgC7B8PDwMPoYy5cvJzQ0lAULFhAcHMzcuXPp3r07EREReHp6Ftg/OzubRx99FE9PT1auXEmNGjW4fPkyrq6uBvs1adKErVu36r+3tCzVoveV2umYVBQFvJxtcHO0MXc4QgghRIVgdGaQnp7Oiy++yM8//6wvgqhWqwkJCWH+/PnY29sX+1hz5sxh3LhxjB49GoAFCxawfv16Fi5cyJtvvllg/4ULF5KYmMju3bv1Y4/8/f0LXpSlJd7e3sZe2n1JCiAKIYQQBRk9Cyw0NJQdO3bwxx9/kJSURFJSEmvXrmXHjh288sorxT5OdnY2Bw8epFu3breDsbCgW7du7Nmzp9DnrFu3jg4dOvDCCy/g5eVF06ZNmTFjBhqNxmC/s2fP4uvrS506dRgxYgSRkZF3jSUrK4uUlBSD2/0i/xpgQgghhNAxOgFatWoVP/zwAz179sTZ2RlnZ2d69erFd999x8qVK4t9nPj4eDQaDV5eXgbbvby8iImJKfQ5Fy5cYOXKlWg0GjZs2MC7777Lp59+ygcffKDfJzg4mJ9++olNmzbx9ddfc/HiRR588EFSU1OLjGXmzJm4uLjob35+fsW+jopOKkALIYQQBRndBZaRkVEgaQHw9PQkIyPDJEEVRavV4unpybfffotarSYoKIhr164xe/Zspk6dCkDPnj31+zdv3pzg4GBq167Nb7/9xtixYws97ltvvUVoaKj++5SUlPsiCdJoFU7HSAuQEEIIcSejW4A6dOjA1KlTyczM1G+7efMm06dPp0OHDsU+jru7O2q1mtjYWIPtsbGxRY7f8fHxoUGDBqjVt6sZN2rUiJiYGLKzswt9jqurKw0aNODcuXNFxmJjY6Nvzcq73Q8uJaSTmaPFzkqNv5uDucMRQgghKgyjE6DPP/+cXbt2UbNmTbp27UrXrl3x8/Nj9+7dfP7558U+jrW1NUFBQYSFhem3abVawsLCikykHnjgAc6dO2ewAv2ZM2fw8fHB2tq60OekpaVx/vx5fHx8ih3b/SJv/E9DHyfUFrJorRBCCJHH6ASoadOmnD17lpkzZ9KyZUtatmzJRx99xNmzZ2nSpIlRxwoNDeW7775j0aJFhIeHM2HCBNLT0/WzwkJCQnjrrbf0+0+YMIHExEQmT57MmTNnWL9+PTNmzOCFF17Q7/Pqq6+yY8cOLl26xO7duxkwYABqtZphw4YZe6mV3u0K0PdHi5YQQghhKiUqkGNvb8+4ceNKffIhQ4Zw/fp1pkyZQkxMDC1btmTTpk36MUaRkZFYWNzO0fz8/Pjrr794+eWXad68OTVq1GDy5Mm88cYb+n2uXr3KsGHDSEhIwMPDg06dOvHff/+VqE5RZRcuU+CFEEKIQqmUYqxqum7dOnr27ImVlRXr1q27676PP/64yYIzl5SUFFxcXEhOTq7U44HafbiVuNQsVk3oSFDtauYORwghhChTxnx+F6sFqH///sTExODp6Un//v2L3E+lUhWoySPMIz4ti7jULFQqaOgtS2AIIYQQ+RUrAco/6Dj/fVFx5XV/Bbg54GBTdZcCEUIIIQpj9CDon3/+maysrALbs7Oz+fnnn00SlCi9cBkALYQQQhTJ6ARo9OjRJCcnF9iempqqn70lzC9vCrxUgBZCCCEKMjoBUhQFlapgTZmrV6/i4uJikqBE6d2eAi/jf4QQQog7FXtwSKtWrVCpVKhUKrp27Yql5e2najQaLl68SI8ePcokSGGczBwN56+nA9DYR5JSIYQQ4k7FToDyZn8dOXKE7t274+joqH/M2toaf39/nnzySZMHKIx3NjYNjVahmr0VXs425g5HCCGEqHCKnQDlLTbq7+/PkCFDsLW1LbOgROnkXwG+sO5KIYQQoqozen70yJEjyyIOYUKnpAK0EEIIcVdGJ0AajYbPPvuM3377jcjIyAKrsCcmJposOFEyeTPAZAq8EEIIUTijZ4FNnz6dOXPmMGTIEJKTkwkNDeWJJ57AwsKCadOmlUGIwhiKohh0gQkhhBCiIKMToMWLF/Pdd9/xyiuvYGlpybBhw/j++++ZMmUK//33X1nEKIxw9cZNUrNysVZbUNfD8d5PEEIIIaogoxOgmJgYmjVrBoCjo6O+KGKfPn1Yv369aaMTRssb/1PfyxErtdE/XiGEEKJKMPoTsmbNmkRHRwNQt25dNm/eDMD+/fuxsZEp1+amrwAt43+EEEKIIhmdAA0YMICwsDAAXnzxRd59913q169PSEgIY8aMMXmAwjinZA0wIYQQ4p6MngX20Ucf6e8PGTKEWrVqsWfPHurXr0/fvn1NGpwwngyAFkIIIe7N6AToTh06dKBDhw6miEWUUvLNHK7euAlAI29JgIQQQoiiFCsBWrduXbEP+Pjjj5c4GFE6p2+1/tRwtcPF3srM0QghhBAVV7ESoLx1wPKoVCoURSmwDXSFEoV5nJLuLyGEEKJYijUIWqvV6m+bN2+mZcuWbNy4kaSkJJKSkti4cSOtW7dm06ZNZR2vuItwGQAthBBCFIvRY4BeeuklFixYQKdOnfTbunfvjr29PePHjyc8PNykAYrikzXAhBBCiOIxehr8+fPncXV1LbDdxcWFS5cumSAkURI5Gi1nYtIASYCEEEKIezE6AWrbti2hoaHExsbqt8XGxvLaa6/Rrl07kwYniu/C9XSyNVqcbCypWc3O3OEIIYQQFZrRCdDChQuJjo6mVq1a1KtXj3r16lGrVi2uXbvGDz/8UBYximI4Fa1bkqShjxMWFiozRyOEEEJUbEaPAapXrx7Hjh1jy5YtnD59GoBGjRrRrVs3/UwwUf7Co1MB6f4SQgghiqNEhRBVKhWPPfYYjz32mKnjESWkXwNMpsALIYQQ91SsBGjevHmMHz8eW1tb5s2bd9d9J02aZJLARPEpiiJrgAkhhBBGUCl3VjQsREBAAAcOHMDNzY2AgICiD6ZSceHCBZMGaA4pKSm4uLiQnJyMs3PFTyhiUzIJnhGG2kLFyendsbVSmzskIYQQotwZ8/ldrBagixcvFnpfVAx53V913B0k+RFCCCGKwehZYKLikSUwhBBCCOMUqwUoNDS02AecM2dOiYMRJSMVoIUQQgjjFCsBOnz4cLEOJtPgzSM8SgZACyGEEMYoVgK0ffv2so5DlFBGdi4XE9IBSYCEEEKI4pIxQJXc6ZhUFAU8nGzwcLIxdzhCCCFEpVCiQogHDhzgt99+IzIykuzsbIPHVq9ebZLARPGEy/gfIYQQwmhGtwAtW7aMjh07Eh4ezu+//05OTg4nT55k27ZtuLi4lEWM4i6kArQQQghhPKMToBkzZvDZZ5/xxx9/YG1tzeeff87p06cZPHgwtWrVKosYxV1IBWghhBDCeEYnQOfPn6d3794AWFtbk56ejkql4uWXX+bbb781eYCiaBqtQkSMLIIqhBBCGMvoBKhatWqkpuo+dGvUqMGJEycASEpKIiMjw7TRibu6nJBORrYGWysLAtwdzB2OEEIIUWkYPQj6oYceYsuWLTRr1oxBgwYxefJktm3bxpYtW+jatWtZxCiKEB6tS0QDvZ1RW0gNJiGEEKK4ip0AnThxgqZNm/LFF1+QmZkJwDvvvIOVlRW7d+/mySef5H//+1+ZBSoKOhWdDEBjHyczRyKEEEJULsVOgJo3b07btm155plnGDp0KAAWFha8+eabZRacuLu8FiAZ/yOEEEIYp9hjgHbs2EGTJk145ZVX8PHxYeTIkfz7779lGZu4B5kCL4QQQpRMsROgBx98kIULFxIdHc38+fO5dOkSnTt3pkGDBsyaNYuYmJiyjFPcITE9m5gUXVdkoLckQEIIIYQxjJ4F5uDgwOjRo9mxYwdnzpxh0KBBfPnll9SqVYvHH3/c6AC+/PJL/P39sbW1JTg4mH379t11/6SkJF544QV8fHywsbGhQYMGbNiwoVTHrIzyKkD7u9njaFOigt5CCCFElVWqtcDq1avH22+/zf/+9z+cnJxYv369Uc9fvnw5oaGhTJ06lUOHDtGiRQu6d+9OXFxcoftnZ2fz6KOPcunSJVauXElERATfffcdNWrUKPExK6tTsgK8EEIIUWIlToD++ecfRo0ahbe3N6+99hpPPPEEu3btMuoYc+bMYdy4cYwePZrGjRuzYMEC7O3tWbhwYaH7L1y4kMTERNasWcMDDzyAv78/nTt3pkWLFiU+ZmUla4AJIYQQJWdUAhQVFcWMGTNo0KABDz/8MOfOnWPevHlERUXx3Xff0b59+2IfKzs7m4MHD9KtW7fbwVhY0K1bN/bs2VPoc9atW0eHDh144YUX8PLyomnTpsyYMQONRlPiYwJkZWWRkpJicKvo8pbAkAHQQgghhPGKPXikZ8+ebN26FXd3d0JCQhgzZgyBgYElPnF8fDwajQYvLy+D7V5eXpw+fbrQ51y4cIFt27YxYsQINmzYwLlz53j++efJyclh6tSpJTomwMyZM5k+fXqJr6W8ZeVqOBeXBkgXmBBCCFESxU6ArKysWLlyJX369EGtVpdlTEXSarV4enry7bffolarCQoK4tq1a8yePZupU6eW+LhvvfUWoaGh+u9TUlLw8/MzRchl4mxsGrlaBVd7K3xcbM0djhBCCFHpFDsBWrdunUlP7O7ujlqtJjY21mB7bGws3t7ehT7Hx8cHKysrgwSsUaNGxMTEkJ2dXaJjAtjY2GBjY1OKqylf+hXgvZ1RqWQJDCGEEMJYpZoFVhrW1tYEBQURFham36bVagkLC6NDhw6FPueBBx7g3LlzaLVa/bYzZ87g4+ODtbV1iY5ZGYXL+B8hhBCiVMyWAAGEhoby3XffsWjRIsLDw5kwYQLp6emMHj0agJCQEN566y39/hMmTCAxMZHJkydz5swZ1q9fz4wZM3jhhReKfcz7gb4CtIz/EUIIIUrErBX0hgwZwvXr15kyZQoxMTG0bNmSTZs26QcxR0ZGYmFxO0fz8/Pjr7/+4uWXX6Z58+bUqFGDyZMn88YbbxT7mJWdoii3u8AkARJCCCFKRKUoimLuICqalJQUXFxcSE5Oxtm5YiUZV29k0GnWdqzUKk5O74G1pVkb8YQQQogKw5jPb/n0rGTyur/qeTpJ8iOEEEKUkHyCVjLh0amAjP8RQgghSkMSoErmVHQyAI18nMwciRBCCFF5SQJUyehbgGQKvBBCCFFikgBVIimZOUQmZgDSBSaEEEKUhiRAlcjpW60/vi62uNpbmzkaIYQQovKSBKgSkQrQQgghhGlIAlSJ5E2BlwKIQgghROlIAlSJhMfIEhhCCCGEKUgCVEnkarScjpEZYEIIIYQpSAJUSVyITyc7V4uDtRq/avbmDkcIIYSo1CQBqiTC8y2AamGhMnM0QgghROUmCVAlIQOghRBCCNORBKiSOCVT4IUQQgiTkQSoElAURd8CJDPAhBBCiNKTBKgSuJ6aRUJ6NhYqCPSWRVCFEEKI0pIEqBLI6/6q4+GIrZXazNEIIYQQlZ8kQJXAqWgZAC2EEEKYkiRAlUD4rUVQZfyPEEIIYRqSAFUCp6KSAWjkI+N/hBBCCFOQBKiCu5mt4WJ8OiBT4IUQQghTkQSogouITUWrgLujDZ5OtuYORwghhLgvSAJUwd2uAC3dX0IIIYSpSAJUwYVLBWghhBDC5CQBquD0S2DIDDAhhBDCZCQBqsC0WoXTkgAJIYQQJicJUAUWmZhBerYGG0sLAtwdzB2OEEIIcd+QBKgCy+v+CvR2wlItPyohhBDCVORTtQILl+4vIYQQokxIAlSB3Z4CLwmQEEIIYUqSAFVgMgVeCCGEKBuSAFVQN9KziUrOBKChtxRBFEIIIUxJEqAKKq/1p1Z1e5xsrcwcjRBCCHF/kQSogpICiEIIIUTZkQSogspLgGQAtBBCCGF6kgBVUOHRqYAMgBZCCCHKgiRAFVB2rpZzcboESFaBF0IIIUxPEqAK6FxcGjkaBWdbS2q42pk7HCGEEOK+IwlQBXQqX/0flUpl5miEEEKI+48kQBWQVIAWQgghypYkQBWQrAEmhBBClC1JgCoYRVFkCrwQQghRxiQBqmCikzNJvpmDpYWK+l6O5g5HCCGEuC9JAlTB5I3/qefpiI2l2szRCCGEEPenCpEAffnll/j7+2Nra0twcDD79u0rct+ffvoJlUplcLO1tTXYZ9SoUQX26dGjR1lfhknIEhhCCCFE2bM0dwDLly8nNDSUBQsWEBwczNy5c+nevTsRERF4enoW+hxnZ2ciIiL03xc2VbxHjx78+OOP+u9tbGxMH3wZCM83BV4IIYQQZcPsLUBz5sxh3LhxjB49msaNG7NgwQLs7e1ZuHBhkc9RqVR4e3vrb15eXgX2sbGxMdinWrVqZXkZJiMDoIUQQoiyZ9YEKDs7m4MHD9KtWzf9NgsLC7p168aePXuKfF5aWhq1a9fGz8+Pfv36cfLkyQL7/P3333h6ehIYGMiECRNISEgok2swpbSsXC4nZACSAAkhhBBlyawJUHx8PBqNpkALjpeXFzExMYU+JzAwkIULF7J27Vp+/fVXtFotHTt25OrVq/p9evTowc8//0xYWBizZs1ix44d9OzZE41GU+gxs7KySElJMbiZw+lbrT8+LrZUd7A2SwxCCCFEVWD2MUDG6tChAx06dNB/37FjRxo1asQ333zD+++/D8DQoUP1jzdr1ozmzZtTt25d/v77b7p27VrgmDNnzmT69OllH/w9SPeXEEIIUT7M2gLk7u6OWq0mNjbWYHtsbCze3t7FOoaVlRWtWrXi3LlzRe5Tp04d3N3di9znrbfeIjk5WX+7cuVK8S/ChKQCtBBCCFE+zJoAWVtbExQURFhYmH6bVqslLCzMoJXnbjQaDcePH8fHx6fIfa5evUpCQkKR+9jY2ODs7GxwMwdZA0wIIYQoH2afBRYaGsp3333HokWLCA8PZ8KECaSnpzN69GgAQkJCeOutt/T7v/fee2zevJkLFy5w6NAhnnrqKS5fvswzzzwD6AZIv/baa/z3339cunSJsLAw+vXrR7169ejevbtZrrE4cjVaTsekAjIFXgghhChrZh8DNGTIEK5fv86UKVOIiYmhZcuWbNq0ST8wOjIyEguL23najRs3GDduHDExMVSrVo2goCB2795N48aNAVCr1Rw7doxFixaRlJSEr68vjz32GO+//36FrgV0KSGdrFwt9tZqale3N3c4QgghxH1NpSiKYu4gKpqUlBRcXFxITk4ut+6wdUejmLT0MK1rubL6+QfK5ZxCCCHE/cSYz2+zd4EJnbzxP9L9JYQQQpQ9s3eBCR2ZAi+EuBeNRkNOTo65wxDCbKysrFCrTbNQuCRAFYRMgRdCFEVRFGJiYkhKSjJ3KEKYnaurK97e3oWuA2oMSYAqgOupWVxPzUKlgkBvJ3OHI4SoYPKSH09PT+zt7Uv9xi9EZaQoChkZGcTFxQHctfxNcUgCVAHktf4EuDtgby0/EiHEbRqNRp/8uLm5mTscIczKzs4OgLi4ODw9PUvVHSaDoCuAU9L9JYQoQt6YH3t7KY8hBNz+WyjteDhJgCoAqQAthLgX6fYSQsdUfwuSAFUA+gHQMgVeCCGK5O/vz9y5c4u9/99//41KpSqXweNr1qyhXr16qNVqXnrppTI/3/2iPH9Gd5IEyMwyczScv54GSBeYEOL+oFKp7nqbNm1aiY67f/9+xo8fX+z9O3bsSHR0NC4uLiU6nzGeffZZBg4cyJUrV3j//ffL/HwV0U8//YSrq6u5wyg2GXFrZmdiU9Eq4OZgjadTxV2qQwghiis6Olp/f/ny5UyZMoWIiAj9NkdHR/19RVHQaDRYWt7748jDw8OoOKytrfH29jbqOSWRlpZGXFwc3bt3x9fXt9B9NBoNKpXKYGmnyiI7Oxtra2tzh2Fyle8ncZ/JXwFa+viFEPcDb29v/c3FxQWVSqX//vTp0zg5ObFx40aCgoKwsbFh586dnD9/nn79+uHl5YWjoyNt27Zl69atBse9swtMpVLx/fffM2DAAOzt7alfvz7r1q3TP35n90peC8Vff/1Fo0aNcHR0pEePHgYJW25uLpMmTcLV1RU3NzfeeOMNRo4cSf/+/Qu91r///hsnJ135kkceeQSVSsXff/+tP9e6deto3LgxNjY2REZGcuPGDUJCQqhWrRr29vb07NmTs2fP6o+X97w///yTwMBA7O3tGThwIBkZGSxatAh/f3+qVavGpEmT0Gg0Rf4Mjh49SpcuXXBycsLZ2ZmgoCAOHDigf3zXrl08/PDD2NvbU61aNbp3786NGzcAePjhh5k4cSIvvfQS7u7u+oXE58yZQ7NmzXBwcMDPz4/nn3+etLQ0/eswevRokpOTC7T0ZWVl8cYbb+Dn54eNjQ316tXjhx9+MIj34MGDtGnTBnt7ezp27GiQMJcVSYDMTCpACyGMoSgKGdm5ZrmZcunIN998k48++ojw8HCaN29OWloavXr1IiwsjMOHD9OjRw/69u1LZGTkXY8zffp0Bg8ezLFjx+jVqxcjRowgMTGxyP0zMjL45JNP+OWXX/jnn3+IjIzk1Vdf1T8+a9YsFi9ezI8//siuXbtISUlhzZo1RR4v/4f1qlWriI6OpmPHjvpzzZo1i++//56TJ0/i6enJqFGjOHDgAOvWrWPPnj0oikKvXr0MZjRlZGQwb948li1bxqZNm/j7778ZMGAAGzZsYMOGDfzyyy988803rFy5ssi4RowYQc2aNdm/fz8HDx7kzTffxMrKCoAjR47QtWtXGjduzJ49e9i5cyd9+/Y1SKgWLVqEtbU1u3btYsGCBQBYWFgwb948Tp48yaJFi9i2bRuvv/66/nWYO3cuzs7OREdHEx0drX9dQ0JCWLp0KfPmzSM8PJxvvvnGoBUQ4J133uHTTz/lwIEDWFpaMmbMmCKvzVSkC8zMpAK0EMIYN3M0NJ7yl1nOfeq97iarVfbee+/x6KOP6r+vXr06LVq00H///vvv8/vvv7Nu3TomTpxY5HFGjRrFsGHDAJgxYwbz5s1j37599OjRo9D9c3JyWLBgAXXr1gVg4sSJvPfee/rH58+fz1tvvcWAAQMA+OKLL9iwYUOR57e2tsbT01N/Dfm73HJycvjqq6/013X27FnWrVvHrl279EnS4sWL8fPzY82aNQwaNEj/vK+//lof48CBA/nll1+IjY3F0dGRxo0b06VLF7Zv386QIUMKjSsyMpLXXnuNhg0bAlC/fn39Yx9//DFt2rThq6++0m9r0qSJwfPr16/Pxx9/bLAt/+Buf39/PvjgA5577jm++uorrK2tDVr78pw5c4bffvuNLVu20K1bNwDq1KlTIN4PP/yQzp07A7rkuHfv3mRmZmJra1vo9ZmCtACZkVarEB6dCkgLkBCiamnTpo3B92lpabz66qs0atQIV1dXHB0dCQ8Pv2cLUPPmzfX3HRwccHZ21lcKLoy9vb0+sQBdNeG8/ZOTk4mNjaVdu3b6x9VqNUFBQUZdWx5ra2uD+MLDw7G0tCQ4OFi/zc3NjcDAQMLDw4uM0cvLC39/f4NWEy8vr7teZ2hoKM888wzdunXjo48+4vz58/rH8lqA7qawa966dStdu3alRo0aODk58fTTT5OQkEBGRkaRxzly5AhqtVqf3BQl/+uUV+H5btdnCtICZEZXb9wkLSsXa0sL6ng4mDscIUQlYGel5tR73c12blNxcDB8z3v11VfZsmULn3zyCfXq1cPOzo6BAweSnZ191+PkdevkUalUaLVao/Y3ZddefnZ2diUa21lYjMZe57Rp0xg+fDjr169n48aNTJ06lWXLljFgwAB9NeW7ufPnc+nSJfr06cOECRP48MMPqV69Ojt37mTs2LFkZ2cXWaizOOcCw2vOe83udn2mIC1AZnQqOhmABl6OWKnlRyGEuDeVSoW9taVZbmU5UWPXrl2MGjWKAQMG0KxZM7y9vbl06VKZna8wLi4ueHl5sX//fv02jUbDoUOHTHL8Ro0akZuby969e/XbEhISiIiIoHHjxiY5R34NGjTg5ZdfZvPmzTzxxBP8+OOPgK61JSwszKhjHTx4EK1Wy6effkr79u1p0KABUVFRBvtYW1sXGJjdrFkztFotO3bsKN3FlAH51DWjU7e6v2T8jxCiqqtfvz6rV6/myJEjHD16lOHDh5d5C0BhXnzxRWbOnMnatWuJiIhg8uTJ3LhxwyTJX/369enXrx/jxo1j586dHD16lKeeeooaNWrQr18/E0Svc/PmTSZOnMjff//N5cuX2bVrF/v376dRo0YAvPXWW+zfv5/nn3+eY8eOcfr0ab7++mvi4+OLPGa9evXIyclh/vz5XLhwgV9++UU/ODqPv78/aWlphIWFER8fT0ZGBv7+/owcOZIxY8awZs0aLl68yN9//81vv/1msustKUmAzEg/BV4SICFEFTdnzhyqVatGx44d6du3L927d6d169blHscbb7zBsGHDCAkJoUOHDjg6OtK9e3eTDcb98ccfCQoKok+fPnTo0AFFUdiwYUOBLq7SUKvVJCQkEBISQoMGDRg8eDA9e/Zk+vTpgK5laPPmzRw9epR27drRoUMH1q5de9daTC1atGDOnDnMmjWLpk2bsnjxYmbOnGmwT8eOHXnuuecYMmQIHh4e+kHUX3/9NQMHDuT555+nYcOGjBs3jvT0dJNdb0mplLLq/KzEUlJScHFxITk5GWfnsktOHvhoG9eSbrJ8fHuC68gqz0KIgjIzM7l48SIBAQFlOiNGFE6r1dKoUSMGDx5cZSs8VzR3+5sw5vNbBkGbSXJGDteSbgLQSNYAE0KICuHy5cts3ryZzp07k5WVxRdffMHFixcZPny4uUMTJiZdYGaSVwCxZjU7nG1N1/QphBCi5CwsLPjpp59o27YtDzzwAMePH2fr1q368TPi/iEtQGYiBRCFEKLi8fPzY9euXeYOQ5QDaQEyk7wWoMbS/SWEEEKUO0mAzCRvBphUgBZCCCHKnyRAZpCdq+VcnG4FXekCE0IIIcqfJEBmcP56GtkaLU62ltSsVrwy4UIIIYQwHUmAzCBvAHQjH+cyLS0vhBBCiMJJAmQGUgFaCCGEMC9JgMzglEyBF0KIchcTE8Ojjz6Kg4MDrq6uJj32Tz/9ZPJjltSlS5dQqVQcOXLE3KFUaJIAlTNFUW7XAJIp8EKI+5BKpbrrbdq0aWaJ67PPPiM6OpojR45w5swZs8RQHvz8/IiOjqZp06bmDqVCk0KI5SwmJZMbGTmoLVTU83Q0dzhCCGFy0dHR+vvLly9nypQpRERE6Lc5Ot5+71MUBY1Gc9eFOE3l/PnzBAUFUb9+/RIfIzs7G2traxNGVXwajQaVSoWFxd3bLtRqNd7e3uUUVeUlLUDlLK/1p56HI7ZWajNHI4QQpuft7a2/ubi4oFKp9N+fPn0aJycnNm7cSFBQEDY2NuzcuZPz58/Tr18/vLy8cHR0pG3btmzdutXguP7+/syYMYMxY8bg5ORErVq1+Pbbb/WPZ2dnM3HiRHx8fLC1taV27dr6Fcv9/f1ZtWoVP//8MyqVilGjRgGQlJTEM888g4eHB87OzjzyyCMcPXpUf8xp06bRsmVLvv/+e6MXpF27di2tW7fG1taWOnXqMH36dHJzc/WPz5kzh2bNmuHg4ICfnx/PP/88aWlp+sfzutXWrVtH48aNsbGxITIy8p6vw51dYH///TcqlYqwsDDatGmDvb09HTt2NEhKAT744AM8PT1xcnLimWee4c0336Rly5bFvt7KRhKgcna7AKKTmSMRQlRKigLZ6ea5KYrJLuPNN9/ko48+Ijw8nObNm5OWlkavXr0ICwvj8OHD9OjRg759+xIZGWnwvE8//ZQ2bdpw+PBhnn/+eSZMmKD/IJ83bx7r1q3jt99+IyIigsWLF+Pv7w/A/v376dGjB4MHDyY6OprPP/8cgEGDBhEXF8fGjRs5ePAgrVu3pmvXriQmJurPee7cOVatWsXq1auLPa7m33//JSQkhMmTJ3Pq1Cm++eYbfvrpJz788EP9PhYWFsybN4+TJ0+yaNEitm3bxuuvv25wnIyMDGbNmsX333/PyZMn8fT0vOfrUJR33nmHTz/9lAMHDmBpacmYMWP0jy1evJgPP/yQWbNmcfDgQWrVqsXXX39drGutrKQLrJyFR6cCMv5HCFFCORkww9c85347CqwdTHKo9957j0cffVT/ffXq1WnRooX++/fff5/ff/+ddevWMXHiRP32Xr168fzzzwPwxhtv8Nlnn7F9+3YCAwOJjIykfv36dOrUCZVKRe3atfXP8/DwwMbGBjs7O3330M6dO9m3bx9xcXHY2NgA8Mknn7BmzRpWrlzJ+PHjAV3L0s8//4yHh0exr2/69Om8+eabjBw5EoA6derw/vvv8/rrrzN16lQAXnrpJf3+/v7+fPDBBzz33HN89dVX+u05OTl89dVXBq/NvV6Honz44Yd07twZ0CWgvXv3JjMzE1tbW+bPn8/YsWMZPXo0AFOmTGHz5s0GLVL3G2kBKme3Z4C5mDkSIYQwnzZt2hh8n5aWxquvvkqjRo1wdXXF0dGR8PDwAi1AzZs319/P61qLi4sDYNSoURw5coTAwEAmTZrE5s2b7xrD0aNHSUtLw83NDUdHR/3t4sWLnD9/Xr9f7dq1jUp+8o793nvvGRx33LhxREdHk5GRAcDWrVvp2rUrNWrUwMnJiaeffpqEhAT94wDW1tYG11yc16Eo+Z/j4+MDoH9OREQE7dq1M9j/zu/vN9ICVI7Ss3K5lJAOSBeYEKKErOx1LTHmOreJODgYtiS9+uqrbNmyhU8++YR69ephZ2fHwIEDyc7ONgzBysrge5VKhVarBaB169ZcvHiRjRs3snXrVgYPHky3bt1YuXJloTGkpaXh4+PD33//XeCx/FPa74y1ONLS0pg+fTpPPPFEgcdsbW25dOkSffr0YcKECXz44YdUr16dnTt3MnbsWLKzs7G3173WdnZ2hRbMvdvrUJT8z8k75r2ecz+TBKgcnY5JRVHAy9kGN0cbc4cjhKiMVCqTdUNVJLt27WLUqFEMGDAA0CUQly5dMvo4zs7ODBkyhCFDhjBw4EB69OhBYmIi1atXL7Bv69atiYmJwdLSUj9WyFRat25NREQE9erVK/TxgwcPotVq+fTTT/Wzun777TeTxmCMwMBA9u/fT0hIiH7b/v37zRZPeZAEqBydipYV4IUQojD169dn9erV9O3bF5VKxbvvvmt068ScOXPw8fGhVatWWFhYsGLFCry9vYssUNitWzc6dOhA//79+fjjj2nQoAFRUVGsX7+eAQMGFOimM8aUKVPo06cPtWrVYuDAgVhYWHD06FFOnDjBBx98QL169cjJyWH+/Pn07duXXbt2sWDBghKfr7RefPFFxo0bR5s2bejYsSPLly/n2LFj1KlTx2wxlTUZA1SOUm7mYGtlIRWghRDiDnPmzKFatWp07NiRvn370r17d1q3bm3UMZycnPj4449p06YNbdu25dKlS2zYsKHIujkqlYoNGzbw0EMPMXr0aBo0aMDQoUO5fPkyXl5epbqe7t278+eff7J582batm1L+/bt+eyzz/QDs1u0aMGcOXOYNWsWTZs2ZfHixfop++YwYsQI3nrrLV599VV9V+KoUaOMmvZf2agUxYTzGu8TKSkpuLi4kJycjLOzaZMVjVYhK1eDvbU0vgkh7i0zM5OLFy8aXYNGiNJ69NFH8fb25pdffjF3KAbu9jdhzOe3fAqXM7WFSpIfIYQQFUpGRgYLFiyge/fuqNVqli5dytatW9myZYu5Qysz8kkshBBCVHF53YEffvghmZmZBAYGsmrVKrp162bu0MqMJEBCCCFEFWdnZ1dg6ZH7nQyCFkIIIUSVIwmQEEIIIaqcCpEAffnll/j7+2Nra0twcDD79u0rct+ffvoJlUplcLtzFLiiKEyZMgUfHx/s7Ozo1q0bZ8+eLevLEEKIMiMTdoXQMdXfgtkToOXLlxMaGsrUqVM5dOgQLVq0oHv37ndd08TZ2Zno6Gj97fLlywaPf/zxx8ybN48FCxawd+9eHBwc6N69O5mZmWV9OUIIYVJ5yxfkXx9KiKos72/hzuVAjGX2QdBz5sxh3Lhx+hVoFyxYwPr161m4cCFvvvlmoc/JW/itMIqiMHfuXP73v//Rr18/AH7++We8vLxYs2YNQ4cOLZsLEUKIMqBWq3F1ddX/U2hvb1/o2lBC3O8URSEjI4O4uDhcXV1Rq9WlOp5ZE6Ds7GwOHjzIW2+9pd9mYWFBt27d2LNnT5HPS0tLo3bt2mi1Wlq3bs2MGTNo0qQJABcvXiQmJsZg6p6LiwvBwcHs2bOn0AQoKyuLrKws/fcpKSmmuDwhhDCJvH/47rXatxBVgaura5GNIMYwawIUHx+PRqMpUHLcy8uL06dPF/qcwMBAFi5cSPPmzUlOTuaTTz6hY8eOnDx5kpo1axITE6M/xp3HzHvsTjNnzmT69OkmuCIhhDA9lUqFj48Pnp6e5OTkmDscIczGysqq1C0/eczeBWasDh060KFDB/33HTt2pFGjRnzzzTe8//77JTrmW2+9RWhoqP77lJQU/Pz8Sh2rEEKYklqtNtmbvxBVnVkHQbu7u6NWq4mNjTXYHhsbW+zmLSsrK1q1asW5c+eA203FxhzTxsYGZ2dng5sQQggh7l9mTYCsra0JCgoiLCxMv02r1RIWFmbQynM3Go2G48eP4+PjA0BAQADe3t4Gx0xJSWHv3r3FPqYQQggh7m9m7wILDQ1l5MiRtGnThnbt2jF37lzS09P1s8JCQkKoUaMGM2fOBOC9996jffv21KtXj6SkJGbPns3ly5d55plnAF1f+UsvvcQHH3xA/fr1CQgI4N1338XX15f+/fub6zKFEEIIUYGYPQEaMmQI169fZ8qUKcTExNCyZUs2bdqkH8QcGRmJhcXthqobN24wbtw4YmJiqFatGkFBQezevZvGjRvr93n99ddJT09n/PjxJCUl0alTJzZt2lSgYGJR8oosyWwwIYQQovLI+9wuTrFElSLlRQu4evWqDIIWQgghKqkrV65Qs2bNu+4jCVAhtFotUVFRODk5mbzgWN4MsytXrlTJwdZy/VX7+kFeg6p+/SCvgVx/2V2/oiikpqbi6+tr0HtUGLN3gVVEFhYW98wcS6uqzzaT66/a1w/yGlT16wd5DeT6y+b6XVxcirWf2dcCE0IIIYQob5IACSGEEKLKkQSonNnY2DB16lRsbGzMHYpZyPVX7esHeQ2q+vWDvAZy/RXj+mUQtBBCCCGqHGkBEkIIIUSVIwmQEEIIIaocSYCEEEIIUeVIAiSEEEKIKkcSoHL05Zdf4u/vj62tLcHBwezbt8/cIZWbmTNn0rZtW5ycnPD09KR///5ERESYOyyz+eijj/QL91YV165d46mnnsLNzQ07OzuaNWvGgQMHzB1WudFoNLz77rsEBARgZ2dH3bp1ef/994u1ZlFl9M8//9C3b198fX1RqVSsWbPG4HFFUZgyZQo+Pj7Y2dnRrVs3zp49a55gy8jdXoOcnBzeeOMNmjVrhoODA76+voSEhBAVFWW+gE3sXr8D+T333HOoVCrmzp1bbvFJAlROli9fTmhoKFOnTuXQoUO0aNGC7t27ExcXZ+7QysWOHTt44YUX+O+//9iyZQs5OTk89thjpKenmzu0crd//36++eYbmjdvbu5Qys2NGzd44IEHsLKyYuPGjZw6dYpPP/2UatWqmTu0cjNr1iy+/vprvvjiC8LDw5k1axYff/wx8+fPN3doZSI9PZ0WLVrw5ZdfFvr4xx9/zLx581iwYAF79+7FwcGB7t27k5mZWc6Rlp27vQYZGRkcOnSId999l0OHDrF69WoiIiJ4/PHHzRBp2bjX70Ce33//nf/++w9fX99yiuwWRZSLdu3aKS+88IL+e41Go/j6+iozZ840Y1TmExcXpwDKjh07zB1KuUpNTVXq16+vbNmyRencubMyefJkc4dULt544w2lU6dO5g7DrHr37q2MGTPGYNsTTzyhjBgxwkwRlR9A+f333/Xfa7VaxdvbW5k9e7Z+W1JSkmJjY6MsXbrUDBGWvTtfg8Ls27dPAZTLly+XT1DlqKjrv3r1qlKjRg3lxIkTSu3atZXPPvus3GKSFqBykJ2dzcGDB+nWrZt+m4WFBd26dWPPnj1mjMx8kpOTAahevbqZIylfL7zwAr179zb4XagK1q1bR5s2bRg0aBCenp60atWK7777ztxhlauOHTsSFhbGmTNnADh69Cg7d+6kZ8+eZo6s/F28eJGYmBiDvwMXFxeCg4Or7Hsi6N4XVSoVrq6u5g6lXGi1Wp5++mlee+01mjRpUu7nl8VQy0F8fDwajQYvLy+D7V5eXpw+fdpMUZmPVqvlpZde4oEHHqBp06bmDqfcLFu2jEOHDrF//35zh1LuLly4wNdff01oaChvv/02+/fvZ9KkSVhbWzNy5Ehzh1cu3nzzTVJSUmjYsCFqtRqNRsOHH37IiBEjzB1auYuJiQEo9D0x77GqJjMzkzfeeINhw4ZVmQVSZ82ahaWlJZMmTTLL+SUBEuXuhRde4MSJE+zcudPcoZSbK1euMHnyZLZs2YKtra25wyl3Wq2WNm3aMGPGDABatWrFiRMnWLBgQZVJgH777TcWL17MkiVLaNKkCUeOHOGll17C19e3yrwGonA5OTkMHjwYRVH4+uuvzR1OuTh48CCff/45hw4dQqVSmSUG6QIrB+7u7qjVamJjYw22x8bG4u3tbaaozGPixIn8+eefbN++nZo1a5o7nHJz8OBB4uLiaN26NZaWllhaWrJjxw7mzZuHpaUlGo3G3CGWKR8fHxo3bmywrVGjRkRGRpopovL32muv8eabbzJ06FCaNWvG008/zcsvv8zMmTPNHVq5y3vfk/fE28nP5cuX2bJlS5Vp/fn333+Ji4ujVq1a+vfEy5cv88orr+Dv718uMUgCVA6sra0JCgoiLCxMv02r1RIWFkaHDh3MGFn5URSFiRMn8vvvv7Nt2zYCAgLMHVK56tq1K8ePH+fIkSP6W5s2bRgxYgRHjhxBrVabO8Qy9cADDxQoe3DmzBlq165tpojKX0ZGBhYWhm+5arUarVZrpojMJyAgAG9vb4P3xJSUFPbu3Vtl3hPhdvJz9uxZtm7dipubm7lDKjdPP/00x44dM3hP9PX15bXXXuOvv/4qlxikC6ychIaGMnLkSNq0aUO7du2YO3cu6enpjB492tyhlYsXXniBJUuWsHbtWpycnPT9/C4uLtjZ2Zk5urLn5ORUYLyTg4MDbm5uVWIc1Msvv0zHjh2ZMWMGgwcPZt++fXz77bd8++235g6t3PTt25cPP/yQWrVq0aRJEw4fPsycOXMYM2aMuUMrE2lpaZw7d07//cWLFzly5AjVq1enVq1avPTSS3zwwQfUr1+fgIAA3n33XXx9fenfv7/5gjaxu70GPj4+DBw4kEOHDvHnn3+i0Wj074vVq1fH2traXGGbzL1+B+5M+KysrPD29iYwMLB8Aiy3+WZCmT9/vlKrVi3F2tpaadeunfLff/+ZO6RyAxR6+/HHH80dmtlUpWnwiqIof/zxh9K0aVPFxsZGadiwofLtt9+aO6RylZKSokyePFmpVauWYmtrq9SpU0d55513lKysLHOHVia2b99e6N/8yJEjFUXRTYV/9913FS8vL8XGxkbp2rWrEhERYd6gTexur8HFixeLfF/cvn27uUM3iXv9DtypvKfBqxTlPi1DKoQQQghRBBkDJIQQQogqRxIgIYQQQlQ5kgAJIYQQosqRBEgIIYQQVY4kQEIIIYSociQBEkIIIUSVIwmQEEIIIaocSYCEEKIYVCoVa9asMXcYQggTkQRICFHhjRo1CpVKVeDWo0cPc4cmhKikZC0wIUSl0KNHD3788UeDbTY2NmaKRghR2UkLkBCiUrCxscHb+//t3b1L61AYBvAnVZEkKESL3jiJKKUKuihSdFEHqZNSESFItlI/ioubinVw1TEg6CYKFYSCqKBjQXSxdqj+A1JUXExBl547CIUg93I/bEvN84NAzjlJ+77bQ3JKfzgOTdMAfLyesiwLwWAQsiyjra0Nh4eHjvvT6TSGh4chyzIaGxsRDodh27bjmt3dXXR1daG2tha6rmNhYcGx/vz8jImJCSiKgo6ODiQSieI2TURFwwBERN/C6uoqQqEQUqkUDMPA9PQ0MpkMACCXy2F0dBSapuH6+hrxeBzn5+eOgGNZFubn5xEOh5FOp5FIJNDe3u74jvX1dUxNTeH29hZjY2MwDAMvLy8l7ZOIvkjJ/naViOgfmaYpqqqqhKqqjmNjY0MIIQQAEYlEHPf09/eL2dlZIYQQ29vbQtM0Ydt2Yf34+Fh4PB6RzWaFEEK0tLSI5eXlX9YAQKysrBTGtm0LAOLk5OTL+iSi0uEeICKqCENDQ7AsyzHX0NBQOA8EAo61QCCAm5sbAEAmk0FPTw9UVS2sDwwMIJ/P4/7+HpIk4eHhASMjI7+tobu7u3Cuqirq6+vx+Pj4ry0RURkxABFRRVBV9dMrqa8iy/IfXVdTU+MYS5KEfD5fjJKIqMi4B4iIvoXLy8tPY7/fDwDw+/1IpVLI5XKF9WQyCY/HA5/Ph7q6OrS2tuLi4qKkNRNR+fAJEBFVhPf3d2SzWcdcdXU1vF4vACAej6O3txeDg4PY29vD1dUVdnZ2AACGYWBtbQ2maSIWi+Hp6QnRaBQzMzNobm4GAMRiMUQiETQ1NSEYDOL19RXJZBLRaLS0jRJRSTAAEVFFOD09ha7rjjmfz4e7uzsAH7/QOjg4wNzcHHRdx/7+Pjo7OwEAiqLg7OwMi4uL6Ovrg6IoCIVC2NzcLHyWaZp4e3vD1tYWlpaW4PV6MTk5WboGiaikJCGEKHcRRET/Q5IkHB0dYXx8vNylEFGF4B4gIiIich0GICIiInId7gEioorHN/lE9Lf4BIiIiIhchwGIiIiIXIcBiIiIiFyHAYiIiIhchwGIiIiIXIcBiIiIiFyHAYiIiIhchwGIiIiIXIcBiIiIiFznJ3dpG3X4uI7hAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Plotting the accuracy for both settings\n",
        "plt.plot([acc.cpu().item() for acc in from_scratch_valid_acc], label=\"Training from scratch\")\n",
        "plt.plot([acc.cpu().item() for acc in pretrained_valid_acc], label=\"Transfer learning\")\n",
        "\n",
        "plt.title(\"Model Accuracy on Validation Set by Epoch\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Validation Accuracy\")\n",
        "plt.legend()\n",
        "\n",
        "# Saving the figure\n",
        "plt.savefig(\"sentiment_analysis_accuracy_comparison.png\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " ntokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tDlvejePvEoN",
        "outputId": "8b6fa9bf-4dec-4156-bfda-617eaf9f7e2f"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "50001"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.6.15 ('altegrad')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.15"
    },
    "vscode": {
      "interpreter": {
        "hash": "1f3cfdeab8dd8f9900bd16266619de191cf0f5e09365d74b1fba1714dce58066"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}