all-in-one-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.fft\n",
    "import torch.utils.data\n",
    "from torch import nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.loggers import WandbLogger\n",
    "from einops import rearrange, repeat\n",
    "from einops.layers.torch import Rearrange\n",
    "import torchmetrics\n",
    "import wandb\n",
    "import gc\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import random\n",
    "from dataset import CoughDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Transformer implementation\n",
    "# Adapted from Holger Severin Bovbjerg <hsbo@es.aau.dk> and Sarthak Yadav <sarthaky@es.aau.dk>\n",
    "# Where the KWT model is based on the model from https://github.com/ID56/Torch-KWT/blob/main/models/kwt.py\"\"\"\n",
    "\n",
    "class PreNorm(nn.Module):\n",
    "    \"\"\"\n",
    "    Pre layer normalization\n",
    "    \"\"\"\n",
    "    def __init__(self, dim, fn):\n",
    "        \"\"\"\n",
    "        Initialises PreNorm module\n",
    "        :param dim: model dimension\n",
    "        :param fn: torch module\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.norm = nn.LayerNorm(dim)\n",
    "        self.fn = fn\n",
    "\n",
    "    def forward(self, x, **kwargs):\n",
    "        \"\"\"\n",
    "        Forward method for PreNorm module\n",
    "        :param x: input tensor\n",
    "        :param kwargs: Keyword arguments\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        return self.fn(self.norm(x), **kwargs)\n",
    "\n",
    "\n",
    "class PostNorm(nn.Module):\n",
    "    \"\"\"\n",
    "    Post layer normalization\n",
    "    \"\"\"\n",
    "    def __init__(self, dim, fn):\n",
    "        \"\"\"\n",
    "        Initialises PostNorm module\n",
    "        :param dim: model dimension\n",
    "        :param fn: torch module\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.norm = nn.LayerNorm(dim)\n",
    "        self.fn = fn\n",
    "\n",
    "    def forward(self, x, **kwargs):\n",
    "        \"\"\"\n",
    "        Forward method for PostNorm module\n",
    "        :param x: input tensor\n",
    "        :param kwargs: Keyword arguments\n",
    "        :return: PostNorm output\n",
    "        \"\"\"\n",
    "        return self.norm(self.fn(x, **kwargs))\n",
    "\n",
    "\n",
    "class FeedForward(nn.Module):\n",
    "    \"\"\"\n",
    "    Feed forward model\n",
    "    \"\"\"\n",
    "    def __init__(self, dim, hidden_dim, dropout=0.):\n",
    "        \"\"\"\n",
    "        Initialises FeedForward module\n",
    "        :param dim: feedforward dim\n",
    "        :param hidden_dim: hidden dimension of feedforward layer\n",
    "        :param dropout: feedforward dropout percentage\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(dim, hidden_dim),\n",
    "            nn.GELU(),\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(hidden_dim, dim),\n",
    "            nn.Dropout(dropout)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Forward method for feedforward module\n",
    "        :param x: input tensor\n",
    "        :return: FeedForward output\n",
    "        \"\"\"\n",
    "        return self.net(x)\n",
    "\n",
    "\n",
    "class SHA(nn.Module):\n",
    "    def __init__(self, head_dim, attn_drop=0.):\n",
    "        super().__init__()\n",
    "        self.head_dim = head_dim\n",
    "        self.scale = head_dim ** -0.5\n",
    "        self.attn_drop = nn.Dropout(attn_drop)\n",
    "    \n",
    "    def forward(self, q, k, v):\n",
    "        # print(f\"in SHA, q:{q.shape}, k:{k.shape}, v:{v.shape}\")\n",
    "        attn = (q @ k.transpose(-2, -1)) * self.scale\n",
    "        attn = attn.softmax(dim=-1)\n",
    "        attn = self.attn_drop(attn)\n",
    "        x = (attn @ v)\n",
    "        # print(\"output shape:\", x.shape)\n",
    "        return x\n",
    "\n",
    "\n",
    "class MHA(nn.Module):\n",
    "    def __init__(self, dim, heads=8, dim_head=64, dropout=0.):\n",
    "        super().__init__()\n",
    "\n",
    "        self.num_heads = heads\n",
    "        self.dim = dim\n",
    "        self.head_dim = dim_head\n",
    "        inner_dim = dim_head * heads\n",
    "\n",
    "        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)\n",
    "        self.attention_heads = nn.ModuleList([\n",
    "            SHA(self.head_dim) for _ in range(self.num_heads)\n",
    "        ])\n",
    "        project_out = not (heads == 1 and dim_head == dim)\n",
    "        self.scale = dim_head ** -0.5\n",
    "\n",
    "        self.to_out = nn.Sequential(\n",
    "                nn.Linear(inner_dim, dim),\n",
    "                nn.Dropout(dropout)\n",
    "            ) if project_out else nn.Identity() \n",
    "  \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Forward method for Attention module\n",
    "        :param x: input tensor\n",
    "        :return: Attention module output\n",
    "        \"\"\"\n",
    "\n",
    "        B, N, C = x.shape\n",
    "        qkv = self.to_qkv(x)\n",
    "\n",
    "        qkv = qkv.reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 3, 0, 1, 4)\n",
    "        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)\n",
    "\n",
    "        o = []\n",
    "        for i in range(self.num_heads):\n",
    "            head_i = self.attention_heads[i](q[i],k[i],v[i]).unsqueeze(0)\n",
    "            o.append(head_i)\n",
    "        o = torch.concat(o, dim=0)\n",
    "        o = o.permute(1, 2, 0, 3).reshape(B, N, -1)\n",
    "\n",
    "        return self.to_out(o)\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    \"\"\"\n",
    "    Transformer model\n",
    "    \"\"\"\n",
    "    def __init__(self, dim, depth, heads, dim_head, mlp_dim, pre_norm=True, dropout=0., mha_block=MHA):\n",
    "        \"\"\"\n",
    "        Initialises Transformer model\n",
    "        :param dim: transformer dimension\n",
    "        :param depth: number of transformer layers\n",
    "        :param heads: number of attention heads for each transformer layer\n",
    "        :param dim_head: dimension of each attention head\n",
    "        :param mlp_dim: MLP dimension\n",
    "        :param pre_norm: specifies whether PreNorm (True) or PostNorm (False) is used\n",
    "        :param dropout: dropout percentage of Attention of FeedForward modules\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([])\n",
    "\n",
    "        P_Norm = PreNorm if pre_norm else PostNorm\n",
    "        if mha_block is None:\n",
    "          mha_block = MHA\n",
    "        for _ in range(depth):\n",
    "            self.layers.append(nn.ModuleList([\n",
    "                P_Norm(dim, mha_block(dim, heads=heads, dim_head=dim_head, dropout=dropout)),\n",
    "                P_Norm(dim, FeedForward(dim, mlp_dim, dropout=dropout))\n",
    "            ]))\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Forward method for Transformer model\n",
    "        :param x: input tensor\n",
    "        :return: Tuple of model output, hidden states of transformer and attentions from each transformer layer\n",
    "        \"\"\"\n",
    "        hidden_states = []\n",
    "        attentions = []\n",
    "        for attn, ff in self.layers:\n",
    "            x = attn(x) + x\n",
    "            attentions.append(x)\n",
    "            x = ff(x) + x\n",
    "            hidden_states.append(x)\n",
    "        return x, hidden_states, attentions\n",
    "\n",
    "\n",
    "class KWT(nn.Module):\n",
    "    \"\"\"\n",
    "    KWT model\n",
    "    \"\"\"\n",
    "    def __init__(self, input_res, patch_res, num_classes, dim, depth, heads, mlp_dim, pool='cls', channels=1,\n",
    "                 dim_head=64, dropout=0., emb_dropout=0., pre_norm=True, mha_block=MHA, **kwargs):\n",
    "        \"\"\"\n",
    "        Initialises KWT model\n",
    "        :param input_res: input spectrogram size\n",
    "        :param patch_res: patch size\n",
    "        :param num_classes: number of keyword classes\n",
    "        :param dim: transformer dimension\n",
    "        :param depth: number of transformer layers\n",
    "        :param heads: number of attention heads\n",
    "        :param mlp_dim: MLP dimension\n",
    "        :param pool: specifies whether CLS token or average pooling of transformer model is used for classification\n",
    "        :param channels: Number of input channels\n",
    "        :param dim_head: dimension of attention heads\n",
    "        :param dropout: dropout of transformer attention and feed forward layers\n",
    "        :param emb_dropout: dropout of embeddings\n",
    "        :param pre_norm: specifies whether PreNorm (True) or PostNorm (False) is used\n",
    "        :param kwargs: Keyword arguments\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "\n",
    "        num_patches = int(input_res[0] / patch_res[0] * input_res[1] / patch_res[1])\n",
    "\n",
    "        patch_dim = channels * patch_res[0] * patch_res[1]\n",
    "        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'\n",
    "\n",
    "        self.to_patch_embedding = nn.Sequential(\n",
    "            Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1=patch_res[0], p2=patch_res[1]),\n",
    "            nn.Linear(patch_dim, dim),\n",
    "        )\n",
    "\n",
    "        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n",
    "        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n",
    "        self.dropout = nn.Dropout(emb_dropout)\n",
    "        self.mask_embedding = nn.Parameter(torch.FloatTensor(dim).uniform_())\n",
    "        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, pre_norm, dropout, mha_block=mha_block)\n",
    "\n",
    "        self.pool = pool\n",
    "        self.to_latent = nn.Identity()\n",
    "\n",
    "        # Create classification head\n",
    "        self.mlp_head = nn.Sequential(\n",
    "            nn.LayerNorm(dim),\n",
    "            nn.Linear(dim, num_classes)\n",
    "        )\n",
    "\n",
    "    def forward(self, x, mask=None, output_hidden_states=False, output_attentions=False):\n",
    "        \"\"\"\n",
    "        Forward method of KWT model\n",
    "        :param x: input tensor\n",
    "        :param mask: input mask\n",
    "        :param output_hidden_states: specifies whether hidden states are output\n",
    "        :param output_attentions: specifies whether attentions are output\n",
    "        :return: KWT model output, if output_hidden_states and/or output_attentions the classification head is skipped\n",
    "        \"\"\"\n",
    "        x = self.to_patch_embedding(x)\n",
    "        b, n, _ = x.shape\n",
    "\n",
    "        # Add cls token embedding\n",
    "        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)\n",
    "        x = torch.cat((cls_tokens, x), dim=1)\n",
    "\n",
    "        # Mask input\n",
    "        if mask is not None:\n",
    "            x[mask] = self.mask_embedding\n",
    "\n",
    "        x += self.pos_embedding[:, :(n + 1)]\n",
    "        x = self.dropout(x)\n",
    "\n",
    "        x, hidden_states, attentions = self.transformer(x)\n",
    "\n",
    "        x = x.mean(dim=1) if self.pool == 'mean' else x[:, 0]\n",
    "\n",
    "        x = self.to_latent(x)\n",
    "\n",
    "        if any([output_hidden_states, output_attentions]):\n",
    "            outputs = (self.mlp_head(x), hidden_states) if output_hidden_states else (self.mlp_head(x), )\n",
    "            outputs = outputs + (attentions, ) if output_attentions else outputs\n",
    "            return outputs\n",
    "        return self.mlp_head(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CoughDataModule(pl.LightningDataModule):\n",
    "    def __init__(self, \n",
    "                 df, \n",
    "                 data_path, \n",
    "                 batch_size=32, \n",
    "                 num_workers=4, \n",
    "                 train_size=0.8, \n",
    "                 val_size=0.1, \n",
    "                 test_size=0.1,\n",
    "                 duration=10.0,\n",
    "                 sample_rate=48000,\n",
    "                 channels=1,\n",
    "                 spectrogram_type='mel-spectrogram',\n",
    "                 n_mels=64,\n",
    "                 n_fft=1024, \n",
    "                 top_db=80,\n",
    "                 augment_masking_val='min',\n",
    "                 n_freq_masks=2,\n",
    "                 n_time_masks=1,\n",
    "                 max_mask_pct=0.1):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.df = df\n",
    "        self.data_path = data_path\n",
    "        self.batch_size = batch_size\n",
    "        self.num_workers = num_workers\n",
    "        \n",
    "        self.train_size = train_size\n",
    "        self.val_size = val_size\n",
    "        self.test_size = test_size\n",
    "        \n",
    "        self.duration = duration\n",
    "        self.sample_rate = sample_rate\n",
    "        self.channels = channels\n",
    "        \n",
    "        self.spectrogram_type = spectrogram_type\n",
    "        self.n_mels = n_mels\n",
    "        self.n_fft = n_fft\n",
    "        self.top_db = top_db\n",
    "        \n",
    "        self.augment_masking_val = augment_masking_val\n",
    "        self.n_freq_masks = n_freq_masks\n",
    "        self.n_time_masks = n_time_masks\n",
    "        self.max_mask_pct = max_mask_pct\n",
    "        \n",
    "        if self.train_size + self.val_size + self.test_size != 1.0:\n",
    "            raise Exception('train_size + val_size + test_size must be equal to 1.0')\n",
    "    \n",
    "        self.dataset1 = CoughDataset(df=self.df, \n",
    "                            data_path=self.data_path,\n",
    "                            duration=self.duration,\n",
    "                            sample_rate=self.sample_rate,\n",
    "                            channels=self.channels,\n",
    "                            spectrogram_type=self.spectrogram_type,\n",
    "                            n_mels=self.n_mels,\n",
    "                            n_fft=self.n_fft,\n",
    "                            top_db=self.top_db,\n",
    "                            n_freq_masks=0,\n",
    "                            n_time_masks=0)\n",
    "        \n",
    "        self.dataset2 = CoughDataset(df=self.df, \n",
    "                            data_path=self.data_path,\n",
    "                            duration=self.duration,\n",
    "                            sample_rate=self.sample_rate,\n",
    "                            channels=self.channels,\n",
    "                            spectrogram_type=self.spectrogram_type,\n",
    "                            n_mels=self.n_mels,\n",
    "                            n_fft=self.n_fft,\n",
    "                            top_db=self.top_db,\n",
    "                            augment_masking_val=self.augment_masking_val,\n",
    "                            n_freq_masks=self.n_freq_masks,\n",
    "                            n_time_masks=self.n_time_masks,\n",
    "                            max_mask_pct=self.max_mask_pct)\n",
    "        \n",
    "        self.dataset = torch.utils.data.ConcatDataset([self.dataset1, self.dataset2])\n",
    "        self.train_dataset, self.val_dataset, self.test_dataset = random_split(self.dataset, [self.train_size, self.val_size, self.test_size])\n",
    "    def train_dataloader(self):\n",
    "        # Concat the two datasets\n",
    "        return DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers)\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        return DataLoader(self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers)\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        return DataLoader(self.test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "METADATA_FILE = 'data/metadata_compiled.csv'\n",
    "DATA_PATH = 'data/'\n",
    "\n",
    "\n",
    "metadata_df = pd.read_csv(METADATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "symptomatic    699\n",
      "healthy        699\n",
      "COVID-19       699\n",
      "Name: status, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_303/3226820504.py:8: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  filtered_df = filtered_df[filtered_df['status'] == 'healthy'].sample(n=covid_size, random_state=42).append(filtered_df[filtered_df['status'] != 'healthy'])\n",
      "/tmp/ipykernel_303/3226820504.py:9: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  filtered_df = filtered_df[filtered_df['status'] == 'symptomatic'].sample(n=covid_size, random_state=42).append(filtered_df[filtered_df['status'] != 'symptomatic'])\n"
     ]
    }
   ],
   "source": [
    "not_nan_df = metadata_df[metadata_df['status'].isna() == False]\n",
    "filtered_df = not_nan_df[not_nan_df['cough_detected'] > 0.9]    # TODO: Set as a hyperparameter\n",
    "filtered_df[['uuid', 'cough_detected', 'SNR', 'age', 'gender', 'status']]\n",
    "# Augment the Covid-19 samples\n",
    "\n",
    "# get size of Covid-19 samples\n",
    "covid_size = filtered_df[filtered_df['status'] == 'COVID-19'].shape[0]\n",
    "filtered_df = filtered_df[filtered_df['status'] == 'healthy'].sample(n=covid_size, random_state=42).append(filtered_df[filtered_df['status'] != 'healthy'])\n",
    "filtered_df = filtered_df[filtered_df['status'] == 'symptomatic'].sample(n=covid_size, random_state=42).append(filtered_df[filtered_df['status'] != 'symptomatic'])\n",
    "print(filtered_df['status'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @title self-attention\n",
    "from torch.nn.modules.module import T\n",
    "class SelfAttention(nn.Module):\n",
    "  def __init__(self, head_dim):\n",
    "    super().__init__()\n",
    "    self.head_dim = head_dim\n",
    "    ## Complete what the scale should be. Then uncomment\n",
    "    self.scale = self.head_dim ** -0.5\n",
    "\n",
    "  def forward(self, q, k, v, mask=None):\n",
    "    \"\"\"\n",
    "    q, k, v are query, key and value tensors, respectively of shapes (B, N, C)\n",
    "    where B is Batch Size\n",
    "          N is number of patches\n",
    "          C is the dimensions\n",
    "    \"\"\"\n",
    "\n",
    "    d_k = q.size()[-1]\n",
    "    attn_logits = torch.matmul(q, k.transpose(-2, -1))\n",
    "    attn_logits = attn_logits / math.sqrt(d_k)\n",
    "    if mask is not None:\n",
    "        attn_logits = attn_logits.masked_fill(mask == 0, -9e15)\n",
    "    attention = F.softmax(attn_logits, dim=-1)\n",
    "    values = torch.matmul(attention, v)\n",
    "    return values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title MultiHeadAttention, already implemented\n",
    "class MultiHeadAttentionCustom(nn.Module):\n",
    "    def __init__(self, dim, heads=8, dim_head=64, dropout=0.):\n",
    "        super().__init__()\n",
    "\n",
    "        self.num_heads = heads\n",
    "        self.dim = dim\n",
    "        self.head_dim = dim_head\n",
    "        inner_dim = dim_head * heads\n",
    "\n",
    "        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)\n",
    "        self.attention_heads = nn.ModuleList([\n",
    "            SelfAttention(self.head_dim) for _ in range(self.num_heads)\n",
    "        ])\n",
    "        project_out = not (heads == 1 and dim_head == dim)\n",
    "        self.scale = dim_head ** -0.5\n",
    "\n",
    "        self.to_out = nn.Sequential(\n",
    "                nn.Linear(inner_dim, dim),\n",
    "                nn.Dropout(dropout)\n",
    "            ) if project_out else nn.Identity() \n",
    "  \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Forward method for Attention module\n",
    "        :param x: input tensor\n",
    "        :return: Attention module output\n",
    "        \"\"\"\n",
    "\n",
    "        B, N, C = x.shape\n",
    "        qkv = self.to_qkv(x)\n",
    "        #print(\"in attention qkv shape:\", qkv.shape)\n",
    "        qkv = qkv.reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 3, 0, 1, 4)\n",
    "        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)\n",
    "\n",
    "        o = []\n",
    "        for i in range(self.num_heads):\n",
    "            head_i = self.attention_heads[i](q[i],k[i],v[i]).unsqueeze(0)\n",
    "            o.append(head_i)\n",
    "        o = torch.concat(o, dim=0)\n",
    "        o = o.permute(1, 2, 0, 3).reshape(B, N, -1)\n",
    "\n",
    "        return self.to_out(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(trained_model, set):\n",
    "    predictor = trained_model.model.eval().to(torch.device('cuda'))\n",
    "    test_set = set\n",
    "\n",
    "    labels, predictions = [], []\n",
    "    for i in range(len(test_set)):\n",
    "        img, label = test_set[i]\n",
    "        labels.append(label)\n",
    "        with torch.no_grad():\n",
    "            img = img.to(torch.device('cuda'))\n",
    "            pred = predictor(torch.unsqueeze(img, 0))[0].cpu().numpy()\n",
    "            predictions.append(np.argmax(pred))\n",
    "        \n",
    "    from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "    plt.figure()\n",
    "    cm = confusion_matrix(labels, predictions, labels=[0,1,2])\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['COVID-19', 'healthy', 'symptomatic'])\n",
    "    disp.plot()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title This is the PyTorch-Lightning Module implementation, which takes care of a lot of boilerplate code related to training for us.\n",
    "class KWTLightning(pl.LightningModule):\n",
    "  def __init__(self, hparams, mha_block):\n",
    "    super().__init__()\n",
    "    #self.w1 = hparams[\"w1\"]\n",
    "    #self.w2 = hparams[\"w2\"]\n",
    "    #self.w3 = hparams[\"w3\"]\n",
    "    self.model = KWT(**hparams, mha_block=mha_block)\n",
    "\n",
    "  def forward(self, x):\n",
    "    return self.model(x)\n",
    "  \n",
    "  def configure_optimizers(self):\n",
    "    return torch.optim.AdamW(self.parameters(), lr=1e-3, weight_decay=0.01)\n",
    "  \n",
    "  def training_step(self, train_batch, batch_idx):\n",
    "    x, y = train_batch\n",
    "    # make a weight tensor with size of 3\n",
    "    #w = torch.ones(3)\n",
    "    # set the weight of the 0 class to 2\n",
    "    #w[0] = self.w1 # Covid\n",
    "    #w[1] = self.w2 # Healthy\n",
    "    #w[2] = self.w3 #\n",
    "    #w = w.to(device='cuda:0')\n",
    "    preds = self.model(x)\n",
    "    loss = nn.functional.cross_entropy(preds, y)\n",
    "    acc = (preds.argmax(dim=-1) == y).float().mean()\n",
    "    self.log(\"train_loss\", loss)\n",
    "    self.log(\"train_accuracy\", acc, prog_bar=True, on_step=False, on_epoch=True)\n",
    "    return loss\n",
    "  \n",
    "  def validation_step(self, val_batch, batch_idx):\n",
    "    x, y = val_batch\n",
    "    preds = self.model(x)\n",
    "    loss = nn.functional.cross_entropy(preds, y)\n",
    "    acc = (preds.argmax(dim=-1) == y).float().mean()\n",
    "    self.log('val_loss', loss)\n",
    "    self.log(\"val_accuracy\", acc, prog_bar=True, on_epoch=True)\n",
    "  \n",
    "  def test_step(self, batch, batch_idx):\n",
    "    x, y = batch\n",
    "    preds = self.model(x)\n",
    "    loss = nn.functional.cross_entropy(preds, y)\n",
    "    acc = (preds.argmax(dim=-1) == y).float().mean()\n",
    "    self.log('test_loss', loss)\n",
    "    self.log(\"test_accuracy\", acc)\n",
    "    \n",
    "    \n",
    "def create_pl(mha_block, MODEL_HPARAMS):\n",
    "  kwt_pl = KWTLightning(MODEL_HPARAMS, mha_block)\n",
    "  return kwt_pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(config=None):\n",
    "    with wandb.init(config=config):\n",
    "        config = wandb.config\n",
    "        \n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()\n",
    "        \n",
    "        try:    \n",
    "            torch.manual_seed(69) # noice\n",
    "            \n",
    "            wandb_logger = WandbLogger(log_model=True)\n",
    "            \n",
    "            print(\"Loading data...\")\n",
    "            # Load first data patch, with no augmentation\n",
    "            data_module = CoughDataModule(df=filtered_df, \n",
    "                              data_path=DATA_PATH,\n",
    "                              duration=10.0,\n",
    "                              sample_rate=config.sample_rate,\n",
    "                              channels=1,\n",
    "                              n_mels=config.n_mels,\n",
    "                              n_fft=config.n_fft,\n",
    "                              top_db=80,\n",
    "                              )\n",
    "            # Load second data patch, with augmentation\n",
    "            data_module_aug = CoughDataModule(df=filtered_df, \n",
    "                              data_path=DATA_PATH,\n",
    "                              duration=10.0,\n",
    "                              sample_rate=config.sample_rate,\n",
    "                              channels=1,\n",
    "                              n_mels=config.n_mels,\n",
    "                              n_fft=config.n_fft,\n",
    "                              top_db=80,\n",
    "                              augment=True,\n",
    "            )\n",
    "            # Get size of random sample of data\n",
    "            size = data_module.train_dataset[0][0].shape\n",
    "        \n",
    "            BATCH_SIZE = 128\n",
    "            N_CLASSES = 3\n",
    "            heads = 8\n",
    "            dim = 512\n",
    "            MODEL_HPARAMS = {\n",
    "                \"input_res\":[size[1], size[2]],\n",
    "                \"patch_res\":[size[1], 1],\n",
    "                \"num_classes\":N_CLASSES,\n",
    "                \"mlp_dim\":256,\n",
    "                \"dim\":dim,\n",
    "                \"heads\":heads,\n",
    "                \"dim_head\":dim//heads,       # dim_head should be dim divided by number of heads\n",
    "                \"depth\":6,\n",
    "                \"dropout\": 0.1,\n",
    "                \"emb_dropout\": 0.1\n",
    "            }\n",
    "            print(\"Size of random sample of data:\", size)\n",
    "            kwt_pl = create_pl(mha_block=MultiHeadAttentionCustom, MODEL_HPARAMS=MODEL_HPARAMS)\n",
    "            cuda_available = torch.cuda.is_available()\n",
    "            # Combine the two datamodules into one\n",
    "            print(\"Combining datasets...\")\n",
    "            train_set = torch.utils.data.ConcatDataset([data_module.train_dataset, data_module_aug.train_dataset])\n",
    "            val_set = torch.utils.data.ConcatDataset([data_module.val_dataset, data_module_aug.val_dataset])\n",
    "            test_set = torch.utils.data.ConcatDataset([data_module.test_dataset, data_module_aug.test_dataset])\n",
    "            print(data_module.dataset.label_encoder.classes_)\n",
    "            # Print how many Covid, Normal, and Pneumonia samples are in each set\n",
    "            #print(\"Test set class counts:\", count_labels(data_module.test_dataset))\n",
    "            #print(\"Train set class counts:\", count_labels(data_module.train_dataset))\n",
    "            #print(\"Val set class counts:\", count_labels(data_module.val_dataset))\n",
    "            trainer = pl.Trainer(\n",
    "                max_epochs=config.max_epochs,\n",
    "                logger=wandb_logger,\n",
    "                accelerator='gpu' if cuda_available else None,\n",
    "            )\n",
    "            trainer.fit(kwt_pl, datamodule=data_module)\n",
    "            trainer.test(kwt_pl, test_set)\n",
    "            # plot confusion matrix from trainer\n",
    "            plot_confusion_matrix(kwt_pl, data_module.test_dataset)\n",
    "            gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "            \n",
    "            \n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            wandb.finish()\n",
    "            raise e\n",
    "        \n",
    "        del wandb_logger\n",
    "        del data_module\n",
    "        del model\n",
    "        del classifier\n",
    "        del trainer\n",
    "        \n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_new(config=None):\n",
    "    with wandb.init(config=config):\n",
    "        config = wandb.config\n",
    "        \n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()\n",
    "        \n",
    "        try:    \n",
    "            torch.manual_seed(69) # noice\n",
    "            \n",
    "            wandb_logger = WandbLogger(log_model=True)\n",
    "            \n",
    "            print(\"Loading data...\")\n",
    "            # Load first data patch, with no augmentation\n",
    "            data_module = CoughDataModule(df=filtered_df, \n",
    "                              data_path=DATA_PATH,\n",
    "                              sample_rate=config.sample_rate,\n",
    "                              batch_size=config.batch_size,\n",
    "                              spectrogram_type=config.spectrogram_type,\n",
    "                              n_mels=config.n_mels,\n",
    "                              n_fft=config.n_fft,\n",
    "                              augment_masking_val=config.augment_masking_val,\n",
    "                              n_freq_masks=config.n_freq_masks,\n",
    "                              n_time_masks=config.n_time_masks)\n",
    "            # Load second data patch, with augmentation\n",
    "            size = data_module.train_dataset[0][0].shape\n",
    "            \n",
    "            N_CLASSES = 3\n",
    "            heads = config.heads\n",
    "            dim = config.dim\n",
    "            MODEL_HPARAMS = {\n",
    "                \"input_res\":[size[1], size[2]],\n",
    "                \"patch_res\":[size[1], 1],\n",
    "                \"num_classes\":N_CLASSES,\n",
    "                \"mlp_dim\":256,\n",
    "                \"dim\":dim,\n",
    "                \"heads\":heads,\n",
    "                \"dim_head\":dim//heads,       # dim_head should be dim divided by number of heads\n",
    "                \"depth\":6,\n",
    "                \"dropout\": 0.1,\n",
    "                \"emb_dropout\": 0.1\n",
    "            }\n",
    "            print(\"Size of random sample of data:\", size)\n",
    "            kwt_pl = create_pl(mha_block=MultiHeadAttentionCustom, MODEL_HPARAMS=MODEL_HPARAMS)\n",
    "            cuda_available = torch.cuda.is_available()\n",
    "            # Combine the two datamodules into one\n",
    "        \n",
    "            trainer = pl.Trainer(\n",
    "                max_epochs=config.max_epochs,\n",
    "                logger=wandb_logger,\n",
    "                accelerator='gpu' if cuda_available else None,\n",
    "            )\n",
    "            trainer.fit(kwt_pl, data_module)\n",
    "            trainer.test(kwt_pl, data_module)\n",
    "            plot_confusion_matrix(kwt_pl, data_module.test_dataset)\n",
    "            wandb.finish()\n",
    "            gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            wandb.finish()\n",
    "            del wandb_logger\n",
    "            del data_module\n",
    "            del kwt_pl\n",
    "            del trainer\n",
    "            gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "            raise e\n",
    "        \n",
    "        del wandb_logger\n",
    "        del data_module\n",
    "        del kwt_pl\n",
    "        del trainer\n",
    "        \n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_module = CoughDataModule(df=filtered_df, \n",
    "#                     data_path=DATA_PATH,\n",
    "#                     duration=10.0,\n",
    "#                     sample_rate=16000,\n",
    "#                     channels=1,\n",
    "#                     n_mels=64,\n",
    "#                     n_fft=1024,\n",
    "#                     top_db=80,\n",
    "#                     )\n",
    "\n",
    "# size = data_module.train_dataset[0][0].shape\n",
    "        \n",
    "# BATCH_SIZE = 128\n",
    "# N_CLASSES = 3\n",
    "# heads = 16\n",
    "# dim = 128\n",
    "# MODEL_HPARAMS = {\n",
    "#     \"input_res\":[size[1], size[2]],\n",
    "#     \"patch_res\":[size[1], 1],\n",
    "#     \"num_classes\":N_CLASSES,\n",
    "#     \"mlp_dim\":256,\n",
    "#     \"dim\":dim,\n",
    "#     \"heads\":heads,\n",
    "#     \"dim_head\":dim//heads,       # dim_head should be dim divided by number of heads\n",
    "#     \"depth\":6,\n",
    "#     \"dropout\": 0.1,\n",
    "#     \"emb_dropout\": 0.1,\n",
    "#     \"w1\": 1.19070340331838,\n",
    "#     \"w2\": 0.6683332489272846,\n",
    "#     \"w3\": 1.2838483214224414,\n",
    "# }\n",
    "# kwt_pl = create_pl(mha_block=MultiHeadAttentionCustom, MODEL_HPARAMS=MODEL_HPARAMS)\n",
    "# trainer = pl.Trainer(\n",
    "#     max_epochs=5,\n",
    "#     accelerator='gpu',\n",
    "# )\n",
    "# trainer.fit(kwt_pl, datamodule=data_module)\n",
    "\n",
    "# plot_confusion_matrix(kwt_pl, data_module.test_dataset)\n",
    "# #plot_confusion_matrix(kwt_pl, data_module.train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create sweep with ID: wou8ulsv\n",
      "Sweep URL: https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: a12qpgp6 with config:\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \taugment_masking_val: mean\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 8\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tdim: 16\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \theads: 8\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tmax_epochs: 5\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_fft: 512\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_freq_masks: 1\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_mels: 64\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_time_masks: 1\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tsample_rate: 16000\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tspectrogram_type: mel-spectrogram\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkajmoerk\u001b[0m (\u001b[33mdl-miniproject\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.14.0"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/ubuntu/git_repos/Deep-Learning-Mini-Project/wandb/run-20230331_231134-a12qpgp6</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/dl-miniproject/cough-classifier/runs/a12qpgp6' target=\"_blank\">crimson-sweep-1</a></strong> to <a href='https://wandb.ai/dl-miniproject/cough-classifier' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>Sweep page: <a href='https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/dl-miniproject/cough-classifier' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View sweep at <a href='https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/dl-miniproject/cough-classifier/runs/a12qpgp6' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/runs/a12qpgp6</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/miniconda3/envs/deep_learning/lib/python3.9/site-packages/pytorch_lightning/loggers/wandb.py:395: UserWarning: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of random sample of data: torch.Size([1, 64, 626])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA GeForce RTX 3070 Ti') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type | Params\n",
      "-------------------------------\n",
      "0 | model | KWT  | 68.6 K\n",
      "-------------------------------\n",
      "68.6 K    Trainable params\n",
      "0         Non-trainable params\n",
      "68.6 K    Total params\n",
      "0.274     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 420/420 [01:10<00:00,  5.92it/s, v_num=pgp6, val_accuracy=0.389, train_accuracy=0.404]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=5` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|██████████| 420/420 [01:10<00:00,  5.92it/s, v_num=pgp6, val_accuracy=0.389, train_accuracy=0.404]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA GeForce RTX 3070 Ti') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing DataLoader 0: 100%|██████████| 53/53 [00:07<00:00,  7.56it/s]\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
      "       Test metric             DataLoader 0\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
      "      test_accuracy         0.4152744710445404\n",
      "        test_loss           1.0804041624069214\n",
      "────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX+klEQVR4nO3deVxUVf8H8M9lG4Zl2JSdWBTFBXdDtAJzS400ciF9SkNNU1Myl8eflWSKaamklpk9IflkZm6ZqYlktpimJj2muKOiMqKJ7Isw5/cHOTmBeodhGWc+79frvl7dc++598tA8uV7zrlXEkIIEBEREZkxi4YOgIiIiKihMSEiIiIis8eEiIiIiMweEyIiIiIye0yIiIiIyOwxISIiIiKzx4SIiIiIzJ5VQwdADU+j0eDKlStwdHSEJEkNHQ4REelJCIH8/Hx4e3vDwqLuah0lJSUoKysz+Do2NjawtbWthYhqDxMiwpUrV+Dn59fQYRARkYEyMzPh6+tbJ9cuKSlBoL8D1NkVBl/L09MTGRkZRpUUMSEiODo6AgAu/BYAlQNHUU3doPCIhg6B6tGp+fxjxxxoiktxefIC7b/ndaGsrAzq7ApcOBwAlWPNf1fk5Wvg3/E8ysrKmBCRcbk9TKZysDDoh5weDFYWNg0dAtUjC6Xx/MKhulcf0x4cHCU4ONb8PhoY59QMJkREREQkW4XQoMKAt6BWCE3tBVOLmBARERGRbBoIaFDzjMiQvnWJ4yNERERk9lghIiIiItk00MCQQS/DetcdJkREREQkW4UQqBA1H/YypG9d4pAZERERmT1WiIiIiEg2U51UzYSIiIiIZNNAoMIEEyIOmREREZHZY4WIiIiIZOOQGREREZk9rjIjIiIiMlGsEBEREZFsmr82Q/obIyZEREREJFuFgavMDOlbl5gQERERkWwVAga+7b72YqlNnENEREREZo8VIiIiIpKNc4iIiIjI7GkgoQKSQf2NEYfMiIiIyOyxQkRERESyaUTlZkh/Y8SEiIiIiGSrMHDIzJC+dYlDZkRERGT2WCEiIiIi2Uy1QsSEiIiIiGTTCAkaYcAqMwP61iUOmREREZHZY4WIiIiIZOOQGREREZm9CligwoABpopajKU2MSEiIiIi2YSBc4gE5xARERERGSdWiIiIiEg2ziEiIiIis1chLFAhDJhDZKSv7uCQGRERERmt8vJyvPbaawgMDIRSqURQUBDmzJkDjUajPUcIgfj4eHh7e0OpVCIyMhLHjh3T6z5MiIiIiEg2DSRoYGHApt+Q2YIFC/Dhhx9i+fLlSE9Px8KFC/HOO+9g2bJl2nMWLlyIxYsXY/ny5Th48CA8PT3Rq1cv5Ofny74Ph8yIiIhItvqeQ/TLL79gwIAB6N+/PwAgICAAn3/+OQ4dOgSgsjqUmJiIWbNmITo6GgCQnJwMDw8PrF27FmPHjpV1H1aIiIiIqN7l5eXpbKWlpdWe98gjjyA1NRWnTp0CAPz+++/46aef0K9fPwBARkYG1Go1evfure2jUCgQERGBffv2yY6HFSIiIiKSzfBJ1ZWzqv38/HTaZ8+ejfj4+Crnz5gxA7m5uQgJCYGlpSUqKiowb948PPvsswAAtVoNAPDw8NDp5+HhgQsXLsiOiwkRERERyVY5h8iAl7v+1TczMxMqlUrbrlAoqj3/iy++wH//+1+sXbsWrVq1QlpaGuLi4uDt7Y0RI0Zoz5Mk3ZiEEFXa7oUJEREREdU7lUqlkxDdzbRp0/Dvf/8bMTExAIDQ0FBcuHAB8+fPx4gRI+Dp6QmgslLk5eWl7ZednV2lanQvnENEREREsmn+epdZTTeNnqlHUVERLCx0+1haWmqX3QcGBsLT0xMpKSna42VlZdi7dy+6du0q+z6sEBEREZFstTWHSK6oqCjMmzcPDz30EFq1aoUjR45g8eLFiI2NBVA5VBYXF4eEhAQEBwcjODgYCQkJsLOzw7Bhw2TfhwkRERERyaapQZVHt79+CdGyZcvw+uuvY/z48cjOzoa3tzfGjh2LN954Q3vO9OnTUVxcjPHjxyMnJwdhYWHYtWsXHB0dZd+HCREREREZLUdHRyQmJiIxMfGu50iShPj4+GpXqcnFhIiIiIhkqxASKoQBD2Y0oG9dYkJEREREst2eHF3z/sb5dleuMiMiIiKzxwoRERERyaYRFtAYsMpMo+cqs/rChIiIiIhk45AZERERkYlihYiIiIhk08CwlWKa2gulVjEhIiIiItkMfzCjcQ5OGWdURERERPWIFSIiIiKSzfB3mRlnLYYJEREREcmmgQQNDJlDxCdVExER0QOOFSKiB1BFObBmkSe+2+SCnGvWcHW/hV5DbmBY3FVY/PX/ZM41K/xnnjcO73VEYa4lWncpwIS5l+ATVNawwVOtcHMvxQtxZ9DpkT9ho9Dg8gU7vDc7BGfSVQ0dGtWQ66YrcNui1mkrd7JCxrI2QLmA28YrsP89F9bZZdDYWaKolSOuD/FGhYtNA0VMDwLjTNNqkVqtxssvv4ygoCAoFAr4+fkhKioKqamp2nP27duHfv36wcXFBba2tggNDcWiRYtQUVEBANi4cSMsLS1x8eLFau8REhKCSZMmAQAiIyMRFxenPRYZGQlJkiBJEhQKBXx8fBAVFYVNmzbJin/y5Mno2LEjFAoF2rVrV+0569evR7t27WBnZwd/f3+88847sq5tDr543wPffNoIE+Zdxqq9JzD6tSvYsMIdX33SCAAgBPBmbCCyLtggPukc3t91Eh6+Zfj30KYoKTL5/z1MnoPjLbybfBgV5RLeGN8O454Ow8eLmqIgn38LPuhKfWxxbmmodrs4rwUAwKJMA9vzRbgxwAsX3wpB1qQgWKtL4L3kXANHbDpuP5jRkM0YGWdUteT8+fPo2LEjvvvuOyxcuBBHjx7Fzp070b17d0yYMAEAsHnzZkRERMDX1xd79uzBiRMnMHnyZMybNw8xMTEQQuCpp56Cm5sbkpOTq9zj559/xsmTJzFq1Ki7xjFmzBhkZWXhzJkz2LhxI1q2bImYmBi8+OKL9/0ahBCIjY3F0KFDqz2+Y8cODB8+HOPGjcMff/yBDz74AIsXL8by5ctlfkqmLf2wHcL75CKsZx48/crw6JO56BCRj9O/2wEALp9TIP2wPV5++xKatyuGX9NSTJx/CcVFFtiz2blhgyeDDYq9gGtXFVjyRkuc+kOF7CtK/H7AFepLdg0dGhnKUkKFs/Xfm8oaAKCxs8TlGcEoCHPBLS9blDS1x7Xn/GB7vghW11n1rQ0aIRm8GSOT/jNp/PjxkCQJv/76K+zt7bXtrVq1QmxsLAoLCzFmzBg89dRT+Oijj7THR48eDQ8PDzz11FNYv349hg4diueeew6rV6/Ga6+9Bkn6+5v5ySefoGPHjmjbtu1d47Czs4OnpycAwM/PD126dEFISAhiY2MxZMgQ9OzZ8659ly5dCgC4du0a/ve//1U5vmbNGgwcOBDjxo0DAAQFBWHGjBlYsGABJkyYoBOrOWrduRDfrGmES2cV8G1SirPHbHHsV3uMe/MyAOBWWeXnY6P4+1FhlpaAtbXAsYMO6Dv8RoPETbWjS+R1HN7nhpnvHkVop5v486oC29b74NuNPg0dGhnIWl2KwElHIawklDSxx/XB3ih3V1R7rkVRBYQEaOwt6zlKepCYbIXoxo0b2LlzJyZMmKCTDN3m7OyMXbt24c8//8TUqVOrHI+KikKzZs3w+eefAwBGjRqFc+fOYe/evdpzCgsLsX79+ntWh+5mxIgRcHFxkT10djelpaWwtbXVaVMqlbh06RIuXLhw1z55eXk6m6kaMjEbkQNzMPqxEPR7qC0m9G6Op8dcQ/enbwIA/JqWwMO3DJ/M90L+TUvcKpPwxTJ33Mi2xo2rJv33glnw9C1B/yGXceWiHV4b1w7bv/TBuBmn8XhUVkOHRgYoaWKPq2P9cXlaU1yNfQiWubfg99ZJWOSXVzlXKtOg0foryA93gUbJhKg2aAwcLuODGevZmTNnIIRASEjIXc85deoUAKBFixbVHg8JCdGe07JlS4SFhSEpKUl7fP369aioqMCzzz6rd3wWFhZo1qwZzp8/r3ffO/Xp0webNm1CamoqNBoNTp06hcTERABAVlb1/+jPnz8fTk5O2s3Pz8+gGIzZ3q+ckbrRBf9+/wLe//Ykpr53ERs+dEfKehcAgJU18PrHGbh81haDWobiqSZt8PsvDuj8eB4s+G/nA0+yEDiT7oDkpU1w7oQjdmzwwc6N3ug/5HJDh0YGKGrrhILOLijzU6K4tQpXXm0CAFD99KfuieUCnh9kAELg2oiHGiBS03T7bfeGbMbIOKOqBUJUvk1XzpDR7XOra7+z/6hRo7Bhwwbk5+cDqBwui46OhrOzc41jvH39vn37wsHBAQ4ODmjVqpXsa4wZMwYTJ07Ek08+CRsbG3Tp0gUxMTEAAEvL6n+jz5w5E7m5udotMzOzRvE/CFa95Y2hE7MROfAmAluUoOegHESPuYZ1yzy05wS3KcaK3Sex6cT/8HnaH0hYew55OZbw9CttwMipNuRcs0HmOd0KcWaGHRp7ljRQRFQXhMISZb5KWF+94//ZcgGv98/B+loZLk8PZnWI7stkE6Lg4GBIkoT09PS7ntOsWTMAuOs5J06cQHBwsHY/JiYGkiThiy++wJkzZ/DTTz/VaLgMACoqKnD69GkEBgYCAD7++GOkpaUhLS0N27dvl30dSZKwYMECFBQU4MKFC1Cr1Xj44YcBAAEBAdX2USgUUKlUOpupKi2xgGShm/BaWApUlwPbqzRwdqvA5XM2OP27HcL7mO5Qork4nuYMn4AinTYf/2JkZ9nepQc9iKRbGlhfKUGFc+XEam0ypC7F5RlNoXHk8HdtqoBk8GaMTDYhcnV1RZ8+ffD++++jsLCwyvGbN2+id+/ecHV1xaJFi6oc37p1K06fPq0zHObo6IjBgwcjKSkJn3zyCYKCghAZGVmj+JKTk5GTk4NnnnkGAODj44OmTZuiadOm8Pf31/t6lpaW8PHxgY2NDT7//HOEh4fD3d29RrGZki698rBuqQcO7FZBnWmDn3c4YdNKd3R9Ild7zg9fO+H3fQ7IumCDfTtVmBnTFOFP5KJjZH4DRk61YfMaP4SE5mHI6PPw8itCZD81+g66jG3rfBs6NDJAo88vQXkiH1bXSqE4WwjPZedgUVyBvEdcgQoBr2XnoMgogvqlAEADWN68Bcubt4ByY33P+oPFVIfMTDpt/uCDD9C1a1c8/PDDmDNnDtq0aYPy8nKkpKRgxYoVSE9Px8qVK7VL4CdOnAiVSoXU1FRMmzYNgwYNwpAhQ3SuOWrUKDz66KM4fvw4pk6dKmtIrqioCGq1GuXl5bh8+TI2bdqEJUuW4KWXXkL37t3v2ffMmTMoKCiAWq1GcXEx0tLSAFTOabKxscH169exYcMGREZGoqSkBElJSfjyyy91Jn+bs/FzLyF5oReWz/TFzT+t4OZxC/2eu47hr1zVnnPjqjVWxvvg5nUruLqXo+fgygc30oPv9DEV5r4SipGTz2LY2PNQX7bFyoXB+H67Z0OHRgawunELnh+ch2V+OSpUVihpYo9Ls5ujvJECVtdK4XCk8g8e/9dO6PS7NDMYxS0cGyJkegBI4m4TaExEVlYW5s2bh23btiErKwuNGzdGx44d8corr2irOz/++CMSEhLwyy+/oLi4GE2bNkVsbCzi4uKqnYcTEhKC06dP48KFC/D11f1LMzIyEu3atdNObI6MjNQmJzY2NnBzc0PHjh0RGxuLp59++r7x39n/ThkZGQgICMD169cRFRWFo0ePQgiB8PBwzJs3D2FhYbI/o7y8PDg5OSHnVBBUjsaZuVPt6demR0OHQPXoxGL9K8704NEUlyDzxTnIzc2ts2kQt39XvHGgJ2wdrGt8nZKCW5gTtrtOY60Jk0+I6P6YEJkXJkTmhQmReajPhOi1/b0NTojmdtlldAmRSQ+ZERERUe0y1Ze7GmdURERERPWIFSIiIiKSTUCCxoCl88JIl90zISIiIiLZOGRGREREZKJYISIiIiLZNEKCRtR82MuQvnWJCRERERHJdvut9Yb0N0bGGRURERERKt/LKUlSlW3ChAkAKl+UHh8fD29vbyiVSkRGRuLYsWN634cJEREREcl2e8jMkE0fBw8eRFZWlnZLSUkBAAwePBgAsHDhQixevBjLly/HwYMH4enpiV69eiE/X7/3UTIhIiIiItk0sDB400fjxo3h6emp3bZt24YmTZogIiICQggkJiZi1qxZiI6ORuvWrZGcnIyioiKsXbtWr/swISIiIqJ6l5eXp7OVlpbet09ZWRn++9//IjY2FpIkISMjA2q1Gr1799aeo1AoEBERgX379ukVDxMiIiIikq1CSAZvAODn5wcnJyftNn/+/Pvee8uWLbh58yZGjhwJAFCr1QAADw8PnfM8PDy0x+TiKjMiIiKSrbaW3WdmZuq83FWhUNy373/+8x/07dsX3t7eOu2SpBuPEKJK2/0wISIiIiLZhLCAxoCnTYu/+qpUKr3edn/hwgXs3r0bmzZt0rZ5enoCqKwUeXl5aduzs7OrVI3uh0NmREREZPSSkpLg7u6O/v37a9sCAwPh6empXXkGVM4z2rt3L7p27arX9VkhIiIiItkqIKHCgBe01qSvRqNBUlISRowYASurv1MXSZIQFxeHhIQEBAcHIzg4GAkJCbCzs8OwYcP0ugcTIiIiIpJNIwx7/YZG6N9n9+7duHjxImJjY6scmz59OoqLizF+/Hjk5OQgLCwMu3btgqOjo173YEJERERERq13794QovpMSpIkxMfHIz4+3qB7MCEiIiIi2TQGTqo2pG9dYkJEREREsmkgQWPAHCJD+tYl40zTiIiIiOoRK0REREQk251Pm65pf2PEhIiIiIhkM9U5RMYZFREREVE9YoWIiIiIZNPAwHeZGemkaiZEREREJJswcJWZYEJERERED7raetu9seEcIiIiIjJ7rBARERGRbKa6yowJEREREcnGITMiIiIiE8UKEREREclmqu8yY0JEREREsnHIjIiIiMhEsUJEREREsplqhYgJEREREclmqgkRh8yIiIjI7LFCRERERLKZaoWICRERERHJJmDY0nlRe6HUKiZEREREJJupVog4h4iIiIjMHitEREREJJupVoiYEBEREZFsppoQcciMiIiIzB4rRERERCSbqVaImBARERGRbEJIEAYkNYb0rUscMiMiIiKzxwoRERERyaaBZNCDGQ3pW5eYEBEREZFspjqHiENmREREZNQuX76Mf/3rX3Bzc4OdnR3atWuHw4cPa48LIRAfHw9vb28olUpERkbi2LFjet2DCRERERHJdntStSGbPnJyctCtWzdYW1tjx44dOH78OBYtWgRnZ2ftOQsXLsTixYuxfPlyHDx4EJ6enujVqxfy8/Nl34dDZkRERCRbfQ+ZLViwAH5+fkhKStK2BQQEaP9bCIHExETMmjUL0dHRAIDk5GR4eHhg7dq1GDt2rKz7sEJEREREstVWhSgvL09nKy0trfZ+W7duRadOnTB48GC4u7ujffv2WLVqlfZ4RkYG1Go1evfurW1TKBSIiIjAvn37ZH9dTIiIiIio3vn5+cHJyUm7zZ8/v9rzzp07hxUrViA4OBjffvstxo0bh0mTJuHTTz8FAKjVagCAh4eHTj8PDw/tMTk4ZEZa3xYpYGdp2dBhUB2ruP5nQ4dA9cjN1a2hQ6B6UFFUisx6upcwcMjsdoUoMzMTKpVK265QKKo9X6PRoFOnTkhISAAAtG/fHseOHcOKFSvw/PPPa8+TJN2YhBBV2u6FFSIiIiKSTQAQwoDtr+uoVCqd7W4JkZeXF1q2bKnT1qJFC1y8eBEA4OnpCQBVqkHZ2dlVqkb3woSIiIiIjFa3bt1w8uRJnbZTp07B398fABAYGAhPT0+kpKRoj5eVlWHv3r3o2rWr7PtwyIyIiIhk00CCVI9Pqn7llVfQtWtXJCQkYMiQIfj111/x0Ucf4aOPPgJQOVQWFxeHhIQEBAcHIzg4GAkJCbCzs8OwYcNk34cJEREREclW3y937dy5MzZv3oyZM2dizpw5CAwMRGJiIoYPH649Z/r06SguLsb48eORk5ODsLAw7Nq1C46OjrLvw4SIiIiIjNqTTz6JJ5988q7HJUlCfHw84uPja3wPJkREREQkm0ZIkEzwXWZMiIiIiEi226vFDOlvjLjKjIiIiMweK0REREQkW31Pqq4vTIiIiIhINiZEREREZPZMdVI15xARERGR2WOFiIiIiGQz1VVmTIiIiIhItsqEyJA5RLUYTC3ikBkRERGZPVaIiIiISDauMiMiIiKzJ/7aDOlvjDhkRkRERGaPFSIiIiKSjUNmRERERCY6ZsaEiIiIiOQzsEIEI60QcQ4RERERmT1WiIiIiEg2PqmaiIiIzJ6pTqrmkBkRERGZPVaIiIiISD4hGTYx2kgrREyIiIiISDZTnUPEITMiIiIye6wQERERkXx8MCMRERGZO1NdZSYrIVq6dKnsC06aNKnGwRARERE1BFkJ0ZIlS2RdTJIkJkRERESmzkiHvQwhKyHKyMio6ziIiIjoAWCqQ2Y1XmVWVlaGkydPory8vDbjISIiImMmamEzQnonREVFRRg1ahTs7OzQqlUrXLx4EUDl3KG333671gMkIiIiqmt6J0QzZ87E77//ju+//x62trba9p49e+KLL76o1eCIiIjI2Ei1sBkfvROiLVu2YPny5XjkkUcgSX9/US1btsTZs2drNTgiIiIyMvU8ZBYfHw9JknQ2T0/Pv8MRAvHx8fD29oZSqURkZCSOHTum95eld0J07do1uLu7V2kvLCzUSZCIiIiIakOrVq2QlZWl3Y4ePao9tnDhQixevBjLly/HwYMH4enpiV69eiE/P1+ve+idEHXu3BnffPONdv92ErRq1SqEh4frezkiIiJ6kDTApGorKyt4enpqt8aNG1eGIgQSExMxa9YsREdHo3Xr1khOTkZRURHWrl2r3z30DWr+/Pl44okncPz4cZSXl+O9997DsWPH8Msvv2Dv3r36Xo6IiIgeJLX0tvu8vDydZoVCAYVCUW2X06dPw9vbGwqFAmFhYUhISEBQUBAyMjKgVqvRu3dvnetERERg3759GDt2rOyw9K4Qde3aFT///DOKiorQpEkT7Nq1Cx4eHvjll1/QsWNHfS9HREREZsjPzw9OTk7abf78+dWeFxYWhk8//RTffvstVq1aBbVaja5du+LPP/+EWq0GAHh4eOj08fDw0B6Tq0bvMgsNDUVycnJNuhIREdEDTIjKzZD+AJCZmQmVSqVtv1t1qG/fvtr/Dg0NRXh4OJo0aYLk5GR06dIFAKrMYRZC6D2vuUYJUUVFBTZv3oz09HRIkoQWLVpgwIABsLLiu2KJiIhMWi297V6lUukkRHLZ29sjNDQUp0+fxsCBAwEAarUaXl5e2nOys7OrVI3uR+8M5o8//sCAAQOgVqvRvHlzAMCpU6fQuHFjbN26FaGhofpekoiIiEiW0tJSpKen49FHH0VgYCA8PT2RkpKC9u3bA6h8k8bevXuxYMECva6r9xyi0aNHo1WrVrh06RJ+++03/Pbbb8jMzESbNm3w4osv6ns5IiIiepDcnlRtyKaHqVOnYu/evcjIyMCBAwcwaNAg5OXlYcSIEZAkCXFxcUhISMDmzZvxxx9/YOTIkbCzs8OwYcP0uo/eFaLff/8dhw4dgouLi7bNxcUF8+bNQ+fOnfW9HBERET1AJFG5GdJfH5cuXcKzzz6L69evo3HjxujSpQv2798Pf39/AMD06dNRXFyM8ePHIycnB2FhYdi1axccHR31uo/eCVHz5s1x9epVtGrVSqc9OzsbTZs21fdyRERE9CCppTlEcq1bt+6exyVJQnx8POLj42seE2QOmeXl5Wm3hIQETJo0CRs2bMClS5dw6dIlbNiwAXFxcXqP1xEREREZA1kVImdnZ53la0IIDBkyRNsm/lpDFxUVhYqKijoIk4iIiIxCLT2Y0djISoj27NlT13EQERHRg6Ceh8zqi6yEKCIioq7jICIiImowNX6SYlFRES5evIiysjKd9jZt2hgcFBERERkpc64Q3enatWt44YUXsGPHjmqPcw4RERGRCTPRhEjvBzPGxcUhJycH+/fvh1KpxM6dO5GcnIzg4GBs3bq1LmIkIiIiqlN6V4i+++47fPXVV+jcuTMsLCzg7++PXr16QaVSYf78+ejfv39dxElERETGwERXmeldISosLIS7uzsAwNXVFdeuXQNQ+Qba3377rXajIyIiIqNy+0nVhmzGqEZPqj558iQCAgLQrl07rFy5EgEBAfjwww913jT7IIuMjES7du2QmJhYZ/cICAhAXFwc4uLi7npOfHw8tmzZgrS0tDqLw9QlRTRB/mXrKu2hw3PQ/c2rKLpuiZ8XuuPiT3YozbOEd+ciRM6+CueAWw0QLdW21mEFGDz+GoJDi+DmWY742AD8stOpocMiAyk/uw7l5zd02jTOlrj53yYAACmnHHarr8P6SCGkQg1utVKiaKw7ND42DREuPSD0Toji4uKQlZUFAJg9ezb69OmDzz77DDY2Nli9enVtx2c2JEnC5s2bMXDgwIYOxaQM3XQeQvP3/p+nFNgy4iEE982HEMC2cb6wsBZ48sPLsHHQ4MgnLtj8/EP4185zsLYz0j9jSDZbOw3OHbPFrnUueOM/Fxo6HKpF5Q/ZIH+e798Nt8c7hIDj3CsQVhLyX/OBsLOA7ZYcOL52CbkrAgBbvQdG6J9MdFK13gnR8OHDtf/dvn17nD9/HidOnMBDDz2ERo0a1WpwRIayc9Nd9Xh4pQOcHiqDT1gRbp63hjpNieHbz8GtWeXjIyLfvIqPw1Q4+bUKrYfmNkTIVIsO7VHh0B7VX3tMiEyKpQThUvVXmMWVW7A6WYLc9/1R4a8AABS95A7nf52FYm8+SvuwQkjVMzhVtrOzQ4cOHUwuGdJoNJg+fTpcXV3h6emp89K43NxcvPjii3B3d4dKpcLjjz+O33//XXv87NmzGDBgADw8PODg4IDOnTtj9+7dd71XQEAAAODpp5+GJEna/dvWrFmDgIAAODk5ISYmBvn5+QCATz/9FG5ubigtLdU5/5lnnsHzzz9v2AdggirKgBNfqdByUC4kCagoq/zxt1T8/eeKhSVgYS2QddiuocIkIhksr5TB+fmzcBp1DvYLsmChrvyjRrpV+f+zsLlj4q6lBFhJsDpe3BChmhwJBs4haugv4C5kVYimTJki+4KLFy+ucTDGJDk5GVOmTMGBAwfwyy+/YOTIkejWrRt69uyJ/v37w9XVFdu3b4eTkxNWrlyJHj164NSpU3B1dUVBQQH69euHuXPnwtbWFsnJyYiKisLJkyfx0EMPVbnXwYMH4e7ujqSkJDzxxBOwtLTUHjt79iy2bNmCbdu2IScnB0OGDMHbb7+NefPmYfDgwZg0aRK2bt2KwYMHAwCuX7+Obdu2YefOnXf92kpLS3WSqLy8vFr85IzX2RRHlOZZosUzlZUfl6BSOPrcwr53G+PxuWpYKzU48okriq5ZoTDb8j5XI6KGUt5ciYIpCmh8bCDdLIdy3Q2opmYi94MAVPjaoMLdCsrk6yia6AGhqBwys8ipgMWN8oYOnYyYrIToyJEjsi525wtgH3Rt2rTB7NmzAQDBwcFYvnw5UlNTYWlpiaNHjyI7OxsKRWU59t1338WWLVuwYcMGvPjii2jbti3atm2rvdbcuXOxefNmbN26FRMnTqxyr8aNGwOofImup6enzjGNRoPVq1fD0dERAPDcc88hNTUV8+bNg1KpxLBhw5CUlKRNiD777DP4+voiMjLyrl/b/Pnz8eabb9b8w3lAHf/SCf6PFcLBo/IfRUtroP/7l7B7phc+6tgMkqWAX9dC+EcUNHCkRHQvtzrZ37GnQH6IEs6jM6BIzUPJ0y4o+D9v2L93FS4xZyEsgFvt7FDWkVXfWmOiy+75cte7+OcrSLy8vJCdnY3Dhw+joKAAbm5uOseLi4tx9uxZAJWPJnjzzTexbds2XLlyBeXl5SguLsbFixf1jiMgIECbDN0Zx21jxoxB586dcfnyZfj4+CApKQkjR468Z3I6c+ZMnapfXl4e/Pz89I7tQZJ32QqZ++zR7/3LOu3urUsx7OvzKM23QEWZBDu3CnzxjD/cW5c0UKREpDdbC1QEKGBxpXLYrKKpLfKW+UMqrADKBYSTFVRTLqI8WNHAgZoITqo2L9bWuku1JUmCRqOBRqOBl5cXvv/++yp9nJ2dAQDTpk3Dt99+i3fffRdNmzaFUqnEoEGDqrz3zZA4bmvfvj3atm2LTz/9FH369MHRo0fx9ddf3/OaCoVCW90yF8c3OEPpVoHA7tVXfxSOlZ/pzfPWyD5qiy5x1+ozPCIyxC0NLDPLcKuVUqdZ2FcOfVtcLoPlmRIU/cutut5EAJgQ6a1Dhw5Qq9WwsrKqMvn5th9//BEjR47E008/DQAoKCjA+fPn73lda2vrGr8HbvTo0ViyZAkuX76Mnj17mny1R19CA6RvdEKLp3Nh8Y+f+NPbHaF0rYCj9y1cP6nAD3M9ENSrAP6PFjVMsFSrbO0q4B349x8inn5lCGpVjPyblrh2mc+keVAp/3MNtx62h6axNaTcyjlEUpEGZT0qVxRa/5QPobKExt0KlufLYPdRNm51cUB5B/v7XJlkYYWIAKBnz54IDw/HwIEDsWDBAjRv3hxXrlzB9u3bMXDgQHTq1AlNmzbFpk2bEBUVBUmS8Prrr+tUdaoTEBCA1NRUdOvWDQqFAi4uLrJjGj58OKZOnYpVq1bh008/NfRLNDkXf7ZD/hVrtBx8s8qxwmtW+DHBHUV/WsG+cTlCns7FwxOu13+QVCeatS3GOxvPavfHvXkFALDrCxcseqXqAgd6MFhcL4fDO1mQ8iogVJYoD1Eid5EfNO6VFXWLG+Ww/fgaLG6WQ+NihbLHVSiOYXWothj6tGmTeVK1uZMkCdu3b8esWbMQGxuLa9euwdPTE4899hg8PDwAAEuWLEFsbCy6du2KRo0aYcaMGfddybVo0SJMmTIFq1atgo+Pz30rSndSqVR45pln8M033/DBjtXwf7QIk86cqPZYuxE5aDcip54jovryv18c0Me77f1PpAdK4Yx7vxWh9CkXlD4l/49KIgCQhBBGmquRPnr16oUWLVpg6dKlevfNy8uDk5MT1qW1gJ0jl5ubuqVNQxo6BKpHN7Y1a+gQqB5UFJXi98GLkJubC5VKdf8ONXD7d0XA3HmwsLWt8XU0JSU4/9qsOo21Jmr0YMY1a9agW7du8Pb2xoULlU9/TUxMxFdffVWrwdH93bhxA+vWrcN3332HCRMmNHQ4RERk6kQtbEZI74RoxYoVmDJlCvr164ebN29qJwI7OzvX6ctQqXodOnTA2LFjtfOZiIiISH96J0TLli3DqlWrMGvWLJ0nKnfq1AlHjx6t1eDo/s6fP4/c3FxMnTq1oUMhIiIzYNBrOwyckF2X9J5UnZGRgfbt21dpVygUKCwsrJWgiIiIyEiZ6JOq9a4QBQYGIi0trUr7jh070LJly9qIiYiIiIyVic4h0rtCNG3aNEyYMAElJSUQQuDXX3/F559/jvnz5+Pjjz+uixiJiIiI6pTeCdELL7yA8vJyTJ8+HUVFRRg2bBh8fHzw3nvvISYmpi5iJCIiIiPBBzPeYcyYMRgzZgyuX78OjUYDd3f32o6LiIiIjBFf3VFVo0aNaisOIiIiogajd0IUGBgISbr7DPFz584ZFBAREREZMUOXzhtphUjvVWZxcXGYPHmydhs/fjzCw8ORm5uLF198sS5iJCIiImPRgKvM5s+fD0mSEBcX93c4QiA+Ph7e3t5QKpWIjIzEsWPH9L623hWiyZMnV9v+/vvv49ChQ3oHQERERHQ/Bw8exEcffYQ2bdrotC9cuBCLFy/G6tWr0axZM8ydOxe9evXCyZMn4ejoKPv6NXqXWXX69u2LjRs31tbliIiIyBg1QIWooKAAw4cPx6pVq+Di4vJ3KEIgMTERs2bNQnR0NFq3bo3k5GQUFRVh7dq1et2j1hKiDRs2wNXVtbYuR0REREaotl7dkZeXp7OVlpbe9Z4TJkxA//790bNnT532jIwMqNVq9O7dW9umUCgQERGBffv26fV16T1k1r59e51J1UIIqNVqXLt2DR988IG+lyMiIiIz5Ofnp7M/e/ZsxMfHVzlv3bp1+O2333Dw4MEqx9RqNQDAw8NDp93DwwMXLlzQKx69E6KBAwfq7FtYWKBx48aIjIxESEiIvpcjIiIiM5SZmQmVSqXdVygU1Z4zefJk7Nq1C7a2tne91j9Xvwsh7rkivjp6JUTl5eUICAhAnz594OnpqdeNiIiIyATU0oMZVSqVTkJUncOHDyM7OxsdO3bUtlVUVOCHH37A8uXLcfLkSQCVlSIvLy/tOdnZ2VWqRvej1xwiKysrvPTSS/cc5yMiIiLTVVtziOTo0aMHjh49irS0NO3WqVMnDB8+HGlpaQgKCoKnpydSUlK0fcrKyrB371507dpVr69L7yGzsLAwHDlyBP7+/vp2JSIiIpLN0dERrVu31mmzt7eHm5ubtj0uLg4JCQkIDg5GcHAwEhISYGdnh2HDhul1L70TovHjx+PVV1/FpUuX0LFjR9jb2+sc/+fzAYiIiMjEGNHTpqdPn47i4mKMHz8eOTk5CAsLw65du/R6BhGgR0IUGxuLxMREDB06FAAwadIk7TFJkrQTmCoqKvQKgIiIiB4gDfxy1++//15nX5IkxMfHV7tCTR+yE6Lk5GS8/fbbyMjIMOiGRERERMZGdkIkRGVKx7lDRERE5kvfidHV9TdGes0h0ndNPxEREZmYBh4yqyt6JUTNmjW7b1J048YNgwIiIiIiqm96JURvvvkmnJyc6ioWIiIiMnIcMgMQExMDd3f3uoqFiIiIjJ2JDpnJflI15w8RERGRqdJ7lRkRERGZMROtEMlOiDQaTV3GQURERA8AziEiIiIiMtEKkV5vuyciIiIyRawQERERkXwmWiFiQkRERESymeocIg6ZERERkdljhYiIiIjk45AZERERmTsOmRERERGZKFaIiIiISD4OmREREZHZM9GEiENmREREZPZYISIiIiLZpL82Q/obIyZEREREJJ+JDpkxISIiIiLZuOyeiIiIyESxQkRERETycciMiIiICEab1BiCQ2ZERERk9lghIiIiItlMdVI1EyIiIiKSz0TnEHHIjIiIiMweK0REREQkm6kOmbFCRERERPKJWtj0sGLFCrRp0wYqlQoqlQrh4eHYsWPH3+EIgfj4eHh7e0OpVCIyMhLHjh3T+8tiQkRERERGy9fXF2+//TYOHTqEQ4cO4fHHH8eAAQO0Sc/ChQuxePFiLF++HAcPHoSnpyd69eqF/Px8ve7DITPSmvrTUFgobRs6DKpjLRplNHQIVI9eDU5p6BCoHhQVVGBMPd2rvofMoqKidPbnzZuHFStWYP/+/WjZsiUSExMxa9YsREdHAwCSk5Ph4eGBtWvXYuzYsbLvwwoRERERyVdLQ2Z5eXk6W2lp6X1vXVFRgXXr1qGwsBDh4eHIyMiAWq1G7969tecoFApERERg3759en1ZTIiIiIhIvlpKiPz8/ODk5KTd5s+ff9dbHj16FA4ODlAoFBg3bhw2b96Mli1bQq1WAwA8PDx0zvfw8NAek4tDZkRERFTvMjMzoVKptPsKheKu5zZv3hxpaWm4efMmNm7ciBEjRmDv3r3a45Ik6ZwvhKjSdj9MiIiIiEi22ppDdHvVmBw2NjZo2rQpAKBTp044ePAg3nvvPcyYMQMAoFar4eXlpT0/Ozu7StXofjhkRkRERPLV87L7akMQAqWlpQgMDISnpydSUv5ePFBWVoa9e/eia9euel2TFSIiIiIyWv/3f/+Hvn37ws/PD/n5+Vi3bh2+//577Ny5E5IkIS4uDgkJCQgODkZwcDASEhJgZ2eHYcOG6XUfJkREREQkmyQEJFHzMo++fa9evYrnnnsOWVlZcHJyQps2bbBz50706tULADB9+nQUFxdj/PjxyMnJQVhYGHbt2gVHR0e97sOEiIiIiOSr55e7/uc//7nncUmSEB8fj/j4+JrHBM4hIiIiImKFiIiIiOQz1Ze7MiEiIiIi+ep5yKy+cMiMiIiIzB4rRERERCQbh8yIiIiITHTIjAkRERERyWaqFSLOISIiIiKzxwoRERERycchMyIiIiLjHfYyBIfMiIiIyOyxQkRERETyCVG5GdLfCDEhIiIiItm4yoyIiIjIRLFCRERERPJxlRkRERGZO0lTuRnS3xhxyIyIiIjMHitEREREJB+HzIiIiMjcmeoqMyZEREREJJ+JPoeIc4iIiIjI7LFCRERERLJxyIyIiIjIRCdVc8iMiIiIzB4rRERERCQbh8yIiIiIuMqMiIiIyDSxQkRERESycciMiIiIiKvMiIiIiEwTK0REREQkm6kOmbFCRERERPJphOGbHubPn4/OnTvD0dER7u7uGDhwIE6ePKlzjhAC8fHx8Pb2hlKpRGRkJI4dO6bXfZgQERERkXyiFjY97N27FxMmTMD+/fuRkpKC8vJy9O7dG4WFhdpzFi5ciMWLF2P58uU4ePAgPD090atXL+Tn58u+D4fMiIiIyGjt3LlTZz8pKQnu7u44fPgwHnvsMQghkJiYiFmzZiE6OhoAkJycDA8PD6xduxZjx46VdR9WiIiIiEg2CX/PI6rR9td18vLydLbS0lJZ98/NzQUAuLq6AgAyMjKgVqvRu3dv7TkKhQIRERHYt2+f7K+LCRERERHJd/tJ1YZsAPz8/ODk5KTd5s+fL+PWAlOmTMEjjzyC1q1bAwDUajUAwMPDQ+dcDw8P7TE5OGRGRERE9S4zMxMqlUq7r1Ao7ttn4sSJ+N///oeffvqpyjFJknT2hRBV2u6FCRERERHJVlvL7lUqlU5CdD8vv/wytm7dih9++AG+vr7adk9PTwCVlSIvLy9te3Z2dpWq0b1wyIyIiIjkq+dVZkIITJw4EZs2bcJ3332HwMBAneOBgYHw9PRESkqKtq2srAx79+5F165dZd+HFSIiIiIyWhMmTMDatWvx1VdfwdHRUTsvyMnJCUqlEpIkIS4uDgkJCQgODkZwcDASEhJgZ2eHYcOGyb4PEyIiIiKSTRICkqj5mJm+fVesWAEAiIyM1GlPSkrCyJEjAQDTp09HcXExxo8fj5ycHISFhWHXrl1wdHSUfR8mRERERCSf5q/NkP56EDISKEmSEB8fj/j4+JrFBM4hIiIiImKFiIiIiOSr7yGz+sKEiIiIiOSrwUqxKv2NEBMiIiIiku+Op03XuL8R4hwiIiIiMnusEBEREZFstfWkamPDhIhMmttXl+H2dZZOW7nKCucWt6vcEQJuW6/A6YfrsCgqR0mgPbKH+6PMR1n/wVKdcHMvxQtxZ9DpkT9ho9Dg8gU7vDc7BGfS5b8ygIzLl4/7oOBy1V9fIcPyET77BpKa+1fbr9O0HISOzqvr8EyfiQ6ZMSF6AK1evRpxcXG4efNmQ4fyQCj1tsWlV5v/3XDHQLHLTjWcU67i6guBKPO0heu2K/BdfAoZ81pD2FrWf7BUqxwcb+Hd5MP430FnvDG+HW7esIaXXzEK8vlP34MsakMWNBV/7988bYNvX/BAwBOFAIChP2XqnH/5ByV+muWGgD5F9RkmPWA4h+gfJEnCli1bGjoMrYCAACQmJuq0DR06FKdOnWqYgB5AwlJChZP135uj9V8HBFx2Z+NGfy8UdHRBmY8SV2MDIZVpoDpwo2GDploxKPYCrl1VYMkbLXHqDxWyryjx+wFXqC/ZNXRoZABbVw3sGv+9Ze5RwvGhW/B8uBQAdI7ZNdbgYqodvMJK4OhX3sCRmwZJY/hmjPhn0gNIqVRCqeSQjlw2V0sR9OrvENYSigPt8We0L241VsD6ehmscm+hqJWT9lxhbYHi5o6wPVOA3IjGDRg11YYukddxeJ8bZr57FKGdbuLPqwpsW++Dbzf6NHRoVEsqyoCzW+3R6oU8SFLV48XXLZC5V4lH375e/8GZKhMdMmvQCtGGDRsQGhoKpVIJNzc39OzZE3v37oW1tbX25W23vfrqq3jssccAVA4ZOTs7Y9u2bWjevDns7OwwaNAgFBYWIjk5GQEBAXBxccHLL7+Mioq/66oBAQF46623MGzYMDg4OMDb2xvLli3TOQ4ATz/9NCRJ0u4Dle9SadKkCWxsbNC8eXOsWbNGJz5JkrBy5Uo8+eSTsLOzQ4sWLfDLL7/gzJkziIyMhL29PcLDw3H27Fltn7Nnz2LAgAHw8PCAg4MDOnfujN27d2uPR0ZG4sKFC3jllVcgSRKkv/5vv/3132nr1q3o1KkTbG1t0ahRI0RHR9/1cy8tLUVeXp7OZqqKgxygHhWIS68E4+rzAbDKvQW/+emwKCiHZe4tAJVziu5UrrKCVd6thgiXapmnbwn6D7mMKxft8Nq4dtj+pQ/GzTiNx6Oy7t+ZHggXd9uhLN8CwU8XVnv8zGYHWNtr4N+bw2V0bw2WEGVlZeHZZ59FbGws0tPT8f333yM6OhodO3ZEUFCQTsJRXl6O//73v3jhhRe0bUVFRVi6dCnWrVuHnTt3avtv374d27dvx5o1a/DRRx9hw4YNOvd955130KZNG/z222+YOXMmXnnlFaSkpAAADh48CKDyhXFZWVna/c2bN2Py5Ml49dVX8ccff2Ds2LF44YUXsGfPHp1rv/XWW3j++eeRlpaGkJAQDBs2DGPHjsXMmTNx6NAhAMDEiRO15xcUFKBfv37YvXs3jhw5gj59+iAqKgoXL14EAGzatAm+vr6YM2cOsrKykJVV/T/i33zzDaKjo9G/f38cOXIEqamp6NSp010/+/nz58PJyUm7+fn53fub9QArCnWqHA7ztUNRSxUuTw4GAKj23f2vRWNdAUH6kywEzqQ7IHlpE5w74YgdG3ywc6M3+g+53NChUS05tdEBvo8Vw86jotrjpzc6oElUIawU9RyYKRO1sBmhBhsyy8rKQnl5OaKjo+HvX7kiIDQ0FAAwatQoJCUlYdq0aQAqf+EXFRVhyJAh2v63bt3SVm0AYNCgQVizZg2uXr0KBwcHtGzZEt27d8eePXswdOhQbb9u3brh3//+NwCgWbNm+Pnnn7FkyRL06tULjRtXDpE4OzvD09NT2+fdd9/FyJEjMX78eADAlClTsH//frz77rvo3r279rwXXnhBG+OMGTMQHh6O119/HX369AEATJ48WSepa9u2Ldq2bavdnzt3LjZv3oytW7di4sSJcHV1haWlJRwdHXXi+ad58+YhJiYGb775ps6172bmzJmYMmWKdj8vL8+kk6I7CYUlSn2UsLlaisL2LgAAq7xyVDjbaM+xzC9Hucq6oUKkWpRzzQaZ5+x12jIz7NCtZ3YDRUS1qeCyJbL22aL7smvVHlcfUiA3wxqRidUfp5ox1Vd3NFiFqG3btujRowdCQ0MxePBgrFq1Cjk5OQCAkSNH4syZM9i/fz8A4JNPPsGQIUNgb//3P2x2dnbaZAgAPDw8EBAQAAcHB5227Gzdf/jCw8Or7Kenp98z1vT0dHTr1k2nrVu3blX6tWnTRufewN9J3u22kpIS7RBVYWEhpk+fjpYtW8LZ2RkODg44ceKEtkIkV1paGnr06CH7fIVCAZVKpbOZC+mWBjbqEpQ7W+NWIxuUO1nD7lju3yeUa6A8mY+Spg53vwg9MI6nOcMnQHeoxMe/GNlZtg0UEdWm05scYOtWAb/I4uqPb3CAW6tSuIZwCJzur8ESIktLS6SkpGDHjh1o2bIlli1bhubNmyMjIwPu7u6IiopCUlISsrOzsX37dsTGxur0t7bW/QtekqRq2zSa+09nl6qbiXefc4QQVdruvP/tY9W13Y5p2rRp2LhxI+bNm4cff/wRaWlpCA0NRVlZ2X3juRMnWN9do/WZUJ7Mh9W1UtieK4DXirOwKK5AXlc3QJKQ09MdrtvVcPgtBzaXi+H5yXkIGwvkhbk2dOhUCzav8UNIaB6GjD4PL78iRPZTo++gy9i2zrehQyMDCU1lQtR0YCEsqhnrKCuQcH6nHZoNLqj/4Ezd7UnVhmxGqEFXmUmShG7duqFbt25444034O/vj82bN2PKlCkYPXo0YmJi4OvriyZNmlSp0NTU7arTnfshISHafWtra52J2ADQokUL/PTTT3j++ee1bfv27UOLFi0MiuXHH3/EyJEj8fTTTwOonFN0/vx5nXNsbGyqxPNPbdq0QWpqqs5wHFWyyimD10fnYFlQjgpHKxQH2SPz/1qg3K1yQkHOE56wKNPA/bOLsCgsR0mQPS5NacZnEJmI08dUmPtKKEZOPothY89DfdkWKxcG4/vtdx+CpgfDlX22KLxiheBnqk94Mr6xhxBA0JPVT7YmAwgAhiydN858qOESogMHDiA1NRW9e/eGu7s7Dhw4gGvXrmmTjD59+sDJyQlz587FnDlzau2+P//8MxYuXIiBAwciJSUFX375Jb755hvt8YCAAKSmpqJbt25QKBRwcXHBtGnTMGTIEHTo0AE9evTA119/jU2bNumsCKuJpk2bYtOmTYiKioIkSXj99derVLQCAgLwww8/ICYmBgqFAo0aNapyndmzZ6NHjx5o0qQJYmJiUF5ejh07dmD69OkGxWcK1GOb3PsEScKfA3zw5wAuwzZVv/7QCL/+UPX/G3qw+TxSghdOXrjr8eZDC9B8KKtDdYFziGqZSqXCDz/8gH79+qFZs2Z47bXXsGjRIvTt27cyMAsLjBw5EhUVFTqVGUO9+uqrOHz4MNq3b4+33noLixYt0k56BoBFixYhJSUFfn5+aN++PQBg4MCBeO+99/DOO++gVatWWLlyJZKSkhAZGWlQLEuWLIGLiwu6du2KqKgo9OnTBx06dNA5Z86cOTh//jyaNGminfT9T5GRkfjyyy+xdetWtGvXDo8//jgOHDhgUGxERETmRBLCSFM1AGPGjMHVq1exdevWWrleQEAA4uLiEBcXVyvXMxV5eXlwcnKC7/J4WCg52dTUtfh3RkOHQPXo+Z9/a+gQqB4UFVRgTIffkJubW2cLZW7/rni83b9hZVnz5xiUV5Tiu7S36zTWmjDKJ1Xn5ubi4MGD+Oyzz/DVV181dDhERER0m4k+qdooE6IBAwbg119/xdixY9GrV6+GDoeIiIhMnFEmRN9//32dXPefK7iIiIhITxoA939azb37GyGjTIiIiIjIOHGVGREREZGJYoWIiIiI5OOkaiIiIjJ7JpoQcciMiIiIzB4rRERERCSfiVaImBARERGRfFx2T0REROaOy+6JiIiITBQTIiIiIpLv9hwiQzY9/PDDD4iKioK3tzckScKWLVv+EY5AfHw8vL29oVQqERkZiWPHjun9ZTEhIiIiIvk0wvBND4WFhWjbti2WL19e7fGFCxdi8eLFWL58OQ4ePAhPT0/06tUL+fn5et2Hc4iIiIjIaPXt2xd9+/at9pgQAomJiZg1axaio6MBAMnJyfDw8MDatWsxduxY2fdhhYiIiIjkq6Uhs7y8PJ2ttLRU71AyMjKgVqvRu3dvbZtCoUBERAT27dun17WYEBEREZEeDE2GKhMiPz8/ODk5abf58+frHYlarQYAeHh46LR7eHhoj8nFITMiIiKqd5mZmVCpVNp9hUJR42tJku6DkYQQVdruhwkRERERyVdLT6pWqVQ6CVFNeHp6AqisFHl5eWnbs7Ozq1SN7odDZkRERCRfPa8yu5fAwEB4enoiJSVF21ZWVoa9e/eia9euel2LFSIiIiIyWgUFBThz5ox2PyMjA2lpaXB1dcVDDz2EuLg4JCQkIDg4GMHBwUhISICdnR2GDRum132YEBEREZF8QlO5GdJfD4cOHUL37t21+1OmTAEAjBgxAqtXr8b06dNRXFyM8ePHIycnB2FhYdi1axccHR31ug8TIiIiIpKvnt92HxkZCXGPPpIkIT4+HvHx8TWPCUyIiIiISB+av5fO17y/8eGkaiIiIjJ7rBARERGRfPU8ZFZfmBARERGRfAIGJkS1Fkmt4pAZERERmT1WiIiIiEg+DpkRERGR2dNoABjwHCKNAX3rEIfMiIiIyOyxQkRERETycciMiIiIzJ6JJkQcMiMiIiKzxwoRERERyWeir+5gQkRERESyCaGBMOBt94b0rUtMiIiIiEg+IQyr8nAOEREREZFxYoWIiIiI5BMGziEy0goREyIiIiKST6MBJAPmARnpHCIOmREREZHZY4WIiIiI5OOQGREREZk7odFAGDBkZqzL7jlkRkRERGaPFSIiIiKSj0NmREREZPY0ApBMLyHikBkRERGZPVaIiIiISD4hABjyHCLjrBAxISIiIiLZhEZAGDBkJpgQERER0QNPaGBYhYjL7omIiIiMEitEREREJBuHzIiIiIhMdMiMCRFps3VNcUkDR0L1oVxT1tAhUD0qKqho6BCoHhT/9X2uj+pLOW4Z9FzGctyqvWBqkSSMtXZF9ebSpUvw8/Nr6DCIiMhAmZmZ8PX1rZNrl5SUIDAwEGq12uBreXp6IiMjA7a2trUQWe1gQkTQaDS4cuUKHB0dIUlSQ4dTb/Ly8uDn54fMzEyoVKqGDofqEL/X5sNcv9dCCOTn58Pb2xsWFnW3XqqkpARlZYZXmW1sbIwqGQI4ZEYALCws6uwvigeBSqUyq384zRm/1+bDHL/XTk5OdX4PW1tbo0tkaguX3RMREZHZY0JEREREZo8JEZkthUKB2bNnQ6FQNHQoVMf4vTYf/F5TTXFSNREREZk9VoiIiIjI7DEhIiIiIrPHhIiIiIjMHhMiImpwkZGRiIuLq9N7BAQEIDEx8Z7nxMfHo127dnUaB5mP1atXw9nZuaHDIJmYEFGDU6vVePnllxEUFASFQgE/Pz9ERUUhNTVVe86+ffvQr18/uLi4wNbWFqGhoVi0aBEqKirf37Nx40ZYWlri4sWL1d4jJCQEkyZNAlD1l29kZCQkSYIkSVAoFPDx8UFUVBQ2bdokK/7JkyejY8eOUCgUd/1lun79erRr1w52dnbw9/fHO++8I+vaVHckScKWLVsaOgyqIWP7/lWXcA8dOhSnTp1qmIBIb0yIqEGdP38eHTt2xHfffYeFCxfi6NGj2LlzJ7p3744JEyYAADZv3oyIiAj4+vpiz549OHHiBCZPnox58+YhJiYGQgg89dRTcHNzQ3JycpV7/Pzzzzh58iRGjRp11zjGjBmDrKwsnDlzBhs3bkTLli0RExODF1988b5fgxACsbGxGDp0aLXHd+zYgeHDh2PcuHH4448/8MEHH2Dx4sVYvny5zE+JiB5ESqUS7u7uDR0GySWIGlDfvn2Fj4+PKCgoqHIsJydHFBQUCDc3NxEdHV3l+NatWwUAsW7dOiGEEFOmTBFBQUFCo9HonBcbGys6duyo3Y+IiBCTJ0++6/5tn3zyiQAgUlJSZH0ts2fPFm3btq3S/uyzz4pBgwbptC1ZskT4+vpWidVcRUREiJdffllMmzZNuLi4CA8PDzF79mzt8Zs3b4oxY8aIxo0bC0dHR9G9e3eRlpamPX7mzBnx1FNPCXd3d2Fvby86depU5fvm7+8vlixZov1vVL6vWwAQ/v7+Qoi/v4effvqp8Pf3FyqVSgwdOlTk5eUJIYRITk4Wrq6uoqSkROfa0dHR4rnnnqv9D8ZIfPnll6J169bC1tZWuLq6ih49eojvv/9eWFlZiaysLJ1zp0yZIh599FEhhBBJSUnCyclJfP3116JZs2ZCqVSKZ555RhQUFIjVq1cLf39/4ezsLCZOnCjKy8u11/D39xdz5swRzz77rLC3txdeXl5i6dKlOser+/4JIcQHH3wggoKChLW1tWjWrJn49NNPdeIDID788EPRv39/oVQqRUhIiNi3b584ffq0iIiIEHZ2dqJLly7izJkz2j73+/mKiIjQief2r9bbX/+dvvrqK9GxY0ehUCiEm5ubePrpp2v2TaFaxwoRNZgbN25g586dmDBhAuzt7ascd3Z2xq5du/Dnn39i6tSpVY5HRUWhWbNm+PzzzwEAo0aNwrlz57B3717tOYWFhVi/fv09q0N3M2LECLi4uMgeOrub0tLSKu/+USqVuHTpEi5cuGDQtU1JcnIy7O3tceDAASxcuBBz5sxBSkoKhBDo378/1Go1tm/fjsOHD6NDhw7o0aMHbty4AQAoKChAv379sHv3bhw5cgR9+vRBVFTUXYdQDx48CABISkpCVlaWdh8Azp49iy1btmDbtm3Ytm0b9u7di7fffhsAMHjwYFRUVGDr1q3a869fv45t27bhhRdeqKuPpkFlZWXh2WefRWxsLNLT0/H9998jOjoaHTt2RFBQENasWaM9t7y8HP/97391PouioiIsXboU69atw86dO7X9t2/fju3bt2PNmjX46KOPsGHDBp37vvPOO2jTpg1+++03zJw5E6+88gpSUlIA3P37t3nzZkyePBmvvvoq/vjjD4wdOxYvvPAC9uzZo3Ptt956C88//zzS0tIQEhKCYcOGYezYsZg5cyYOHToEAJg4caL2/Pv9fG3atAm+vr6YM2cOsrKykJWVVe1n+c033yA6Ohr9+/fHkSNHkJqaik6dOtXo+0J1oKEzMjJfBw4cEADEpk2b7nrO22+/LQCInJycao8/9dRTokWLFtr9sLAw8fzzz2v3P/nkE6FUKnX6y60Q3b5e3759ZX09d6sQrVy5UtjZ2Yndu3eLiooKcfLkSRESEiIAiH379sm6tqmLiIgQjzzyiE5b586dxYwZM0RqaqpQqVRVqjJNmjQRK1euvOs1W7ZsKZYtW6bdv7NCJERlpWDz5s06fWbPni3s7Oy0FSEhhJg2bZoICwvT7r/00ks6PxOJiYnVViZNxeHDhwUAcf78+SrHFixYoPP/35YtW4SDg4O24puUlCQA6FRbxo4dK+zs7ER+fr62rU+fPmLs2LHafX9/f/HEE0/o3Gvo0KE6n3t137+uXbuKMWPG6LQNHjxY9OvXT6ffa6+9pt3/5ZdfBADxn//8R9v2+eefC1tb2+o/kL/c7+dLiKoVovDwcDF8+PB7XpcaDitE1GDEXw9JlyRJ9rnVtd/Zf9SoUdiwYQPy8/MBAJ988gmio6NrvNLjzuv37dsXDg4OcHBwQKtWrWRfY8yYMZg4cSKefPJJ2NjYoEuXLoiJiQEAWFpa1iguU9SmTRudfS8vL2RnZ+Pw4cMoKCiAm5ub9vN3cHBARkYGzp49C6CyEjh9+nS0bNkSzs7OcHBwwIkTJ+5aIbqXgIAAODo6VonjtjFjxmDXrl24fPkygMoqxciRI2X9HD+I2rZtix49eiA0NBSDBw/GqlWrkJOTAwAYOXIkzpw5g/379wOo/P9tyJAhOhVfOzs7NGnSRLvv4eGBgIAAODg46LTd+RkDQHh4eJX99PT0e8aanp6Obt266bR169atSr87f9Y8PDwAAKGhoTptJSUlyMvLA1B7P19paWno0aOHXn2o/jAhogYTHBwMSZLu+Y9cs2bNAOCu55w4cQLBwcHa/ZiYGEiShC+++AJnzpzBTz/9VKPhMgCoqKjA6dOnERgYCAD4+OOPkZaWhrS0NGzfvl32dSRJwoIFC1BQUIALFy5ArVbj4YcfBlD5y5cqWVtb6+xLkgSNRgONRgMvLy/tZ397O3nyJKZNmwYAmDZtGjZu3Ih58+bhxx9/RFpaGkJDQ1FWVlZrcdzWvn17tG3bFp9++il+++03HD16FCNHjtT/C35AWFpaIiUlBTt27EDLli2xbNkyNG/eHBkZGXB3d0dUVBSSkpKQnZ2N7du3IzY2Vqd/dZ/n/T7ju5GTdP7znH/+0fTPmG4fq67tdky19fOlVCr1Op/ql1VDB0Dmy9XVFX369MH777+PSZMmVZlHdPPmTfTu3Ruurq5YtGgRunbtqnN869atOH36NN566y1tm6OjIwYPHoykpCScO3cOQUFBiIyMrFF8ycnJyMnJwTPPPAMA8PHxqdF1brO0tNRe4/PPP0d4eDhXoMjQoUMHqNVqWFlZ3TWB/PHHHzFy5Eg8/fTTACrnfJw/f/6e17W2ttY+tkFfo0ePxpIlS3D58mX07NkTfn5+NbrOg0KSJHTr1g3dunXDG2+8AX9/f2zevBlTpkzB6NGjERMTA19fXzRp0qRKhaambled7twPCQnR7lf3/WvRogV++uknPP/889q2ffv2oUWLFgbFIufny8bG5r4/T23atEFqaqrJzjd70DEhogb1wQcfoGvXrnj44YcxZ84ctGnTBuXl5UhJScGKFSuQnp6OlStXapfAT5w4ESqVCqmpqZg2bRoGDRqEIUOG6Fxz1KhRePTRR3H8+HFMnTpV1l+VRUVFUKvVKC8vx+XLl7Fp0yYsWbIEL730Erp3737PvmfOnEFBQQHUajWKi4uRlpYGAGjZsiVsbGxw/fp1bNiwAZGRkSgpKUFSUhK+/PJLncnfdHc9e/ZEeHg4Bg4ciAULFqB58+a4cuUKtm/fjoEDB6JTp05o2rQpNm3ahKioKEiShNdff/2+FYeAgACkpqaiW7duUCgUcHFxkR3T8OHDMXXqVKxatQqffvqpoV+iUTtw4ABSU1PRu3dvuLu748CBA7h27Zo2yejTpw+cnJwwd+5czJkzp9bu+/PPP2PhwoUYOHAgUlJS8OWXX+Kbb77RHq/u+zdt2jQMGTJEO+n+66+/xqZNm7B7926DYpHz8xUQEIAffvgBMTExUCgUaNSoUZXrzJ49Gz169ECTJk0QExOD8vJy7NixA9OnTzcoPqolDTqDiUgIceXKFTFhwgTh7+8vbGxshI+Pj3jqqafEnj17tOf88MMP4oknnhBOTk7CxsZGtGzZUrz77rs6S3Xv1Lx5c2FhYSEyMzOrHKtuUjX+WiprY2MjvLy8xJNPPnnPyd7/vB7+seQWgMjIyBBCCHHt2jXRpUsXYW9vL+zs7ESPHj3E/v37ZX8+5qC6ie0DBgwQI0aMEEIIkZeXJ15++WXh7e0trK2thZ+fnxg+fLi4ePGiEEKIjIwM0b17d6FUKoWfn59Yvnx5lWv+c9Lr1q1bRdOmTYWVlVWVZfd3WrJkic6y7tuee+65apfgm5rjx4+LPn36iMaNGwuFQiGaNWumM5lYCCFef/11YWlpKa5cuaLTXt2y8+o+4xEjRogBAwZo9/39/cWbb74phgwZIuzs7ISHh4dITEzU6VPd908Iecvu75yMnZGRIQCII0eOaNv27Nmjs5hDzs/XL7/8Itq0aSMUCsU9l91v3LhRtGvXTtjY2IhGjRpV+0gRahiSEHeZrUpERHfVq1cvtGjRAkuXLm3oUBrcmDFjcPXqVZ3HERgiICAAcXFxdf46F6I7cciMiEgPN27cwK5du/Ddd9+Z/dPGc3NzcfDgQXz22Wf46quvGjocIoMwISIi0kOHDh2Qk5Ojnc9kzgYMGIBff/0VY8eORa9evRo6HCKDcMiMiIiIzB6fQ0RERERmjwkRERERmT0mRERERGT2mBARERGR2WNCRERERGaPCRERGYX4+Hi0a9dOuz9y5EgMHDiw3uM4f/48JEnSvoKlOgEBAUhMTJR9zdWrV8PZ2dng2CRJwpYtWwy+DhFVxYSIiO5q5MiRkCRJ+4byoKAgTJ06FYWFhXV+7/feew+rV6+Wda6cJIaI6F74YEYiuqcnnngCSUlJuHXrFn788UeMHj0ahYWFWLFiRZVzb926BWtr61q5r5OTU61ch4hIDlaIiOieFAoFPD094efnh2HDhmH48OHaYZvbw1yffPIJgoKCoFAoIIRAbm4uXnzxRbi7u0OlUuHxxx/H77//rnPdt99+Gx4eHnB0dMSoUaNQUlKic/yfQ2YajQYLFixA06ZNoVAo8NBDD2HevHkAgMDAQABA+/btIUkSIiMjtf2SkpLQokUL2NraIiQkBB988IHOfX799Ve0b98etra26NSpE44cOaL3Z7R48WKEhobC3t4efn5+GD9+PAoKCqqct2XLFjRr1gy2trbo1asXMjMzdY5//fXX6NixI2xtbREUFIQ333wT5eXlesdDRPpjQkREelEqlbh165Z2/8yZM1i/fj02btyoHbLq378/1Go1tm/fjsOHD6NDhw7o0aMHbty4AQBYv349Zs+ejXnz5uHQoUPw8vKqkqj808yZM7FgwQK8/vrrOH78ONauXQsPDw8AlUkNAOzevRtZWVnYtGkTAGDVqlWYNWsW5s2bh/T0dCQkJOD1119HcnIyAKCwsBBPPvkkmjdvjsOHDyM+Ph5Tp07V+zOxsLDA0qVL8ccffyA5ORnfffcdpk+frnNOUVER5s2bh+TkZPz888/Iy8tDTEyM9vi3336Lf/3rX5g0aRKOHz+OlStXYvXq1dqkj4jqmCAiuosRI0aIAQMGaPcPHDgg3NzcxJAhQ4QQQsyePVtYW1uL7Oxs7TmpqalCpVKJkpISnWs1adJErFy5UgghRHh4uBg3bpzO8bCwMNG2bdtq752XlycUCoVYtWpVtXFmZGQIAOLIkSM67X5+fmLt2rU6bW+99ZYIDw8XQgixcuVK4erqKgoLC7XHV6xYUe217uTv7y+WLFly1+Pr168Xbm5u2v2kpCQBQOzfv1/blp6eLgCIAwcOCCGEePTRR0VCQoLOddasWSO8vLy0+wDE5s2b73pfIqo5ziEionvatm0bHBwcUF5ejlu3bmHAgAFYtmyZ9ri/vz8aN26s3T98+DAKCgrg5uamc53i4mKcPXsWAJCeno5x48bpHA8PD8eePXuqjSE9PR2lpaXo0aOH7LivXbuGzMxMjBo1CmPGjNG2l5eXa+cnpaeno23btrCzs9OJQ1979uxBQkICjh8/jry8PJSXl6OkpASFhYWwt7cHAFhZWaFTp07aPiEhIXB2dkZ6ejoefvhhHD58GAcPHtSpCFVUVKCkpARFRUU6MRJR7WNCRET31L17d6xYsQLW1tbw9vauMmn69i/82zQaDby8vPD9999XuVZNl54rlUq9+2g0GgCVw2ZhYWE6xywtLQEAohbebX3hwgX069cP48aNw1tvvQVXV1f89NNPGDVqlM7QIlC5bP6fbrdpNBq8+eabiI6OrnKOra2twXES0b0xISKie7K3t0fTpk1ln9+hQweo1WpYWVkhICCg2nNatGiB/fv34/nnn9e27d+//67XDA4OhlKpRGpqKkaPHl3luI2NDYDKisptHh4e8PHxwblz5zB8+PBqr9uyZUusWbMGxcXF2qTrXnFU59ChQygvL8eiRYtgYVE5LXP9+vVVzisvL8ehQ4fw8MMPAwBOnjyJmzdvIiQkBEDl53by5Em9Pmsiqj1MiIioVvXs2RPh4eEYOHAgFixYgObNm+PKlSvYvn07Bg4ciE6dOmHy5MkYMWIEOnXqhEceeQSfffYZjh07hqCgoGqvaWtrixkzZmD69OmwsbFBt27dcO3aNRw7dgyjRo2Cu7s7lEoldu7cCV9fX9ja2sLJyQnx8fGYNGkSVCoV+vbti9LSUhw6dAg5OTmYMmUKhg0bhlmzZmHUqFF47bXXcP78ebz77rt6fb1NmjRBeXk5li1bhqioKPz888/48MMPq5xnbW2Nl19+GUuXLoW1tTUmTpyILl26aBOkN954A08++ST8/PwwePBgWFhY4H//+x+OHj2KuXPn6v+NICK9cJUZEdUqSZKwfft2PPbYY4iNjUWzZs0QExOD8+fPa1eFDR06FG+88QZmzJiBjh074sKFC3jppZfued3XX38dr776Kt544w20aNECQ4cORXZ2NoDK+TlLly7FypUr4e3tjQEDBgAARo8ejY8//hirV69GaGgoIiIisHr1au0yfQcHB3z99dc4fvw42rdvj1mzZmHBggV6fb3t2rXD4sWLsWDBArRu3RqfffYZ5s+fX+U8Ozs7zJgxA8OGDUN4eDiUSiXWrVunPd6nTx9s27YNKSkp6Ny5M7p06YLFixfD399fr3iIqGYkURuD6EREREQPMFaIiIiIyOwxISIiIiKzx4SIiIiIzB4TIiIiIjJ7TIiIiIjI7DEhIiIiIrPHhIiIiIjMHhMiIiIiMntMiIiIiMjsMSEiIiIis8eEiIiIiMze/wNiG/HurYrS/AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>▁▁▁▁▁▁▁▁▂▂▂▂▂▂▂▄▄▄▄▄▄▄▄▄▅▅▅▅▅▅▅▇▇▇▇▇▇▇▇█</td></tr><tr><td>test_accuracy</td><td>▁</td></tr><tr><td>test_loss</td><td>▁</td></tr><tr><td>train_accuracy</td><td>▁▂▃▇█</td></tr><tr><td>train_loss</td><td>▆▆▄▆█▆▆▇▆▇▆▆▇▆▆▅▅▅▆▆▆▆▅▇█▆▃▄▁▅▆▆▆▄▅▄▆▇██</td></tr><tr><td>trainer/global_step</td><td>▁▁▁▂▂▂▂▂▂▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇████</td></tr><tr><td>val_accuracy</td><td>▁▄█▅█</td></tr><tr><td>val_loss</td><td>█▁▃▁▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>epoch</td><td>5</td></tr><tr><td>test_accuracy</td><td>0.41527</td></tr><tr><td>test_loss</td><td>1.0804</td></tr><tr><td>train_accuracy</td><td>0.40435</td></tr><tr><td>train_loss</td><td>1.17192</td></tr><tr><td>trainer/global_step</td><td>2100</td></tr><tr><td>val_accuracy</td><td>0.38902</td></tr><tr><td>val_loss</td><td>1.09338</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">crimson-sweep-1</strong> at: <a href='https://wandb.ai/dl-miniproject/cough-classifier/runs/a12qpgp6' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/runs/a12qpgp6</a><br/>Synced 6 W&B file(s), 0 media file(s), 1 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20230331_231134-a12qpgp6/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Agent Starting Run: vgouh8bq with config:\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \taugment_masking_val: min\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tbatch_size: 32\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tdim: 64\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \theads: 4\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tmax_epochs: 3\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_fft: 512\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_freq_masks: 2\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_mels: 64\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tn_time_masks: 1\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tsample_rate: 16000\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \tspectrogram_type: mel-spectrogram\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.14.0"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/ubuntu/git_repos/Deep-Learning-Mini-Project/wandb/run-20230331_231817-vgouh8bq</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/dl-miniproject/cough-classifier/runs/vgouh8bq' target=\"_blank\">generous-sweep-2</a></strong> to <a href='https://wandb.ai/dl-miniproject/cough-classifier' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>Sweep page: <a href='https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/dl-miniproject/cough-classifier' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View sweep at <a href='https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/sweeps/wou8ulsv</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/dl-miniproject/cough-classifier/runs/vgouh8bq' target=\"_blank\">https://wandb.ai/dl-miniproject/cough-classifier/runs/vgouh8bq</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/miniconda3/envs/deep_learning/lib/python3.9/site-packages/pytorch_lightning/loggers/wandb.py:395: UserWarning: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "Size of random sample of data: torch.Size([1, 64, 626])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3070 Ti') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type | Params\n",
      "-------------------------------\n",
      "0 | model | KWT  | 343 K \n",
      "-------------------------------\n",
      "343 K     Trainable params\n",
      "0         Non-trainable params\n",
      "343 K     Total params\n",
      "1.374     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 105/105 [01:08<00:00,  1.54it/s, v_num=h8bq, val_accuracy=0.353, train_accuracy=0.372]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=3` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|██████████| 105/105 [01:08<00:00,  1.54it/s, v_num=h8bq, val_accuracy=0.353, train_accuracy=0.372]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA GeForce RTX 3070 Ti') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    }
   ],
   "source": [
    "sweep_config = {\n",
    "  \"method\": \"bayes\",\n",
    "  \"metric\": {\n",
    "        \"name\": \"val_accuracy\",\n",
    "        \"goal\": \"maximize\"\n",
    "  },\n",
    "  \"parameters\": {\n",
    "    \"batch_size\": {\n",
    "        \"values\": [8, 16, 32]\n",
    "    },\n",
    "    \"max_epochs\": {\n",
    "        \"values\": [3, 5, 7]\n",
    "    },\n",
    "    \"sample_rate\": {\n",
    "      \"values\": [16000] # Increasing this value will have a significant impact how much data you can fit into memory 48000 aprox double the size of 22050\n",
    "    },\n",
    "    \"spectrogram_type\": {\n",
    "      \"values\": ['mel-spectrogram']\n",
    "    },\n",
    "    \"n_fft\": {\n",
    "      \"values\": [512]\n",
    "    },\n",
    "    \"n_mels\": {\n",
    "      \"values\": [32, 64]\n",
    "    },\n",
    "    \"augment_masking_val\": {\n",
    "      \"values\": ['min', 'mean']\n",
    "    },\n",
    "    \"n_freq_masks\": {\n",
    "      \"values\": [1, 2]\n",
    "    },\n",
    "    \"n_time_masks\": {\n",
    "      \"values\": [1]\n",
    "    },\n",
    "    \"dim\": {\n",
    "      \"values\": [16, 32, 64]\n",
    "    },\n",
    "    \"heads\": {\n",
    "      \"values\": [4, 8]\n",
    "    },\n",
    "  }\n",
    "}\n",
    "\n",
    "sweep_id = wandb.sweep(sweep_config, project='cough-classifier', entity='dl-miniproject')\n",
    "\n",
    "wandb.agent(sweep_id, function=train_new, count=100)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f0b69965270e405d09dae21e4ae2ea2cc233b709569664f2de57e4e4d7e55573"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}