Table_05_06_Evaluation_Module_Comparison_FEWNet_Baselines.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WfYYZnMwfwRZ",
        "outputId": "73af020c-7228-4dbc-d0da-3cffeab2e6d1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ],
      "source": [
        "!pwd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Declaring the working library\n",
        "import os\n",
        "os.chdir('/content')"
      ],
      "metadata": {
        "id": "rHFmTBkuzCq6"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Installing the required library\n",
        "# !pip install -U scikit-learn"
      ],
      "metadata": {
        "id": "B3AZatjdzD0y"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn\n",
        "print(sklearn.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vHVhW57lzFI9",
        "outputId": "e4fe5889-d24e-4f34-efcd-60ac12aa527a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__Evaluation Module__"
      ],
      "metadata": {
        "id": "mi1E41mazMvx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn import metrics"
      ],
      "metadata": {
        "id": "iCyRQ5nnzK9V"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import numpy as np\n",
        "from sklearn.metrics import mean_squared_log_error, r2_score\n",
        "\n",
        "EPSILON = 1e-10\n",
        "\n",
        "\n",
        "def _error(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Simple error \"\"\"\n",
        "    return actual - predicted\n",
        "\n",
        "\n",
        "def _percentage_error(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Percentage error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return _error(actual, predicted) / (actual + EPSILON)\n",
        "\n",
        "\n",
        "def _naive_forecasting(actual: np.ndarray, seasonality: int = 1):\n",
        "    \"\"\" Naive forecasting method which just repeats previous samples \"\"\"\n",
        "    return actual[:-seasonality]\n",
        "\n",
        "\n",
        "def _relative_error(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Relative Error \"\"\"\n",
        "    if benchmark is None or isinstance(benchmark, int):\n",
        "        # If no benchmark prediction provided - use naive forecasting\n",
        "        if not isinstance(benchmark, int):\n",
        "            seasonality = 1\n",
        "        else:\n",
        "            seasonality = benchmark\n",
        "        return _error(actual[seasonality:], predicted[seasonality:]) /\\\n",
        "               (_error(actual[seasonality:], _naive_forecasting(actual, seasonality)) + EPSILON)\n",
        "\n",
        "    return _error(actual, predicted) / (_error(actual, benchmark) + EPSILON)\n",
        "\n",
        "\n",
        "def _bounded_relative_error(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Bounded Relative Error \"\"\"\n",
        "    if benchmark is None or isinstance(benchmark, int):\n",
        "        # If no benchmark prediction provided - use naive forecasting\n",
        "        if not isinstance(benchmark, int):\n",
        "            seasonality = 1\n",
        "        else:\n",
        "            seasonality = benchmark\n",
        "\n",
        "        abs_err = np.abs(_error(actual[seasonality:], predicted[seasonality:]))\n",
        "        abs_err_bench = np.abs(_error(actual[seasonality:], _naive_forecasting(actual, seasonality)))\n",
        "    else:\n",
        "        abs_err = np.abs(_error(actual, predicted))\n",
        "        abs_err_bench = np.abs(_error(actual, benchmark))\n",
        "\n",
        "    return abs_err / (abs_err + abs_err_bench + EPSILON)\n",
        "\n",
        "\n",
        "def _geometric_mean(a, axis=0, dtype=None):\n",
        "    \"\"\" Geometric mean \"\"\"\n",
        "    if not isinstance(a, np.ndarray):  # if not an ndarray object attempt to convert it\n",
        "        log_a = np.log(np.array(a, dtype=dtype))\n",
        "    elif dtype:  # Must change the default dtype allowing array type\n",
        "        if isinstance(a, np.ma.MaskedArray):\n",
        "            log_a = np.log(np.ma.asarray(a, dtype=dtype))\n",
        "        else:\n",
        "            log_a = np.log(np.asarray(a, dtype=dtype))\n",
        "    else:\n",
        "        log_a = np.log(a)\n",
        "    return np.exp(log_a.mean(axis=axis))\n",
        "\n",
        "\n",
        "def mse(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Mean Squared Error \"\"\"\n",
        "    return np.mean(np.square(_error(actual, predicted)))\n",
        "\n",
        "\n",
        "def rmse(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Root Mean Squared Error \"\"\"\n",
        "    return np.sqrt(mse(actual, predicted))\n",
        "\n",
        "\n",
        "def nrmse(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Normalized Root Mean Squared Error \"\"\"\n",
        "    return rmse(actual, predicted) / (actual.max() - actual.min())\n",
        "\n",
        "\n",
        "def me(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Mean Error \"\"\"\n",
        "    return np.mean(_error(actual, predicted))\n",
        "\n",
        "def r2_scr(actual:np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"Calculating the r2 value: perasonian correlation coefficient\"\"\"\n",
        "    return metrics.r2_score(actual, predicted)\n",
        "\n",
        "\n",
        "\n",
        "def mae(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Mean Absolute Error \"\"\"\n",
        "    return np.mean(np.abs(_error(actual, predicted)))\n",
        "\n",
        "\n",
        "mad = mae  # Mean Absolute Deviation (it is the same as MAE)\n",
        "\n",
        "\n",
        "def gmae(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Geometric Mean Absolute Error \"\"\"\n",
        "    return _geometric_mean(np.abs(_error(actual, predicted)))\n",
        "\n",
        "\n",
        "def mdae(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Median Absolute Error \"\"\"\n",
        "    return np.median(np.abs(_error(actual, predicted)))\n",
        "\n",
        "\n",
        "def mpe(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Mean Percentage Error \"\"\"\n",
        "    return np.mean(_percentage_error(actual, predicted))\n",
        "\n",
        "\n",
        "def mape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Mean Absolute Percentage Error\n",
        "    Properties:\n",
        "        + Easy to interpret\n",
        "        + Scale independent\n",
        "        - Biased, not symmetric\n",
        "        - Undefined when actual[t] == 0\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.mean(np.abs(_percentage_error(actual, predicted)))\n",
        "\n",
        "\n",
        "def mdape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Median Absolute Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.median(np.abs(_percentage_error(actual, predicted)))\n",
        "\n",
        "\n",
        "def smape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Symmetric Mean Absolute Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.mean(2.0 * np.abs(actual - predicted) / ((np.abs(actual) + np.abs(predicted)) + EPSILON))\n",
        "\n",
        "\n",
        "def smdape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Symmetric Median Absolute Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.median(2.0 * np.abs(actual - predicted) / ((np.abs(actual) + np.abs(predicted)) + EPSILON))\n",
        "\n",
        "\n",
        "def maape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Mean Arctangent Absolute Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.mean(np.arctan(np.abs((actual - predicted) / (actual + EPSILON))))\n",
        "\n",
        "\n",
        "def mase(actual: np.ndarray, predicted: np.ndarray, seasonality: int = 1):\n",
        "    \"\"\"\n",
        "    Mean Absolute Scaled Error\n",
        "    Baseline (benchmark) is computed with naive forecasting (shifted by @seasonality)\n",
        "    \"\"\"\n",
        "    return mae(actual, predicted) / mae(actual[seasonality:], _naive_forecasting(actual, seasonality))\n",
        "\n",
        "\n",
        "def std_ae(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Normalized Absolute Error \"\"\"\n",
        "    __mae = mae(actual, predicted)\n",
        "    return np.sqrt(np.sum(np.square(_error(actual, predicted) - __mae))/(len(actual) - 1))\n",
        "\n",
        "\n",
        "def std_ape(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Normalized Absolute Percentage Error \"\"\"\n",
        "    __mape = mape(actual, predicted)\n",
        "    return np.sqrt(np.sum(np.square(_percentage_error(actual, predicted) - __mape))/(len(actual) - 1))\n",
        "\n",
        "\n",
        "def rmspe(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Root Mean Squared Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.sqrt(np.mean(np.square(_percentage_error(actual, predicted))))\n",
        "\n",
        "\n",
        "def rmdspe(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\"\n",
        "    Root Median Squared Percentage Error\n",
        "    Note: result is NOT multiplied by 100\n",
        "    \"\"\"\n",
        "    return np.sqrt(np.median(np.square(_percentage_error(actual, predicted))))\n",
        "\n",
        "\n",
        "def rmsse(actual: np.ndarray, predicted: np.ndarray, seasonality: int = 1):\n",
        "    \"\"\" Root Mean Squared Scaled Error \"\"\"\n",
        "    q = np.abs(_error(actual, predicted)) / mae(actual[seasonality:], _naive_forecasting(actual, seasonality))\n",
        "    return np.sqrt(np.mean(np.square(q)))\n",
        "\n",
        "\n",
        "def inrse(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Integral Normalized Root Squared Error \"\"\"\n",
        "    return np.sqrt(np.sum(np.square(_error(actual, predicted))) / np.sum(np.square(actual - np.mean(actual))))\n",
        "\n",
        "\n",
        "def rrse(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Root Relative Squared Error \"\"\"\n",
        "    return np.sqrt(np.sum(np.square(actual - predicted)) / np.sum(np.square(actual - np.mean(actual))))\n",
        "\n",
        "\n",
        "def mre(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Mean Relative Error \"\"\"\n",
        "    return np.mean(_relative_error(actual, predicted, benchmark))\n",
        "\n",
        "\n",
        "def rae(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Relative Absolute Error (aka Approximation Error) \"\"\"\n",
        "    return np.sum(np.abs(actual - predicted)) / (np.sum(np.abs(actual - np.mean(actual))) + EPSILON)\n",
        "\n",
        "\n",
        "def mrae(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Mean Relative Absolute Error \"\"\"\n",
        "    return np.mean(np.abs(_relative_error(actual, predicted, benchmark)))\n",
        "\n",
        "\n",
        "def mdrae(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Median Relative Absolute Error \"\"\"\n",
        "    return np.median(np.abs(_relative_error(actual, predicted, benchmark)))\n",
        "\n",
        "\n",
        "def gmrae(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Geometric Mean Relative Absolute Error \"\"\"\n",
        "    return _geometric_mean(np.abs(_relative_error(actual, predicted, benchmark)))\n",
        "\n",
        "\n",
        "def mbrae(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Mean Bounded Relative Absolute Error \"\"\"\n",
        "    return np.mean(_bounded_relative_error(actual, predicted, benchmark))\n",
        "\n",
        "\n",
        "def umbrae(actual: np.ndarray, predicted: np.ndarray, benchmark: np.ndarray = None):\n",
        "    \"\"\" Unscaled Mean Bounded Relative Absolute Error \"\"\"\n",
        "    __mbrae = mbrae(actual, predicted, benchmark)\n",
        "    return __mbrae / (1 - __mbrae)\n",
        "\n",
        "\n",
        "def mda(actual: np.ndarray, predicted: np.ndarray):\n",
        "    \"\"\" Mean Directional Accuracy \"\"\"\n",
        "    return np.mean((np.sign(actual[1:] - actual[:-1]) == np.sign(predicted[1:] - predicted[:-1])).astype(int))\n",
        "\n",
        "# https://www.kaggle.com/carlolepelaars/understanding-the-metric-rmsle\n",
        "def rmsle(y_true:np.ndarray, y_pred:np.ndarray):\n",
        "    \"\"\"\n",
        "        The Root Mean Squared Log Error (RMSLE) metric\n",
        "\n",
        "        :param y_true: The ground truth labels given in the dataset/actual values\n",
        "        :param y_pred: Our predictions/forecasts\n",
        "        :return: The RMSLE score --> Evaluation metric\n",
        "    \"\"\"\n",
        "    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n",
        "\n",
        "\n",
        "METRICS = {\n",
        "    'mse': mse,\n",
        "    'rmse': rmse,\n",
        "    'nrmse': nrmse,\n",
        "    'me': me,\n",
        "    'mae': mae,\n",
        "    'mad': mad,\n",
        "    'gmae': gmae,\n",
        "    'mdae': mdae,\n",
        "    'mpe': mpe,\n",
        "    'mape': mape,\n",
        "    'mdape': mdape,\n",
        "    'smape': smape,\n",
        "    'smdape': smdape,\n",
        "    'maape': maape,\n",
        "    'mase': mase,\n",
        "    'std_ae': std_ae,\n",
        "    'std_ape': std_ape,\n",
        "    'rmspe': rmspe,\n",
        "    'rmdspe': rmdspe,\n",
        "    'rmsse': rmsse,\n",
        "    'inrse': inrse,\n",
        "    'rrse': rrse,\n",
        "    'mre': mre,\n",
        "    'rae': rae,\n",
        "    'r2_scr':r2_scr,\n",
        "    'mrae': mrae,\n",
        "    'mdrae': mdrae,\n",
        "    'gmrae': gmrae,\n",
        "    'mbrae': mbrae,\n",
        "    'umbrae': umbrae,\n",
        "    'mda': mda,\n",
        "    'rmsle': rmsle\n",
        "}\n",
        "\n",
        "\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('mae', 'mse', 'smape', 'umbrae')):\n",
        "    results = {}\n",
        "    for name in metrics:\n",
        "        try:\n",
        "            results[name] = METRICS[name](actual, predicted)\n",
        "        except Exception as err:\n",
        "            results[name] = np.nan\n",
        "            print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        "    return results\n",
        "\n",
        "\n",
        "def evaluate_all(actual: np.ndarray, predicted: np.ndarray):\n",
        "    return evaluate(actual, predicted, metrics=set(METRICS.keys()))\n",
        "\n",
        "# Module for Theil's U1\n",
        "def theils_u1(f:np.ndarray , y:np.ndarray):\n",
        "    df = pd.DataFrame({'f_i':f, 'y_i': y})\n",
        "    df['(f_i - y_i)^2'] = np.square(df['f_i'] - df['y_i'])\n",
        "    df['y_i^2'] = np.square(df['y_i'])\n",
        "    df['f_i^2'] = np.square(df['f_i'])\n",
        "    return (np.sqrt(np.mean(df['(f_i - y_i)^2'])))/(np.sqrt(np.mean(df['y_i^2']))+np.sqrt(np.mean(df['f_i^2'])))\n",
        "\n",
        "# Module for Theil's U2\n",
        "def theils_u2(f:int , y:int)->float:\n",
        "    y = y.values.flatten()\n",
        "    f = f.values.flatten()\n",
        "    df = pd.DataFrame({'f_i+1':f, 'y_i+1': y})\n",
        "    df['y_i'] = df['y_i+1'].shift(periods=1)\n",
        "    df['numerator'] = np.square((df['f_i+1'] - df['y_i+1']) / df['y_i'])\n",
        "    df['denominator'] = np.square((df['y_i+1'] - df['y_i']) / df['y_i'])\n",
        "    df.dropna(inplace=True)\n",
        "    return np.sqrt(np.sum(df['numerator'])/np.sum(df['denominator']))"
      ],
      "metadata": {
        "id": "xubpkJF9zO9U"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For the forecast values across different algorithms , please refer to 'Table_05_06_Forecasts_FEWNet_Baselines_12M_24M_BRIC_Comparison.xlsx' at \"/FEWNet/dataset\""
      ],
      "metadata": {
        "id": "OwLm8WUhzWFd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "__Brazil: Forecasts Evaluation:  FEWNet and baselines:12M (table 05)__"
      ],
      "metadata": {
        "id": "of9o3ZvWzdww"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([4.517457  ,4.559198  ,5.195379  ,6.099479  ,6.759116  ,8.056819  ,\n",
        "                   8.347072  ,8.994823  ,9.679774 ,10.246209 ,10.672622 ,10.738501])\n",
        "fewnet = np.array([4.51,\n",
        "4.97,\n",
        "5.53,\n",
        "5.86,\n",
        "6.29,\n",
        "7.02,\n",
        "7.62,\n",
        "8.06,\n",
        "8.33,\n",
        "8.40,\n",
        "8.30,\n",
        "8.21])\n",
        "\n",
        "warimax = np.array([5.384066,4.751891,4.339061,4.378773,4.524725,4.851346,5.137784,5.464164,5.688167,5.934548,6.102688,6.309670])\n",
        "rwd = np.array([4.294411 ,4.277599 ,4.260786 ,4.243974 ,4.227162 ,4.210349 ,4.193537 ,4.176725 ,4.159912 ,4.143100 ,4.126287 ,4.109475])\n",
        "ar = np.array([4.458540 ,\n",
        "4.513750 ,\n",
        "4.548190 ,\n",
        "4.582458 ,\n",
        "4.618596 ,\n",
        "4.654877 ,\n",
        "4.690014 ,\n",
        "4.723529 ,\n",
        "4.755369 ,\n",
        "4.785614 ,\n",
        "4.814358 ,\n",
        "4.841688])\n",
        "msgarch = np.array([4.24,\n",
        "4.68,\n",
        "4.77,\n",
        "4.87,\n",
        "4.96,\n",
        "5.31,\n",
        "4.79,\n",
        "5.52,\n",
        "4.61,\n",
        "5.26,\n",
        "5.13,\n",
        "4.72])\n",
        "rw = np.array([4.315522,\n",
        "4.319820,\n",
        "4.324119,\n",
        "4.328418,\n",
        "4.332716,\n",
        "4.337015,\n",
        "4.341313,\n",
        "4.345612,\n",
        "4.349910,\n",
        "4.354209,\n",
        "4.358507,\n",
        "4.362806])\n",
        "\n",
        "deepar = np.array([9.94,\n",
        "10.03,\n",
        "10.05,\n",
        "10.09,\n",
        "9.84,\n",
        "9.28,\n",
        "8.84,\n",
        "8.44,\n",
        "8.03,\n",
        "7.85,\n",
        "7.66,\n",
        "7.35])\n",
        "\n",
        "arnnx = np.array([9.02,\n",
        "7.78,\n",
        "6.53,\n",
        "6.01,\n",
        "6.32,\n",
        "6.83,\n",
        "7.71,\n",
        "7.41,\n",
        "7.34,\n",
        "7.48,\n",
        "8.24,\n",
        "8.80])\n",
        "\n",
        "nbeats = np.array([4.11,\n",
        "4.34,\n",
        "4.46,\n",
        "5.07,\n",
        "5.05,\n",
        "5.28,\n",
        "5.52,\n",
        "5.97,\n",
        "6.11,\n",
        "6.16,\n",
        "6.09,\n",
        "5.77])\n",
        "arfimax = np.array([6.81,\n",
        "5.18,\n",
        "3.95,\n",
        "3.24,\n",
        "3.04,\n",
        "3.19,\n",
        "3.45,\n",
        "3.72,\n",
        "3.94,\n",
        "4.09,\n",
        "4.15,\n",
        "4.14])\n",
        "sarimax = np.array([7.81,\n",
        "6.35,\n",
        "5.59,\n",
        "5.51,\n",
        "5.78,\n",
        "5.93,\n",
        "6.65,\n",
        "6.90,\n",
        "7.05,\n",
        "7.25,\n",
        "7.23,\n",
        "7.21])\n",
        "arimax = np.array([7.87,\n",
        "6.29,\n",
        "5.43,\n",
        "5.21,\n",
        "5.40,\n",
        "5.76,\n",
        "6.13,\n",
        "6.43,\n",
        "6.64,\n",
        "6.73,\n",
        "6.74,\n",
        "6.73])\n",
        "lr = \tnp.array([6.56,\n",
        "8.39,\n",
        "7.55,\n",
        "7.01,\n",
        "6.93,\n",
        "7.05,\n",
        "7.20,\n",
        "7.43,\n",
        "7.62,\n",
        "7.74,\n",
        "7.80,\n",
        "7.83])\n",
        "ardl = np.array([3.39,\n",
        "2.42,\n",
        "2.46,\n",
        "3.14,\n",
        "3.86,\n",
        "4.54,\n",
        "5.21,\n",
        "5.74,\n",
        "6.09,\n",
        "6.31,\n",
        "6.44,\n",
        "6.49])\n",
        "xgboost = np.array([4.73,\n",
        "5.11,\n",
        "5.04,\n",
        "4.82,\n",
        "4.88,\n",
        "5.85,\n",
        "6.35,\n",
        "6.75,\n",
        "6.66,\n",
        "6.57,\n",
        "6.81,\n",
        "7.25])\n",
        "tft = \tnp.array([3.20,\n",
        "4.52,\n",
        "2.34,\n",
        "2.03,\n",
        "3.15,\n",
        "2.36,\n",
        "4.00,\n",
        "3.00,\n",
        "1.90,\n",
        "3.83,\n",
        "2.94,\n",
        "2.82])"
      ],
      "metadata": {
        "id": "PUVnjiqYzTcf"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I30upobX3daR",
        "outputId": "fe299d94-821c-457e-db18-c83ce03e0bed"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 4.133401612636921, 'mase': 6.158683628235606, 'smape': 0.5199990480502511, 'mdrae': 7.17746634012492, 'mdape': 0.47079812857046627}\n",
            "Measuring Theils U1: rw -  0.33131115510414716\n",
            "Evaluation_metrics:  rwd -  {'rmse': 4.287897288243019, 'mase': 6.4013164028417116, 'smape': 0.5480375831336378, 'mdrae': 7.438194613757731, 'mdape': 0.4875108895528768}\n",
            "Measuring Theils U1: rwd -  0.34750641919697445\n",
            "Evaluation_metrics:  ar -  {'rmse': 3.804146060002976, 'mase': 5.581513944176145, 'smape': 0.45479952860361345, 'mdrae': 6.594036905138856, 'mdape': 0.4301842331215173}\n",
            "Measuring Theils U1: ar -  0.29711099642382705\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 3.591668971783951, 'mase': 5.193777001094995, 'smape': 0.41418716439376185, 'mdrae': 5.364442508716404, 'mdape': 0.3636223080715831}\n",
            "Measuring Theils U1: msgarch -  0.27515380245084264\n",
            "Evaluation_metrics:  deepar -  {'rmse': 3.4081455946808155, 'mase': 5.236413413890016, 'smape': 0.368145517107865, 'mdrae': 4.4138048883515335, 'mdape': 0.29891132112999985}\n",
            "Measuring Theils U1: deepar -  0.19880797731154587\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 2.2333271445406853, 'mase': 3.3171842619984693, 'smape': 0.2480773263934397, 'mdrae': 2.4466546555008573, 'mdape': 0.2042249132341781}\n",
            "Measuring Theils U1: arnnx -  0.14273694063516731\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 2.9540253154877076, 'mase': 4.411115709303242, 'smape': 0.34143671573411466, 'mdrae': 5.211721713639119, 'mdape': 0.33748757233065474}\n",
            "Measuring Theils U1: nbeats -  0.2186538462710858\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 4.669228062668274, 'mase': 7.484335011401088, 'smape': 0.6808485004623676, 'mdrae': 8.37983154877067, 'mdape': 0.5865550924231296}\n",
            "Measuring Theils U1: arfimax -  0.3786652421379899\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 2.3716437972867923, 'mase': 3.7665997191682514, 'smape': 0.28517233713332685, 'mdrae': 3.839360771231905, 'mdape': 0.26782735919397144}\n",
            "Measuring Theils U1: sarimax -  0.1604457842465895\n",
            "Evaluation_metrics:  arimax -  {'rmse': 2.690032324689721, 'mase': 4.294110321354423, 'smape': 0.3295074668031606, 'mdrae': 4.43794373547335, 'mdape': 0.2995889630706192}\n",
            "Measuring Theils U1: arimax -  0.18605953995778743\n",
            "Evaluation_metrics:  lr -  {'rmse': 2.1844666235816583, 'mase': 3.4443187376052427, 'smape': 0.25885809393986353, 'mdrae': 3.701180952637509, 'mdape': 0.2286950995708941}\n",
            "Measuring Theils U1: lr -  0.14022835801638928\n",
            "Evaluation_metrics:  ardl -  {'rmse': 3.2614055374243556, 'mase': 5.5663344582249135, 'smape': 0.5150864483779664, 'mdrae': 5.240920882626507, 'mdape': 0.3961097432430172}\n",
            "Measuring Theils U1: ardl -  0.24996903636021162\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 2.400719614242554, 'mase': 3.6208355730431534, 'smape': 0.27153127191430076, 'mdrae': 4.408744566485962, 'mdape': 0.26173765995107556}\n",
            "Measuring Theils U1: xgboost -  0.17019899206325784\n",
            "Evaluation_metrics:  tft -  {'rmse': 5.407974076109794, 'mase': 8.51332106261693, 'smape': 0.849540137668634, 'mdrae': 9.254826313003797, 'mdape': 0.6463390144304579}\n",
            "Measuring Theils U1: tft -  0.48109722590042664\n",
            "Evaluation_metrics:  warimax -  {'rmse': 3.129657621780323, 'mase': 4.879940167598879, 'smape': 0.3915297027456979, 'mdrae': 5.450642298437113, 'mdape': 0.3885009605630496}\n",
            "Measuring Theils U1: warimax -  0.23327673464796397\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 1.3002806259453827, 'mase': 1.8061042085540626, 'smape': 0.1221136210018778, 'mdrae': 1.9706139560391018, 'mdape': 0.09701650481312285}\n",
            "Measuring Theils U1: fewnet -  0.08556488407153964\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__Brazil: Forecasts Evaluation: FEWNet and baselines:24M (table 06)__"
      ],
      "metadata": {
        "id": "Wklzf_3Z4ykZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([4.306152,4.191771,4.005114,3.303158,2.399279,1.877727,2.132417,2.305625,2.438465,3.135329,\n",
        "3.918350,4.311223,4.517457,4.559198,5.195379,6.099479,6.759116,8.056819,8.347072,8.994823\n",
        ",9.679774,10.246209,10.672622,10.738501])\n",
        "rw = np.array([3.28,\n",
        "3.29,\n",
        "3.29,\n",
        "3.30,\n",
        "3.31,\n",
        "3.31,\n",
        "3.32,\n",
        "3.33,\n",
        "3.34,\n",
        "3.34,\n",
        "3.35,\n",
        "3.36,\n",
        "3.36,\n",
        "3.37,\n",
        "3.38,\n",
        "3.38,\n",
        "3.39,\n",
        "3.40,\n",
        "3.40,\n",
        "3.41,\n",
        "3.42,\n",
        "3.42,\n",
        "3.43,\n",
        "3.44])\n",
        "rwd = np.array([3.25,\n",
        "3.23,\n",
        "3.20,\n",
        "3.18,\n",
        "3.16,\n",
        "3.13,\n",
        "3.11,\n",
        "3.09,\n",
        "3.06,\n",
        "3.04,\n",
        "3.02,\n",
        "2.99,\n",
        "2.97,\n",
        "2.95,\n",
        "2.92,\n",
        "2.90,\n",
        "2.88,\n",
        "2.85,\n",
        "2.83,\n",
        "2.81,\n",
        "2.79,\n",
        "2.76,\n",
        "2.74,\n",
        "2.72])\n",
        "ar = np.array([3.71,\n",
        "3.99,\n",
        "4.17,\n",
        "4.30,\n",
        "4.40,\n",
        "4.49,\n",
        "4.56,\n",
        "4.62,\n",
        "4.68,\n",
        "4.74,\n",
        "4.79,\n",
        "4.83,\n",
        "4.88,\n",
        "4.92,\n",
        "4.95,\n",
        "4.99,\n",
        "5.02,\n",
        "5.05,\n",
        "5.08,\n",
        "5.11,\n",
        "5.13,\n",
        "5.16,\n",
        "5.18,\n",
        "5.20])\n",
        "msgarch = np.array([3.50,\n",
        "3.93,\n",
        "4.31,\n",
        "4.82,\n",
        "5.08,\n",
        "5.49,\n",
        "5.08,\n",
        "5.14,\n",
        "5.02,\n",
        "4.93,\n",
        "4.69,\n",
        "4.83,\n",
        "5.07,\n",
        "5.37,\n",
        "5.83,\n",
        "5.39,\n",
        "5.06,\n",
        "5.09,\n",
        "4.91,\n",
        "5.16,\n",
        "5.52,\n",
        "5.41,\n",
        "5.59,\n",
        "5.49])\n",
        "deepar = np.array([10.03,\n",
        "10.57,\n",
        "10.71,\n",
        "10.68,\n",
        "10.70,\n",
        "10.22,\n",
        "9.84,\n",
        "9.58,\n",
        "8.96,\n",
        "8.24,\n",
        "7.79,\n",
        "7.31,\n",
        "7.13,\n",
        "6.70,\n",
        "5.91,\n",
        "5.06,\n",
        "4.26,\n",
        "3.72,\n",
        "3.28,\n",
        "2.80,\n",
        "2.35,\n",
        "2.03,\n",
        "1.63,\n",
        "1.34])\n",
        "arnnx = np.array([10.11,\n",
        "6.98,\n",
        "6.41,\n",
        "6.32,\n",
        "6.25,\n",
        "6.08,\n",
        "7.04,\n",
        "6.65,\n",
        "6.49,\n",
        "7.37,\n",
        "7.85,\n",
        "8.38,\n",
        "5.54,\n",
        "7.31,\n",
        "7.01,\n",
        "7.75,\n",
        "7.85,\n",
        "7.73,\n",
        "6.37,\n",
        "6.09,\n",
        "6.09,\n",
        "5.55,\n",
        "5.99,\n",
        "6.20])\n",
        "nbeats = np.array([2.90,\n",
        "3.53,\n",
        "3.92,\n",
        "4.04,\n",
        "4.57,\n",
        "4.80,\n",
        "4.88,\n",
        "4.50,\n",
        "4.68,\n",
        "4.99,\n",
        "5.04,\n",
        "5.27,\n",
        "5.00,\n",
        "5.35,\n",
        "4.88,\n",
        "4.50,\n",
        "4.56,\n",
        "3.98,\n",
        "4.49,\n",
        "4.44,\n",
        "4.26,\n",
        "4.60,\n",
        "4.73,\n",
        "4.70])\n",
        "arfimax = np.array([5.51,\n",
        "3.77,\n",
        "2.54,\n",
        "1.95,\n",
        "1.74,\n",
        "1.89,\n",
        "2.18,\n",
        "2.46,\n",
        "2.68,\n",
        "2.86,\n",
        "2.94,\n",
        "2.95,\n",
        "2.95,\n",
        "3.01,\n",
        "3.16,\n",
        "3.33,\n",
        "3.38,\n",
        "3.20,\n",
        "2.78,\n",
        "2.27,\n",
        "1.85,\n",
        "1.64,\n",
        "1.62,\n",
        "1.65])\n",
        "sarimax = np.array([8.24,\n",
        "8.27,\n",
        "8.47,\n",
        "8.65,\n",
        "8.37,\n",
        "7.76,\n",
        "7.39,\n",
        "6.86,\n",
        "6.43,\n",
        "6.58,\n",
        "6.73,\n",
        "6.61,\n",
        "6.44,\n",
        "6.09,\n",
        "5.78,\n",
        "5.56,\n",
        "5.40,\n",
        "5.03,\n",
        "5.26,\n",
        "5.04,\n",
        "4.80,\n",
        "4.82,\n",
        "4.74,\n",
        "4.64])\n",
        "arimax = np.array([8.38,\n",
        "8.55,\n",
        "8.73,\n",
        "8.77,\n",
        "8.53,\n",
        "8.00,\n",
        "7.36,\n",
        "6.82,\n",
        "6.54,\n",
        "6.49,\n",
        "6.51,\n",
        "6.42,\n",
        "6.16,\n",
        "5.75,\n",
        "5.33,\n",
        "4.99,\n",
        "4.73,\n",
        "4.52,\n",
        "4.33,\n",
        "4.15,\n",
        "3.99,\n",
        "3.85,\n",
        "3.74,\n",
        "3.64])\n",
        "lr = \tnp.array([5.40,\n",
        "8.06,\n",
        "7.67,\n",
        "6.97,\n",
        "6.77,\n",
        "6.91,\n",
        "7.17,\n",
        "7.39,\n",
        "7.57,\n",
        "7.64,\n",
        "7.66,\n",
        "7.64,\n",
        "7.70,\n",
        "7.76,\n",
        "7.87,\n",
        "8.04,\n",
        "8.11,\n",
        "7.79,\n",
        "7.28,\n",
        "6.69,\n",
        "6.23,\n",
        "6.00,\n",
        "6.01,\n",
        "6.02])\n",
        "ardl = np.array([3.51,\n",
        "2.77,\n",
        "2.61,\n",
        "3.39,\n",
        "4.23,\n",
        "4.82,\n",
        "5.44,\n",
        "6.05,\n",
        "6.44,\n",
        "6.64,\n",
        "6.80,\n",
        "6.95,\n",
        "7.09,\n",
        "7.30,\n",
        "7.59,\n",
        "7.78,\n",
        "7.67,\n",
        "7.24,\n",
        "6.64,\n",
        "6.15,\n",
        "5.93,\n",
        "5.96,\n",
        "6.06,\n",
        "6.02])\n",
        "xgboost = np.array([4.48,\n",
        "4.52,\n",
        "4.33,\n",
        "4.08,\n",
        "4.08,\n",
        "4.54,\n",
        "5.15,\n",
        "5.74,\n",
        "5.93,\n",
        "5.94,\n",
        "6.11,\n",
        "6.46,\n",
        "6.56,\n",
        "6.62,\n",
        "6.69,\n",
        "6.95,\n",
        "7.11,\n",
        "7.01,\n",
        "6.63,\n",
        "6.25,\n",
        "6.06,\n",
        "6.10,\n",
        "6.17,\n",
        "6.00])\n",
        "tft = \tnp.array([3.63,\n",
        "5.12,\n",
        "4.90,\n",
        "4.96,\n",
        "5.86,\n",
        "5.45,\n",
        "6.57,\n",
        "6.13,\n",
        "5.59,\n",
        "6.82,\n",
        "6.55,\n",
        "6.45,\n",
        "6.44,\n",
        "6.41,\n",
        "5.38,\n",
        "5.91,\n",
        "5.45,\n",
        "5.29,\n",
        "5.31,\n",
        "4.45,\n",
        "5.09,\n",
        "4.03,\n",
        "3.52,\n",
        "3.63])\n",
        "warimax = np.array([5.27,\n",
        "4.34,\n",
        "3.57,\n",
        "3.04,\n",
        "2.79,\n",
        "2.92,\n",
        "2.96,\n",
        "2.70,\n",
        "2.69,\n",
        "2.93,\n",
        "2.92,\n",
        "2.79,\n",
        "2.79,\n",
        "2.80,\n",
        "2.80,\n",
        "2.90,\n",
        "2.99,\n",
        "2.87,\n",
        "2.63,\n",
        "2.49,\n",
        "2.40,\n",
        "2.30,\n",
        "2.25,\n",
        "2.21])\n",
        "fewnet = np.array([4.62,\n",
        "5.45,\n",
        "5.63,\n",
        "5.26,\n",
        "5.02,\n",
        "5.08,\n",
        "5.21,\n",
        "5.20,\n",
        "5.10,\n",
        "5.31,\n",
        "5.91,\n",
        "6.04,\n",
        "6.07,\n",
        "5.98,\n",
        "6.05,\n",
        "6.33,\n",
        "6.75,\n",
        "6.90,\n",
        "6.94,\n",
        "6.97,\n",
        "6.96,\n",
        "6.87,\n",
        "6.58,\n",
        "6.55])"
      ],
      "metadata": {
        "id": "C70mq2FL4gIi"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5U6n9eoI7UwP",
        "outputId": "a3ed9a5c-7031-4974-c6ce-6274d18c68fc"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 3.555816399258117, 'mase': 5.338980888576182, 'smape': 0.4991861937985444, 'mdrae': 5.107530353041516, 'mdape': 0.4119374811813378}\n",
            "Measuring Theils U1: rw -  0.3714659866097715\n",
            "Evaluation_metrics:  rwd -  {'rmse': 3.933571489676986, 'mase': 5.891666741959873, 'smape': 0.5689430557055086, 'mdrae': 4.528514846584148, 'mdape': 0.39545922825468316}\n",
            "Measuring Theils U1: rwd -  0.42757604402674076\n",
            "Evaluation_metrics:  ar -  {'rmse': 2.7303800602784754, 'mase': 4.345859794097585, 'smape': 0.3962671416430164, 'mdrae': 2.636474302891367, 'mdape': 0.3823025564687732}\n",
            "Measuring Theils U1: ar -  0.24873636776935457\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 2.7761500228553255, 'mase': 4.635268303062659, 'smape': 0.42406017032076115, 'mdrae': 2.965796307134672, 'mdape': 0.4190532313194846}\n",
            "Measuring Theils U1: msgarch -  0.24636022404475144\n",
            "Evaluation_metrics:  deepar -  {'rmse': 6.156445023325795, 'mase': 11.451390886250358, 'smape': 0.9102433545134189, 'mdrae': 12.667857864043807, 'mdape': 0.8612440638879933}\n",
            "Measuring Theils U1: deepar -  0.450613705574104\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 3.558370489690418, 'mase': 6.6766006568166025, 'smape': 0.566584190468375, 'mdrae': 6.81154716512438, 'mdape': 0.6019029941402643}\n",
            "Measuring Theils U1: arnnx -  0.2700031293623206\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 3.114824061280662, 'mase': 5.095452553808291, 'smape': 0.47124381427658085, 'mdrae': 5.603032872197408, 'mdape': 0.4840476689768525}\n",
            "Measuring Theils U1: nbeats -  0.28920378015396025\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 4.266269113667462, 'mase': 6.04414966686299, 'smape': 0.5811114997319833, 'mdrae': 3.687421865246351, 'mdape': 0.35639427317203665}\n",
            "Measuring Theils U1: arfimax -  0.4727211038144437\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 4.126891109467226, 'mase': 7.672114783490544, 'smape': 0.6406746245456112, 'mdrae': 9.322143773906259, 'mdape': 0.5618914294802227}\n",
            "Measuring Theils U1: sarimax -  0.32339531074681455\n",
            "Evaluation_metrics:  arimax -  {'rmse': 4.486795716161623, 'mase': 8.2680383738179, 'smape': 0.6944715662199813, 'mdrae': 8.306833626302199, 'mdape': 0.6553017066655644}\n",
            "Measuring Theils U1: arimax -  0.3575474426699323\n",
            "Evaluation_metrics:  lr -  {'rmse': 3.6740798659074207, 'mase': 6.926175493791307, 'smape': 0.5809509792752516, 'mdrae': 7.496374693036908, 'mdape': 0.7032762652489537}\n",
            "Measuring Theils U1: lr -  0.27349951851060894\n",
            "Evaluation_metrics:  ardl -  {'rmse': 2.859964028952047, 'mase': 5.22805421137496, 'smape': 0.47366971967931665, 'mdrae': 5.881324222012759, 'mdape': 0.43579617758672196}\n",
            "Measuring Theils U1: ardl -  0.2328446201647163\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 2.559590892974818, 'mase': 4.444009428097305, 'smape': 0.403717396443204, 'mdrae': 5.10452073908345, 'mdape': 0.41327156399235027}\n",
            "Measuring Theils U1: xgboost -  0.21147633047460584\n",
            "Evaluation_metrics:  tft -  {'rmse': 3.575188568602542, 'mase': 6.106100932552137, 'smape': 0.5471432488123552, 'mdrae': 6.849313204982305, 'mdape': 0.49884425762284546}\n",
            "Measuring Theils U1: tft -  0.3063406219594081\n",
            "Evaluation_metrics:  warimax -  {'rmse': 4.106289966348329, 'mase': 5.931917390684671, 'smape': 0.5682814640549855, 'mdrae': 3.765247625476832, 'mdape': 0.3869765645991785}\n",
            "Measuring Theils U1: warimax -  0.44562657332215583\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 2.3058873400998925, 'mase': 4.120430728374381, 'smape': 0.38953078535588476, 'mdrae': 4.847743174110954, 'mdape': 0.36357276869052085}\n",
            "Measuring Theils U1: fewnet -  0.18890182850675324\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__Russia: Forecasts Evaluation: FEWNet and baselines:12M (table 05)__"
      ],
      "metadata": {
        "id": "sa9rek5--4Gu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([4.912471 ,5.194607 ,5.666425 ,5.782023 ,5.519744 ,6.014352 ,\n",
        "                   6.511526 ,6.469069 ,6.692744 ,7.408086,8.135334 ,8.403766])\n",
        "\n",
        "rw = np.array([4.44,\n",
        "4.45,\n",
        "4.47,\n",
        "4.48,\n",
        "4.49,\n",
        "4.51,\n",
        "4.52,\n",
        "4.53,\n",
        "4.55,\n",
        "4.56,\n",
        "4.58,\n",
        "4.59])\n",
        "rwd = np.array([4.39,\n",
        "4.36,\n",
        "4.32,\n",
        "4.29,\n",
        "4.25,\n",
        "4.22,\n",
        "4.19,\n",
        "4.15,\n",
        "4.12,\n",
        "4.08,\n",
        "4.05,\n",
        "4.02])\n",
        "ar = np.array([4.79,\n",
        "5.07,\n",
        "5.27,\n",
        "5.43,\n",
        "5.57,\n",
        "5.69,\n",
        "5.80,\n",
        "5.91,\n",
        "6.01,\n",
        "6.10,\n",
        "6.19,\n",
        "6.27])\n",
        "msgarch = np.array([4.61,\n",
        "5.18,\n",
        "5.53,\n",
        "5.53,\n",
        "5.84,\n",
        "6.86,\n",
        "5.40,\n",
        "5.69,\n",
        "5.46,\n",
        "6.48,\n",
        "6.58,\n",
        "5.57])\n",
        "deepar = np.array([7.48,\n",
        "7.64,\n",
        "7.66,\n",
        "7.67,\n",
        "7.57,\n",
        "7.35,\n",
        "6.95,\n",
        "6.66,\n",
        "6.38,\n",
        "6.02,\n",
        "5.38,\n",
        "4.92])\n",
        "arnnx = np.array([13.00,\n",
        "11.18,\n",
        "10.48,\n",
        "9.69,\n",
        "8.87,\n",
        "8.35,\n",
        "9.20,\n",
        "10.34,\n",
        "8.57,\n",
        "9.71,\n",
        "10.92,\n",
        "11.91])\n",
        "nbeats = np.array([4.43,\n",
        "4.84,\n",
        "4.68,\n",
        "4.46,\n",
        "4.73,\n",
        "4.21,\n",
        "4.30,\n",
        "4.07,\n",
        "4.06,\n",
        "3.83,\n",
        "3.80,\n",
        "4.11])\n",
        "arfimax = np.array([20.36,\n",
        "19.71,\n",
        "19.16,\n",
        "18.75,\n",
        "18.50,\n",
        "18.41,\n",
        "18.47,\n",
        "18.61,\n",
        "18.79,\n",
        "18.95,\n",
        "19.10,\n",
        "19.29])\n",
        "sarimax = np.array([13.76,\n",
        "13.21,\n",
        "12.71,\n",
        "12.46,\n",
        "12.41,\n",
        "12.59,\n",
        "12.63,\n",
        "12.95,\n",
        "13.19,\n",
        "13.16,\n",
        "13.14,\n",
        "13.32])\n",
        "arimax = np.array([14.85,\n",
        "14.26,\n",
        "13.80,\n",
        "13.52,\n",
        "13.45,\n",
        "13.59,\n",
        "13.85,\n",
        "14.13,\n",
        "14.34,\n",
        "14.46,\n",
        "14.57,\n",
        "14.78])\n",
        "lr = \tnp.array([5.04,\n",
        "9.28,\n",
        "11.66,\n",
        "11.26,\n",
        "11.02,\n",
        "10.94,\n",
        "10.99,\n",
        "11.14,\n",
        "11.32,\n",
        "11.44,\n",
        "11.51,\n",
        "11.57])\n",
        "ardl = np.array([7.47,\n",
        "9.13,\n",
        "8.68,\n",
        "8.53,\n",
        "9.35,\n",
        "10.01,\n",
        "10.29,\n",
        "10.63,\n",
        "11.14,\n",
        "11.51,\n",
        "11.56,\n",
        "11.66])\n",
        "xgboost = np.array([7.08,\n",
        "8.94,\n",
        "11.09,\n",
        "13.24,\n",
        "13.47,\n",
        "13.67,\n",
        "13.62,\n",
        "13.45,\n",
        "13.48,\n",
        "13.51,\n",
        "13.53,\n",
        "13.49])\n",
        "tft = \tnp.array([2.58,\n",
        "3.34,\n",
        "2.39,\n",
        "2.09,\n",
        "2.72,\n",
        "2.46,\n",
        "3.48,\n",
        "2.51,\n",
        "2.00,\n",
        "3.15,\n",
        "2.63,\n",
        "2.83])\n",
        "warimax = np.array([10.60,\n",
        "10.16,\n",
        "9.98,\n",
        "9.86,\n",
        "9.70,\n",
        "9.71,\n",
        "9.79,\n",
        "9.78,\n",
        "9.79,\n",
        "9.83,\n",
        "9.83,\n",
        "9.90])\n",
        "fewnet = np.array([5.44,\n",
        "6.38,\n",
        "6.83,\n",
        "6.78,\n",
        "6.51,\n",
        "6.14,\n",
        "5.84,\n",
        "5.80,\n",
        "6.05,\n",
        "6.50,\n",
        "7.02,\n",
        "7.51])"
      ],
      "metadata": {
        "id": "XfjUk6ky7cRV"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NPk3J8vX-86M",
        "outputId": "4c171dd3-e7fd-4d5a-cff5-32cb7e4f4587"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 2.1377974220282, 'mase': 5.038521188028161, 'smape': 0.32949187177799866, 'mdrae': 4.005692171351338, 'mdape': 0.274935853397656}\n",
            "Measuring Theils U1: rw -  0.19442799513468992\n",
            "Evaluation_metrics:  rwd -  {'rmse': 2.4868081445158605, 'mase': 5.87230829826063, 'smape': 0.3971410853840716, 'mdrae': 4.8411958276649, 'mdape': 0.3274353346337984}\n",
            "Measuring Theils U1: rwd -  0.2327192645202085\n",
            "Evaluation_metrics:  ar -  {'rmse': 0.9896407753811901, 'mase': 1.9471408031717006, 'smape': 0.10990040040691755, 'mdrae': 1.8286162420452583, 'mdape': 0.07819110016821623}\n",
            "Measuring Theils U1: ar -  0.08129630737113414\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 1.1425208762339603, 'mase': 2.3050871743424266, 'smape': 0.13260851924887504, 'mdrae': 2.1386569640427116, 'mdape': 0.12285499082217655}\n",
            "Measuring Theils U1: msgarch -  0.09333967069773833\n",
            "Evaluation_metrics:  deepar -  {'rmse': 2.0016712269910237, 'mase': 4.660649316741644, 'smape': 0.26399661410265723, 'mdrae': 4.225305095561149, 'mdape': 0.33260629887807547}\n",
            "Measuring Theils U1: deepar -  0.14995896218362306\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 4.157706189029846, 'mase': 10.173064026802798, 'smape': 0.4567948813872727, 'mdrae': 10.202186010240776, 'mdape': 0.507798519537693}\n",
            "Measuring Theils U1: arnnx -  0.24815849153021965\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 2.4928339340257737, 'mase': 5.630890046504312, 'smape': 0.37778424054951754, 'mdrae': 5.0019235547469725, 'mdape': 0.3198201186304852}\n",
            "Measuring Theils U1: nbeats -  0.2311021292224671\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 12.68334732937762, 'mase': 33.840994115653665, 'smape': 0.9975359063779056, 'mdrae': 40.5549040201785, 'mdape': 1.96888887084617}\n",
            "Measuring Theils U1: arfimax -  0.497441744160826\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 6.654575329122344, 'mase': 17.619028810138865, 'smape': 0.686300408269112, 'mdrae': 18.314634612000646, 'mdape': 1.0475798729903854}\n",
            "Measuring Theils U1: sarimax -  0.34218016464147205\n",
            "Evaluation_metrics:  arimax -  {'rmse': 7.799898142665951, 'mase': 20.764172145519783, 'smape': 0.7614704398181633, 'mdrae': 23.753628470617002, 'mdape': 1.2219175805266715}\n",
            "Measuring Theils U1: arimax -  0.37823283810394204\n",
            "Evaluation_metrics:  lr -  {'rmse': 4.454475838146438, 'mase': 11.279564347352588, 'smape': 0.4846237817885168, 'mdrae': 12.703150364610266, 'mdape': 0.7067123327633722}\n",
            "Measuring Theils U1: lr -  0.25858200378817026\n",
            "Evaluation_metrics:  ardl -  {'rmse': 3.649460491081353, 'mase': 9.667873981460874, 'smape': 0.44348194362928267, 'mdrae': 12.130573101519033, 'mdape': 0.5669911980619914}\n",
            "Measuring Theils U1: ardl -  0.22032995468887084\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 6.212316063216446, 'mase': 16.063222298820367, 'smape': 0.6303477678219319, 'mdrae': 15.478213046396716, 'mdape': 0.9856319259787147}\n",
            "Measuring Theils U1: xgboost -  0.32630257519898603\n",
            "Evaluation_metrics:  tft -  {'rmse': 3.874177945615958, 'mase': 9.954065036288746, 'smape': 0.809758079837256, 'mdrae': 7.5700916320745995, 'mdape': 0.5845978430109944}\n",
            "Measuring Theils U1: tft -  0.421191246291047\n",
            "Evaluation_metrics:  warimax -  {'rmse': 3.715420802452541, 'mase': 9.437632500619198, 'smape': 0.44091085234512345, 'mdrae': 9.142455351609637, 'mdape': 0.5631405110139653}\n",
            "Measuring Theils U1: warimax -  0.22662427582148603\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 0.8760151621388088, 'mase': 2.210967139399371, 'smape': 0.12882850218506564, 'mdrae': 2.873562086565973, 'mdape': 0.114983026253067}\n",
            "Measuring Theils U1: fewnet -  0.06788030956309019\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "__Russia: Forecasts Evaluation: FEWNet and baselines:24M (table 06)__"
      ],
      "metadata": {
        "id": "1KB2oUUX-9US"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([3.046190,2.423893,2.311721,2.546287,3.098436,3.026511,3.211885,3.366394,3.573624,3.666989,3.977587,\n",
        "4.423442,4.912471,5.194607,5.666425,5.782023,5.519744,6.014352,6.511526,6.469069,6.692744,7.408086,\n",
        "8.135334,8.403766])\n",
        "rw = np.array([3.55,\n",
        "3.56,\n",
        "3.57,\n",
        "3.59,\n",
        "3.60,\n",
        "3.61,\n",
        "3.62,\n",
        "3.63,\n",
        "3.64,\n",
        "3.65,\n",
        "3.67,\n",
        "3.68,\n",
        "3.69,\n",
        "3.70,\n",
        "3.71,\n",
        "3.72,\n",
        "3.74,\n",
        "3.75,\n",
        "3.76,\n",
        "3.77,\n",
        "3.78,\n",
        "3.79,\n",
        "3.80,\n",
        "3.82])\n",
        "rwd = np.array([3.50,\n",
        "3.46,\n",
        "3.42,\n",
        "3.38,\n",
        "3.34,\n",
        "3.29,\n",
        "3.25,\n",
        "3.21,\n",
        "3.17,\n",
        "3.13,\n",
        "3.09,\n",
        "3.05,\n",
        "3.01,\n",
        "2.97,\n",
        "2.93,\n",
        "2.89,\n",
        "2.85,\n",
        "2.81,\n",
        "2.77,\n",
        "2.72,\n",
        "2.68,\n",
        "2.64,\n",
        "2.60,\n",
        "2.56])\n",
        "ar = np.array([3.48,\n",
        "3.56,\n",
        "3.72,\n",
        "3.91,\n",
        "4.12,\n",
        "4.33,\n",
        "4.53,\n",
        "4.72,\n",
        "4.89,\n",
        "5.06,\n",
        "5.22,\n",
        "5.37,\n",
        "5.51,\n",
        "5.64,\n",
        "5.77,\n",
        "5.89,\n",
        "6.00,\n",
        "6.10,\n",
        "6.20,\n",
        "6.30,\n",
        "6.39,\n",
        "6.47,\n",
        "6.55,\n",
        "6.62])\n",
        "msgarch = np.array([3.72,\n",
        "4.46,\n",
        "5.10,\n",
        "5.70,\n",
        "5.70,\n",
        "6.02,\n",
        "5.61,\n",
        "5.99,\n",
        "6.09,\n",
        "6.22,\n",
        "5.43,\n",
        "5.73,\n",
        "5.71,\n",
        "5.95,\n",
        "6.13,\n",
        "6.90,\n",
        "6.86,\n",
        "6.43,\n",
        "5.86,\n",
        "6.16,\n",
        "6.66,\n",
        "6.84,\n",
        "7.52,\n",
        "5.93])\n",
        "deepar = np.array([7.10,\n",
        "7.58,\n",
        "7.71,\n",
        "7.94,\n",
        "7.95,\n",
        "7.70,\n",
        "7.53,\n",
        "7.36,\n",
        "7.00,\n",
        "6.68,\n",
        "6.69,\n",
        "6.05,\n",
        "6.16,\n",
        "6.28,\n",
        "6.22,\n",
        "5.74,\n",
        "5.76,\n",
        "5.49,\n",
        "5.46,\n",
        "5.10,\n",
        "4.88,\n",
        "4.86,\n",
        "4.68,\n",
        "4.60])\n",
        "arnnx = np.array([10.11,\n",
        "10.02,\n",
        "9.76,\n",
        "9.29,\n",
        "9.64,\n",
        "8.76,\n",
        "9.11,\n",
        "9.77,\n",
        "9.50,\n",
        "9.63,\n",
        "10.81,\n",
        "11.88,\n",
        "9.29,\n",
        "10.26,\n",
        "11.05,\n",
        "10.97,\n",
        "11.28,\n",
        "11.31,\n",
        "12.59,\n",
        "12.44,\n",
        "13.36,\n",
        "14.45,\n",
        "12.94,\n",
        "12.06])\n",
        "nbeats = np.array([4.61,\n",
        "4.30,\n",
        "4.97,\n",
        "6.09,\n",
        "5.84,\n",
        "5.21,\n",
        "5.80,\n",
        "6.05,\n",
        "7.99,\n",
        "8.80,\n",
        "8.40,\n",
        "9.54,\n",
        "11.13,\n",
        "11.27,\n",
        "11.30,\n",
        "11.59,\n",
        "12.77,\n",
        "13.11,\n",
        "15.19,\n",
        "15.16,\n",
        "13.85,\n",
        "13.57,\n",
        "13.79,\n",
        "10.78])\n",
        "arfimax = np.array([17.05,\n",
        "16.54,\n",
        "16.04,\n",
        "15.63,\n",
        "15.36,\n",
        "15.27,\n",
        "15.31,\n",
        "15.45,\n",
        "15.62,\n",
        "15.78,\n",
        "15.93,\n",
        "16.12,\n",
        "16.42,\n",
        "16.82,\n",
        "17.27,\n",
        "17.63,\n",
        "17.81,\n",
        "17.75,\n",
        "17.50,\n",
        "17.14,\n",
        "16.73,\n",
        "16.34,\n",
        "15.98,\n",
        "15.69])\n",
        "sarimax = np.array([14.81,\n",
        "15.27,\n",
        "15.75,\n",
        "16.11,\n",
        "16.19,\n",
        "16.01,\n",
        "15.70,\n",
        "15.20,\n",
        "14.84,\n",
        "14.50,\n",
        "14.18,\n",
        "14.02,\n",
        "14.08,\n",
        "14.10,\n",
        "13.88,\n",
        "13.57,\n",
        "13.19,\n",
        "13.03,\n",
        "12.85,\n",
        "13.17,\n",
        "13.50,\n",
        "13.44,\n",
        "13.06,\n",
        "12.56])\n",
        "arimax = np.array([14.92,\n",
        "15.38,\n",
        "15.80,\n",
        "16.04,\n",
        "16.03,\n",
        "15.77,\n",
        "15.34,\n",
        "14.84,\n",
        "14.33,\n",
        "13.89,\n",
        "13.58,\n",
        "13.41,\n",
        "13.34,\n",
        "13.26,\n",
        "13.05,\n",
        "12.72,\n",
        "12.35,\n",
        "12.13,\n",
        "12.17,\n",
        "12.42,\n",
        "12.70,\n",
        "12.77,\n",
        "12.54,\n",
        "12.08])\n",
        "lr = \tnp.array([4.50,\n",
        "10.31,\n",
        "11.89,\n",
        "11.49,\n",
        "11.24,\n",
        "11.14,\n",
        "11.18,\n",
        "11.33,\n",
        "11.49,\n",
        "11.64,\n",
        "11.73,\n",
        "11.78,\n",
        "11.91,\n",
        "12.11,\n",
        "12.43,\n",
        "12.73,\n",
        "12.93,\n",
        "12.92,\n",
        "12.73,\n",
        "12.42,\n",
        "12.04,\n",
        "11.67,\n",
        "11.32,\n",
        "11.05])\n",
        "ardl = np.array([7.61,\n",
        "10.01,\n",
        "9.27,\n",
        "8.71,\n",
        "9.52,\n",
        "10.29,\n",
        "10.50,\n",
        "10.70,\n",
        "11.15,\n",
        "11.54,\n",
        "11.58,\n",
        "11.62,\n",
        "12.02,\n",
        "12.62,\n",
        "13.09,\n",
        "13.38,\n",
        "13.45,\n",
        "13.27,\n",
        "12.91,\n",
        "12.38,\n",
        "11.90,\n",
        "11.61,\n",
        "11.34,\n",
        "11.09])\n",
        "xgboost = np.array([3.74,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59,\n",
        "3.66,\n",
        "3.66,\n",
        "3.66,\n",
        "3.66,\n",
        "3.66,\n",
        "3.66,\n",
        "3.64,\n",
        "3.59,\n",
        "3.59,\n",
        "3.59])\n",
        "tft = \tnp.array([6.13,\n",
        "7.10,\n",
        "6.47,\n",
        "6.15,\n",
        "7.08,\n",
        "6.61,\n",
        "7.50,\n",
        "6.86,\n",
        "6.16,\n",
        "6.98,\n",
        "6.55,\n",
        "6.55,\n",
        "6.50,\n",
        "6.58,\n",
        "5.81,\n",
        "6.36,\n",
        "6.28,\n",
        "6.29,\n",
        "6.40,\n",
        "5.59,\n",
        "6.26,\n",
        "5.46,\n",
        "4.99,\n",
        "5.74])\n",
        "warimax = np.array([10.43,\n",
        "9.86,\n",
        "9.32,\n",
        "9.02,\n",
        "8.68,\n",
        "8.57,\n",
        "8.65,\n",
        "8.72,\n",
        "9.00,\n",
        "9.17,\n",
        "9.42,\n",
        "9.80,\n",
        "10.13,\n",
        "10.60,\n",
        "10.91,\n",
        "11.08,\n",
        "11.09,\n",
        "10.83,\n",
        "10.59,\n",
        "10.26,\n",
        "9.94,\n",
        "9.70,\n",
        "9.37,\n",
        "9.13])\n",
        "fewnet = np.array([4.34,\n",
        "4.45,\n",
        "4.25,\n",
        "3.82,\n",
        "3.36,\n",
        "3.17,\n",
        "3.28,\n",
        "3.02,\n",
        "3.05,\n",
        "3.39,\n",
        "4.00,\n",
        "4.67,\n",
        "5.47,\n",
        "6.19,\n",
        "7.03,\n",
        "7.89,\n",
        "8.85,\n",
        "9.52,\n",
        "9.89,\n",
        "10.44,\n",
        "10.76,\n",
        "10.97,\n",
        "11.31,\n",
        "11.73])"
      ],
      "metadata": {
        "id": "frPfvb9V_C74"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "st53uLNB_Hx1",
        "outputId": "8c173c3b-6d0e-4fe5-9789-7c676a8e2901"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 2.061920369021235, 'mase': 4.869301437832342, 'smape': 0.34039984258644984, 'mdrae': 4.578073948545504, 'mdape': 0.33384926992624275}\n",
            "Measuring Theils U1: rw -  0.2337792157026489\n",
            "Evaluation_metrics:  rwd -  {'rmse': 2.738447313689152, 'mase': 6.399132404271898, 'smape': 0.4787911288049116, 'mdrae': 5.799746935089433, 'mdape': 0.4278544681534838}\n",
            "Measuring Theils U1: rwd -  0.3348185766456566\n",
            "Evaluation_metrics:  ar -  {'rmse': 1.02784796858193, 'mase': 2.673775483620843, 'smape': 0.20207957855357686, 'mdrae': 2.1230175725015976, 'mdape': 0.20356405640550818}\n",
            "Measuring Theils U1: ar -  0.09795032946392844\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 1.8181102374951488, 'mase': 4.632206158550134, 'smape': 0.323153286287231, 'mdrae': 4.711706440705008, 'mdape': 0.2685876200005698}\n",
            "Measuring Theils U1: msgarch -  0.1633113888671574\n",
            "Evaluation_metrics:  deepar -  {'rmse': 3.256978451574097, 'mase': 8.388846638634401, 'smape': 0.5082424496080964, 'mdrae': 8.28560478489943, 'mdape': 0.43867901436629536}\n",
            "Measuring Theils U1: deepar -  0.28120616887968997\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 6.116951143803967, 'mase': 18.319594700553772, 'smape': 0.8110779875599867, 'mdrae': 21.96232257327595, 'mdape': 1.3348529053133462}\n",
            "Measuring Theils U1: arnnx -  0.38035184590363913\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 5.263792505394882, 'mase': 14.631575153253102, 'smape': 0.66133597516685, 'mdrae': 15.107530496701337, 'mdape': 1.0906192215545532}\n",
            "Measuring Theils U1: nbeats -  0.34188979555590904\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 11.69097292849707, 'mase': 35.12239887731961, 'smape': 1.1166868676366948, 'mdrae': 41.20492597853337, 'mdape': 2.4933671593785895}\n",
            "Measuring Theils U1: arfimax -  0.5428015606048928\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 9.902088078944999, 'mase': 28.77930869260673, 'smape': 1.00629572094499, 'mdrae': 30.433691726641907, 'mdape': 2.0178261025773967}\n",
            "Measuring Theils U1: sarimax -  0.5085718164866107\n",
            "Evaluation_metrics:  arimax -  {'rmse': 9.492009433677255, 'mase': 27.243160680961093, 'smape': 0.9753873269653414, 'mdrae': 26.857073867505488, 'mdape': 1.8735567611968391}\n",
            "Measuring Theils U1: arimax -  0.49989498623298023\n",
            "Evaluation_metrics:  lr -  {'rmse': 6.972897143334182, 'mase': 20.30457608185719, 'smape': 0.844285366374505, 'mdrae': 24.510849368917487, 'mdape': 1.3834708675576852}\n",
            "Measuring Theils U1: lr -  0.41642168738204444\n",
            "Evaluation_metrics:  ardl -  {'rmse': 6.667129854918931, 'mase': 19.745746928649474, 'smape': 0.8444247988384631, 'mdrae': 24.476696558098656, 'mdape': 1.4725182278019961}\n",
            "Measuring Theils U1: ardl -  0.40284926310379043\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 2.1580587288930144, 'mase': 5.096878921039103, 'smape': 0.3592202267660675, 'mdrae': 4.760036229749616, 'mdape': 0.3455079356797618}\n",
            "Measuring Theils U1: xgboost -  0.24658452050948831\n",
            "Evaluation_metrics:  tft -  {'rmse': 2.7153174881619275, 'mase': 7.001540622602744, 'smape': 0.44441566823544204, 'mdrae': 7.211031803514806, 'mdape': 0.43368681765847955}\n",
            "Measuring Theils U1: tft -  0.2359139463793181\n",
            "Evaluation_metrics:  warimax -  {'rmse': 5.221321340333115, 'mase': 15.031100068216082, 'smape': 0.7216984162515416, 'mdrae': 17.52236974561253, 'mdape': 1.1387839990759336}\n",
            "Measuring Theils U1: warimax -  0.34972037491116\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 2.2390790267841525, 'mase': 5.2799124105939335, 'smape': 0.27485726243480463, 'mdrae': 3.5280609339013544, 'mdape': 0.3774030491789776}\n",
            "Measuring Theils U1: fewnet -  0.1821866427816819\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__India: Forecasts Evaluation: FEWNet and baselines:12M (table 05)__"
      ],
      "metadata": {
        "id": "-rRDxh1O_INk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([3.686636 ,3.162966 ,4.494493 ,5.665658 ,5.139860 ,\n",
        "                   5.257646 ,5.577841 ,5.263880 ,4.811442 ,4.403048\n",
        "                   ,4.518828 ,4.837364])\n",
        "rw = np.array([5.31,\n",
        "5.33,\n",
        "5.36,\n",
        "5.39,\n",
        "5.41,\n",
        "5.44,\n",
        "5.46,\n",
        "5.49,\n",
        "5.51,\n",
        "5.54,\n",
        "5.56,\n",
        "5.59])\n",
        "rwd = np.array([5.29,\n",
        "5.29,\n",
        "5.30,\n",
        "5.30,\n",
        "5.31,\n",
        "5.31,\n",
        "5.32,\n",
        "5.32,\n",
        "5.33,\n",
        "5.33,\n",
        "5.34,\n",
        "5.34])\n",
        "ar = np.array([5.27,\n",
        "5.37,\n",
        "5.48,\n",
        "5.58,\n",
        "5.68,\n",
        "5.77,\n",
        "5.86,\n",
        "5.94,\n",
        "6.02,\n",
        "6.09,\n",
        "6.15,\n",
        "6.21])\n",
        "msgarch = np.array([5.20,\n",
        "5.81,\n",
        "6.02,\n",
        "6.17,\n",
        "6.51,\n",
        "7.12,\n",
        "5.90,\n",
        "7.03,\n",
        "7.03,\n",
        "8.48,\n",
        "8.27,\n",
        "5.74])\n",
        "deepar = np.array([6.24,\n",
        "7.12,\n",
        "7.09,\n",
        "7.12,\n",
        "7.53,\n",
        "8.00,\n",
        "8.03,\n",
        "8.72,\n",
        "8.97,\n",
        "9.36,\n",
        "9.52,\n",
        "9.52])\n",
        "arnnx = np.array([3.44,\n",
        "2.83,\n",
        "3.03,\n",
        "3.12,\n",
        "2.85,\n",
        "2.66,\n",
        "2.77,\n",
        "3.29,\n",
        "3.88,\n",
        "4.25,\n",
        "4.28,\n",
        "4.08])\n",
        "nbeats = np.array([4.79,\n",
        "3.70,\n",
        "5.14,\n",
        "5.80,\n",
        "6.03,\n",
        "5.36,\n",
        "5.12,\n",
        "3.89,\n",
        "3.74,\n",
        "3.39,\n",
        "2.74,\n",
        "2.82])\n",
        "arfimax = np.array([6.01,\n",
        "6.29,\n",
        "6.14,\n",
        "5.66,\n",
        "5.19,\n",
        "5.03,\n",
        "5.35,\n",
        "6.03,\n",
        "6.74,\n",
        "7.17,\n",
        "7.19,\n",
        "6.97])\n",
        "sarimax = np.array([2.79,\n",
        "2.85,\n",
        "2.26,\n",
        "1.53,\n",
        "1.32,\n",
        "1.91,\n",
        "3.05,\n",
        "3.42,\n",
        "3.87,\n",
        "3.87,\n",
        "3.63,\n",
        "3.59])\n",
        "arimax = np.array([2.54,\n",
        "2.18,\n",
        "2.22,\n",
        "2.13,\n",
        "1.71,\n",
        "1.55,\n",
        "2.24,\n",
        "3.47,\n",
        "4.28,\n",
        "4.14,\n",
        "3.49,\n",
        "3.19])\n",
        "lr = \tnp.array([3.40,\n",
        "2.44,\n",
        "2.32,\n",
        "2.03,\n",
        "1.78,\n",
        "1.63,\n",
        "1.89,\n",
        "2.33,\n",
        "2.88,\n",
        "3.39,\n",
        "3.48,\n",
        "3.64])\n",
        "ardl = np.array([3.36,\n",
        "2.53,\n",
        "2.71,\n",
        "2.88,\n",
        "2.62,\n",
        "2.29,\n",
        "2.43,\n",
        "3.10,\n",
        "3.93,\n",
        "4.44,\n",
        "4.43,\n",
        "4.08])\n",
        "xgboost = np.array([3.95,\n",
        "3.51,\n",
        "3.35,\n",
        "3.65,\n",
        "3.07,\n",
        "3.56,\n",
        "3.07,\n",
        "3.19,\n",
        "3.06,\n",
        "2.81,\n",
        "2.88,\n",
        "3.17])\n",
        "tft = \tnp.array([6.18,\n",
        "7.53,\n",
        "5.06,\n",
        "3.97,\n",
        "5.09,\n",
        "4.46,\n",
        "6.41,\n",
        "4.85,\n",
        "3.97,\n",
        "5.95,\n",
        "5.14,\n",
        "4.72])\n",
        "warimax = np.array([4.01,\n",
        "4.35,\n",
        "4.35,\n",
        "3.95,\n",
        "3.52,\n",
        "3.41,\n",
        "3.74,\n",
        "4.35,\n",
        "4.99,\n",
        "5.41,\n",
        "5.49,\n",
        "5.33])\n",
        "fewnet = np.array([4.37,\n",
        "5.04,\n",
        "5.83,\n",
        "6.55,\n",
        "6.04,\n",
        "5.41,\n",
        "4.80,\n",
        "4.40,\n",
        "4.26,\n",
        "4.56,\n",
        "4.57,\n",
        "5.22])"
      ],
      "metadata": {
        "id": "AUO1iF1c_ML8"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MNqGaqSfRB8w",
        "outputId": "8d79c69e-8017-4446-cac6-9ab0e0c5c3f4"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 0.9899000497529198, 'mase': 1.5319119524936373, 'smape': 0.16337058621955172, 'mdrae': 1.5439861376683615, 'mdape': 0.15038743548283953}\n",
            "Measuring Theils U1: rw -  0.09668713391391495\n",
            "Evaluation_metrics:  rwd -  {'rmse': 0.9144226713255563, 'mase': 1.3436426902412522, 'smape': 0.14584704843229593, 'mdrae': 0.8052624179624096, 'mdape': 0.10584150475303086}\n",
            "Measuring Theils U1: rwd -  0.09050704697613471\n",
            "Evaluation_metrics:  ar -  {'rmse': 1.2348712126510062, 'mase': 2.090878153324105, 'smape': 0.21212255475289502, 'mdrae': 2.671212409804483, 'mdape': 0.2352270248009155}\n",
            "Measuring Theils U1: ar -  0.11670597974970655\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 2.1770925700348234, 'mase': 3.6770358827819196, 'smape': 0.32921003593555415, 'mdrae': 4.903562478152491, 'mdape': 0.34681757250241785}\n",
            "Measuring Theils U1: msgarch -  0.18987551214088952\n",
            "Evaluation_metrics:  deepar -  {'rmse': 3.5475866912359124, 'mase': 6.6140363739191255, 'smape': 0.5244980565232608, 'mdrae': 9.191442807454603, 'mdape': 0.6745863464014001}\n",
            "Measuring Theils U1: deepar -  0.273752221350548\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 1.6854735709885318, 'mase': 2.6750053132115905, 'smape': 0.33221181242054165, 'mdrae': 2.1736117453839894, 'mdape': 0.25971529782666614}\n",
            "Measuring Theils U1: arnnx -  0.20528346419158414\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 1.089677921146504, 'mase': 1.8213240463752587, 'smape': 0.22219125974213205, 'mdrae': 1.6929315063375932, 'mdape': 0.19793498017889885}\n",
            "Measuring Theils U1: nbeats -  0.11719230116397213\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 1.8627025796319443, 'mase': 2.9259708591924505, 'smape': 0.2760159252584735, 'mdrae': 2.440175691388477, 'mdape': 0.3834718557020201}\n",
            "Measuring Theils U1: arfimax -  0.16967540637738757\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 2.2749148885546244, 'mase': 3.7211275617270174, 'smape': 0.5149507399876124, 'mdrae': 3.9159278687759236, 'mdape': 0.3040747064894286}\n",
            "Measuring Theils U1: sarimax -  0.293269642124872\n",
            "Evaluation_metrics:  arimax -  {'rmse': 2.3084785520267457, 'mase': 3.876654343587683, 'smape': 0.5435050928555512, 'mdrae': 5.171672901910092, 'mdape': 0.3406701897058984}\n",
            "Measuring Theils U1: arimax -  0.3002873476857321\n",
            "Evaluation_metrics:  lr -  {'rmse': 2.457592747690478, 'mase': 4.192619279367772, 'smape': 0.5842729703170452, 'mdrae': 4.268965030406149, 'mdape': 0.4426197973419784}\n",
            "Measuring Theils U1: lr -  0.32852326027052875\n",
            "Evaluation_metrics:  ardl -  {'rmse': 1.883244888085677, 'mase': 2.9621411498563788, 'smape': 0.3834673448757051, 'mdrae': 2.377640203186566, 'mdape': 0.29857887631641844}\n",
            "Measuring Theils U1: ardl -  0.2322205137027711\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 1.6928849296888138, 'mase': 3.072956765042937, 'smape': 0.3836355432190701, 'mdrae': 3.936606834551787, 'mdape': 0.35878670525637985}\n",
            "Measuring Theils U1: xgboost -  0.20959859921382243\n",
            "Evaluation_metrics:  tft -  {'rmse': 1.6704821839319828, 'mase': 2.3479694899614523, 'smape': 0.23590014050094935, 'mdrae': 1.8597951546642413, 'mdape': 0.1504508915384339}\n",
            "Measuring Theils U1: tft -  0.1643839035812186\n",
            "Evaluation_metrics:  warimax -  {'rmse': 1.188845852147157, 'mase': 2.003691476537036, 'smape': 0.2256007443305709, 'mdrae': 2.4656385751175653, 'mdape': 0.22180554631964228}\n",
            "Measuring Theils U1: warimax -  0.12848593184181797\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 0.8764712793359709, 'mase': 1.4106547603101607, 'smape': 0.14941794694082147, 'mdrae': 1.2188233528613492, 'mdape': 0.1477700608601335}\n",
            "Measuring Theils U1: fewnet -  0.08831043610424189\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__India: Forecasts Evaluation: FEWNet and baselines:24M (table 06)__"
      ],
      "metadata": {
        "id": "ysqYqUV8_MtQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([9.634551,7.491857,6.840391,5.501618,5.448718,5.095541,5.063291,5.329154,5.625000,5.636574,5.902162,\n",
        "5.284787,3.686636,3.162966,4.494493,5.665658,5.139860,5.257646,5.577841,5.263880,4.811442,4.403048,\n",
        "4.518828,4.837364])\n",
        "rw = np.array([8.63,\n",
        "8.65,\n",
        "8.67,\n",
        "8.70,\n",
        "8.72,\n",
        "8.74,\n",
        "8.76,\n",
        "8.78,\n",
        "8.81,\n",
        "8.83,\n",
        "8.85,\n",
        "8.87,\n",
        "8.89,\n",
        "8.91,\n",
        "8.94,\n",
        "8.96,\n",
        "8.98,\n",
        "9.00,\n",
        "9.02,\n",
        "9.05,\n",
        "9.07,\n",
        "9.09,\n",
        "9.11,\n",
        "9.13])\n",
        "rwd = np.array([8.63,\n",
        "8.65,\n",
        "8.68,\n",
        "8.70,\n",
        "8.72,\n",
        "8.74,\n",
        "8.77,\n",
        "8.79,\n",
        "8.81,\n",
        "8.83,\n",
        "8.86,\n",
        "8.88,\n",
        "8.90,\n",
        "8.92,\n",
        "8.95,\n",
        "8.97,\n",
        "8.99,\n",
        "9.01,\n",
        "9.04,\n",
        "9.06,\n",
        "9.08,\n",
        "9.10,\n",
        "9.13,\n",
        "9.15])\n",
        "ar = np.array([8.72,\n",
        "8.68,\n",
        "8.61,\n",
        "8.53,\n",
        "8.46,\n",
        "8.40,\n",
        "8.34,\n",
        "8.28,\n",
        "8.23,\n",
        "8.18,\n",
        "8.13,\n",
        "8.08,\n",
        "8.04,\n",
        "8.00,\n",
        "7.97,\n",
        "7.93,\n",
        "7.90,\n",
        "7.87,\n",
        "7.84,\n",
        "7.82,\n",
        "7.79,\n",
        "7.77,\n",
        "7.75,\n",
        "7.73])\n",
        "msgarch = np.array([8.07,\n",
        "7.62,\n",
        "7.81,\n",
        "8.11,\n",
        "7.94,\n",
        "8.39,\n",
        "7.21,\n",
        "7.91,\n",
        "7.11,\n",
        "6.32,\n",
        "6.00,\n",
        "6.05,\n",
        "6.35,\n",
        "5.71,\n",
        "5.52,\n",
        "5.83,\n",
        "6.17,\n",
        "6.24,\n",
        "5.40,\n",
        "6.06,\n",
        "6.73,\n",
        "7.76,\n",
        "8.08,\n",
        "6.02])\n",
        "deepar = np.array([5.99,\n",
        "7.04,\n",
        "6.96,\n",
        "6.53,\n",
        "6.85,\n",
        "7.39,\n",
        "7.34,\n",
        "7.89,\n",
        "8.14,\n",
        "8.13,\n",
        "7.71,\n",
        "7.29,\n",
        "6.55,\n",
        "6.64,\n",
        "6.06,\n",
        "6.14,\n",
        "5.88,\n",
        "5.63,\n",
        "5.36,\n",
        "5.18,\n",
        "5.17,\n",
        "5.75,\n",
        "6.00,\n",
        "6.31])\n",
        "arnnx = np.array([3.89,\n",
        "1.69,\n",
        "3.11,\n",
        "3.90,\n",
        "3.33,\n",
        "2.83,\n",
        "2.70,\n",
        "3.34,\n",
        "4.28,\n",
        "4.62,\n",
        "4.45,\n",
        "4.10,\n",
        "3.85,\n",
        "3.87,\n",
        "4.17,\n",
        "4.53,\n",
        "4.59,\n",
        "4.38,\n",
        "4.20,\n",
        "3.87,\n",
        "3.69,\n",
        "4.11,\n",
        "5.06,\n",
        "5.68])\n",
        "nbeats = np.array([7.84,\n",
        "6.51,\n",
        "6.18,\n",
        "6.71,\n",
        "6.34,\n",
        "6.85,\n",
        "7.22,\n",
        "6.88,\n",
        "6.84,\n",
        "7.61,\n",
        "6.50,\n",
        "5.94,\n",
        "6.91,\n",
        "8.21,\n",
        "8.92,\n",
        "8.83,\n",
        "8.41,\n",
        "9.29,\n",
        "9.52,\n",
        "11.93,\n",
        "13.14,\n",
        "11.86,\n",
        "12.40,\n",
        "12.39])\n",
        "arfimax = np.array([7.00,\n",
        "6.93,\n",
        "6.69,\n",
        "6.36,\n",
        "6.06,\n",
        "5.97,\n",
        "6.25,\n",
        "6.86,\n",
        "7.54,\n",
        "7.99,\n",
        "8.06,\n",
        "7.86,\n",
        "7.66,\n",
        "7.69,\n",
        "7.97,\n",
        "8.29,\n",
        "8.39,\n",
        "8.16,\n",
        "7.75,\n",
        "7.49,\n",
        "7.63,\n",
        "8.17,\n",
        "8.84,\n",
        "9.29])\n",
        "sarimax = np.array([2.97,\n",
        "3.72,\n",
        "3.86,\n",
        "3.87,\n",
        "3.58,\n",
        "3.08,\n",
        "2.79,\n",
        "3.22,\n",
        "3.64,\n",
        "4.19,\n",
        "4.45,\n",
        "4.61,\n",
        "4.35,\n",
        "4.77,\n",
        "4.30,\n",
        "4.53,\n",
        "5.33,\n",
        "6.30,\n",
        "6.86,\n",
        "6.22,\n",
        "6.02,\n",
        "5.90,\n",
        "5.87,\n",
        "6.43])\n",
        "arimax = np.array([3.76,\n",
        "3.69,\n",
        "3.76,\n",
        "4.01,\n",
        "4.06,\n",
        "3.66,\n",
        "3.15,\n",
        "3.13,\n",
        "3.77,\n",
        "4.65,\n",
        "5.15,\n",
        "5.08,\n",
        "4.71,\n",
        "4.46,\n",
        "4.60,\n",
        "5.12,\n",
        "5.82,\n",
        "6.43,\n",
        "6.73,\n",
        "6.63,\n",
        "6.30,\n",
        "6.09,\n",
        "6.24,\n",
        "6.68])\n",
        "lr = \tnp.array([5.99,\n",
        "3.42,\n",
        "3.16,\n",
        "2.91,\n",
        "2.36,\n",
        "2.06,\n",
        "2.06,\n",
        "2.56,\n",
        "3.07,\n",
        "3.54,\n",
        "3.44,\n",
        "3.06,\n",
        "3.23,\n",
        "3.58,\n",
        "3.80,\n",
        "4.04,\n",
        "4.02,\n",
        "3.57,\n",
        "2.89,\n",
        "2.68,\n",
        "2.91,\n",
        "3.43,\n",
        "4.16,\n",
        "4.75])\n",
        "ardl = np.array([1.30,\n",
        "1.52,\n",
        "4.12,\n",
        "4.40,\n",
        "2.60,\n",
        "2.11,\n",
        "3.45,\n",
        "4.81,\n",
        "5.08,\n",
        "4.65,\n",
        "4.25,\n",
        "4.09,\n",
        "4.09,\n",
        "4.27,\n",
        "4.53,\n",
        "4.62,\n",
        "4.38,\n",
        "3.86,\n",
        "3.43,\n",
        "3.42,\n",
        "3.86,\n",
        "4.51,\n",
        "5.02,\n",
        "5.14])\n",
        "xgboost = np.array([4.42,\n",
        "3.98,\n",
        "3.65,\n",
        "2.93,\n",
        "3.04,\n",
        "3.26,\n",
        "3.26,\n",
        "3.97,\n",
        "4.22,\n",
        "4.24,\n",
        "4.24,\n",
        "4.22,\n",
        "4.22,\n",
        "4.21,\n",
        "4.18,\n",
        "4.22,\n",
        "4.22,\n",
        "4.14,\n",
        "4.21,\n",
        "3.54,\n",
        "3.36,\n",
        "4.19,\n",
        "5.09,\n",
        "5.31])\n",
        "tft = \tnp.array([9.51,\n",
        "11.22,\n",
        "8.87,\n",
        "6.66,\n",
        "9.72,\n",
        "8.86,\n",
        "10.55,\n",
        "8.33,\n",
        "6.90,\n",
        "9.95,\n",
        "9.33,\n",
        "9.33,\n",
        "10.37,\n",
        "11.40,\n",
        "9.27,\n",
        "11.94,\n",
        "12.22,\n",
        "13.43,\n",
        "15.13,\n",
        "13.76,\n",
        "15.58,\n",
        "13.32,\n",
        "10.10,\n",
        "14.45])\n",
        "warimax = np.array([3.67,\n",
        "3.47,\n",
        "3.46,\n",
        "3.06,\n",
        "2.68,\n",
        "2.91,\n",
        "3.40,\n",
        "4.30,\n",
        "5.24,\n",
        "5.78,\n",
        "6.05,\n",
        "5.92,\n",
        "5.83,\n",
        "6.13,\n",
        "6.52,\n",
        "6.98,\n",
        "7.08,\n",
        "6.69,\n",
        "6.25,\n",
        "5.91,\n",
        "6.01,\n",
        "6.62,\n",
        "7.20,\n",
        "7.58])\n",
        "fewnet = np.array([7.69,\n",
        "6.34,\n",
        "5.92,\n",
        "5.11,\n",
        "4.53,\n",
        "4.15,\n",
        "3.95,\n",
        "5.57,\n",
        "5.37,\n",
        "4.25,\n",
        "3.85,\n",
        "3.37,\n",
        "3.86,\n",
        "4.75,\n",
        "5.39,\n",
        "5.65,\n",
        "4.78,\n",
        "4.36,\n",
        "4.87,\n",
        "4.06,\n",
        "4.28,\n",
        "4.73,\n",
        "5.04,\n",
        "4.72])"
      ],
      "metadata": {
        "id": "_ucwEkpG_RHG"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tH-K51Mf_Riq",
        "outputId": "9696a250-213c-49ce-fe0b-3579b287686d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 3.7228611467557005, 'mase': 6.195391103065349, 'smape': 0.512477209686288, 'mdrae': 10.765736223992977, 'mdape': 0.6950976231695414}\n",
            "Measuring Theils U1: rw -  0.2580927628929062\n",
            "Evaluation_metrics:  rwd -  {'rmse': 3.731725546092116, 'mase': 6.20988395860427, 'smape': 0.5133242321008957, 'mdrae': 10.812657908208227, 'mdape': 0.6969947310341758}\n",
            "Measuring Theils U1: rwd -  0.25855647575911767\n",
            "Evaluation_metrics:  ar -  {'rmse': 2.9203220415489284, 'mase': 4.870019464031013, 'smape': 0.4289481159243113, 'mdrae': 8.244371858608192, 'mdape': 0.5437299101788028}\n",
            "Measuring Theils U1: ar -  0.21356142096677064\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 1.9168800646336308, 'mase': 2.769691795690258, 'smape': 0.2641945464590842, 'mdrae': 3.7127232075141507, 'mdape': 0.23632455238496608}\n",
            "Measuring Theils U1: msgarch -  0.15385765549294597\n",
            "Evaluation_metrics:  deepar -  {'rmse': 1.8638063102666633, 'mase': 2.685015403132891, 'smape': 0.2569428825540427, 'mdrae': 3.2479659855433036, 'mdape': 0.30610725954081985}\n",
            "Measuring Theils U1: deepar -  0.15300878447135757\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 2.215878661055963, 'mase': 2.894347744751626, 'smape': 0.34828970655714664, 'mdrae': 4.303130903261097, 'mdape': 0.23609463053102442}\n",
            "Measuring Theils U1: arnnx -  0.23202687365555485\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 4.181734082453892, 'mase': 5.828323014546214, 'smape': 0.45756639922992415, 'mdrae': 6.2193846317824315, 'mdape': 0.4922313742876362}\n",
            "Measuring Theils U1: nbeats -  0.2925218574920606\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 2.722908527296348, 'mase': 4.197152123640643, 'smape': 0.3754259064547967, 'mdrae': 6.472962282243814, 'mdape': 0.4202162584085056}\n",
            "Measuring Theils U1: arfimax -  0.20743553521107566\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 2.1607329320988518, 'mase': 3.01409974628429, 'smape': 0.3432500640871084, 'mdrae': 4.004306749323285, 'mdape': 0.29778998082988406}\n",
            "Measuring Theils U1: sarimax -  0.20953002313815147\n",
            "Evaluation_metrics:  arimax -  {'rmse': 2.01737319482287, 'mase': 2.83083860249452, 'smape': 0.3124869313414736, 'mdrae': 4.064650303973539, 'mdape': 0.2796562121294843}\n",
            "Measuring Theils U1: arimax -  0.19062386283927216\n",
            "Evaluation_metrics:  lr -  {'rmse': 2.3603683617230526, 'mase': 3.609980023552024, 'smape': 0.4736300814002367, 'mdrae': 4.202657602633436, 'mdape': 0.4061772197846746}\n",
            "Measuring Theils U1: lr -  0.2620379317467798\n",
            "Evaluation_metrics:  ardl -  {'rmse': 2.527816408257629, 'mase': 2.9765887578642327, 'smape': 0.3687675030552929, 'mdrae': 2.113991636357627, 'mdape': 0.21315786092946049}\n",
            "Measuring Theils U1: ardl -  0.2640604703827403\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 1.9538114105571107, 'mase': 2.7972456126408556, 'smape': 0.33047581887500327, 'mdrae': 4.271899934642359, 'mdape': 0.255101384038214}\n",
            "Measuring Theils U1: xgboost -  0.20365531003888812\n",
            "Evaluation_metrics:  tft -  {'rmse': 6.168817627770319, 'mase': 9.476573597077092, 'smape': 0.6534851169177678, 'mdrae': 13.465513374616398, 'mdape': 1.0730745641895103}\n",
            "Measuring Theils U1: tft -  0.37062019606766544\n",
            "Evaluation_metrics:  warimax -  {'rmse': 2.3538678449217363, 'mase': 3.3874829516759553, 'smape': 0.3708189126759273, 'mdrae': 3.6898961205044336, 'mdape': 0.4031909401241789}\n",
            "Measuring Theils U1: warimax -  0.21174183812581426\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 1.0392446855079414, 'mase': 1.4908116430106715, 'smape': 0.17022205025935655, 'mdrae': 1.4127997472489435, 'mdape': 0.1441501293546582}\n",
            "Measuring Theils U1: fewnet -  0.09910524186920876\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__China: Forecasts Evaluation: FEWNet and baselines:12M (table 05)__"
      ],
      "metadata": {
        "id": "Y2fQls6g_SG6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([0.2697842 ,-0.2216312 ,-0.4227999  ,0.3219466  ,0.9226485  ,\n",
        "                   1.6443494  ,1.2401969  ,1.9125597  ,0.6415800  ,0.4613803  ,\n",
        "                   1.4328405  ,2.4772978])\n",
        "rw = np.array([-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45,\n",
        "-0.45])\n",
        "rwd = np.array([-0.47,\n",
        "-0.49,\n",
        "-0.50,\n",
        "-0.52,\n",
        "-0.54,\n",
        "-0.56,\n",
        "-0.58,\n",
        "-0.59,\n",
        "-0.61,\n",
        "-0.63,\n",
        "-0.65,\n",
        "-0.67])\n",
        "ar = np.array([-0.61,\n",
        "-0.64,\n",
        "-0.47,\n",
        "-0.26,\n",
        "-0.02,\n",
        "0.22,\n",
        "0.45,\n",
        "0.66,\n",
        "0.86,\n",
        "1.03,\n",
        "1.19,\n",
        "1.33])\n",
        "msgarch = np.array([1.15,\n",
        "1.67,\n",
        "1.98,\n",
        "2.16,\n",
        "2.37,\n",
        "2.41,\n",
        "2.48,\n",
        "2.93,\n",
        "2.83,\n",
        "3.39,\n",
        "3.56,\n",
        "2.85])\n",
        "deepar = np.array([1.90,\n",
        "1.58,\n",
        "1.68,\n",
        "1.69,\n",
        "1.55,\n",
        "1.56,\n",
        "2.19,\n",
        "1.85,\n",
        "2.28,\n",
        "2.38,\n",
        "2.39,\n",
        "1.85])\n",
        "arnnx = np.array([0.64,\n",
        "1.40,\n",
        "1.88,\n",
        "2.20,\n",
        "2.52,\n",
        "2.89,\n",
        "3.31,\n",
        "3.66,\n",
        "3.82,\n",
        "3.73,\n",
        "3.46,\n",
        "3.12])\n",
        "nbeats = np.array([0.10,\n",
        "0.20,\n",
        "-0.02,\n",
        "-0.14,\n",
        "0.15,\n",
        "0.32,\n",
        "0.34,\n",
        "1.15,\n",
        "1.40,\n",
        "2.22,\n",
        "2.73,\n",
        "2.82])\n",
        "arfimax = np.array([10.43,\n",
        "10.49,\n",
        "10.49,\n",
        "10.60,\n",
        "11.03,\n",
        "11.69,\n",
        "12.33,\n",
        "12.63,\n",
        "12.44,\n",
        "11.86,\n",
        "11.12,\n",
        "10.50])\n",
        "sarimax = np.array([4.74,\n",
        "3.92,\n",
        "4.68,\n",
        "4.73,\n",
        "5.75,\n",
        "6.42,\n",
        "6.66,\n",
        "6.80,\n",
        "6.01,\n",
        "5.15,\n",
        "4.72,\n",
        "4.30])\n",
        "arimax = np.array([5.26,\n",
        "5.09,\n",
        "5.21,\n",
        "5.68,\n",
        "6.45,\n",
        "7.18,\n",
        "7.53,\n",
        "7.32,\n",
        "6.64,\n",
        "5.78,\n",
        "5.05,\n",
        "4.61])\n",
        "lr = \tnp.array([0.44,\n",
        "1.60,\n",
        "2.24,\n",
        "2.54,\n",
        "2.97,\n",
        "3.60,\n",
        "4.08,\n",
        "4.39,\n",
        "4.31,\n",
        "3.70,\n",
        "3.45,\n",
        "3.21])\n",
        "ardl = np.array([3.93,\n",
        "2.99,\n",
        "2.76,\n",
        "3.95,\n",
        "4.50,\n",
        "5.20,\n",
        "5.67,\n",
        "5.39,\n",
        "4.67,\n",
        "3.81,\n",
        "3.05,\n",
        "2.63])\n",
        "xgboost = np.array([-0.86,\n",
        "-1.23,\n",
        "-0.69,\n",
        "-0.43,\n",
        "-0.21,\n",
        "0.66,\n",
        "1.56,\n",
        "1.81,\n",
        "2.23,\n",
        "2.29,\n",
        "2.25,\n",
        "2.42])\n",
        "tft = \tnp.array([-0.87,\n",
        "-0.62,\n",
        "-2.07,\n",
        "-2.62,\n",
        "-1.56,\n",
        "-1.93,\n",
        "-0.74,\n",
        "-1.24,\n",
        "-1.51,\n",
        "0.34,\n",
        "0.60,\n",
        "0.87])\n",
        "warimax = np.array([4.48,\n",
        "4.85,\n",
        "4.93,\n",
        "5.00,\n",
        "5.42,\n",
        "6.14,\n",
        "6.65,\n",
        "6.88,\n",
        "6.77,\n",
        "6.18,\n",
        "5.57,\n",
        "5.06])\n",
        "fewnet = np.array([-0.19,\n",
        "-0.45,\n",
        "-0.17,\n",
        "0.50,\n",
        "1.24,\n",
        "2.03,\n",
        "2.44,\n",
        "3.15,\n",
        "2.86,\n",
        "3.34,\n",
        "4.45,\n",
        "5.44])"
      ],
      "metadata": {
        "id": "rdYzPZn0_UzY"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BEPfaQ3m_VMx",
        "outputId": "a79497f4-28c0-4c80-b61e-232c5e72d646"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 1.5814582384278058, 'mase': 2.018272292450006, 'smape': 1.7285308862693662, 'mdrae': 2.2850743434830028, 'mdape': 1.3384535418132404}\n",
            "Measuring Theils U1: rw -  0.9448743673739093\n",
            "Evaluation_metrics:  rwd -  {'rmse': 1.7056237024593608, 'mase': 2.195245979687263, 'smape': 1.7434627263299314, 'mdrae': 2.434899073494707, 'mdape': 1.4606560130320991}\n",
            "Measuring Theils U1: rwd -  0.95039302680613\n",
            "Evaluation_metrics:  ar -  {'rmse': 0.8239597556259518, 'mase': 1.0686489832824013, 'smape': 1.0295806854641663, 'mdrae': 1.0984630964713957, 'mdape': 0.7605606081307084}\n",
            "Measuring Theils U1: ar -  0.4175516727206946\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 1.7509042835226158, 'mase': 2.3972839607462455, 'smape': 1.069851641958242, 'mdrae': 2.4094338635503845, 'mdape': 2.415679213800885}\n",
            "Measuring Theils U1: msgarch -  0.46173190285295046\n",
            "Evaluation_metrics:  deepar -  {'rmse': 1.328492846508596, 'mase': 1.7280992312162613, 'smape': 0.9393726918592739, 'mdrae': 1.2891000541778201, 'mdape': 1.6597876846210093}\n",
            "Measuring Theils U1: deepar -  0.4208480113718435\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 2.0071995280052533, 'mase': 2.7549967328215526, 'smape': 1.1133587067727855, 'mdrae': 2.598954462888346, 'mdape': 1.7000993437824103}\n",
            "Measuring Theils U1: arnnx -  0.4882970337635381\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 0.9033729604789551, 'mase': 1.17641285522624, 'smape': 1.1642728771138062, 'mdrae': 1.2862428100716439, 'mdape': 0.8713654112485169}\n",
            "Measuring Theils U1: nbeats -  0.3427351578732065\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 10.45208741691779, 'mase': 15.680351563809328, 'smape': 1.7101216097220746, 'mdrae': 15.93996618850121, 'mdape': 14.672173495580378}\n",
            "Measuring Theils U1: arfimax -  0.832593836196486\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 4.536037994537874, 'mase': 6.677285900313961, 'smape': 1.4622209905409422, 'mdrae': 7.269052212991399, 'mdape': 6.7997790530555635}\n",
            "Measuring Theils U1: sarimax -  0.6844385171310147\n",
            "Evaluation_metrics:  arimax -  {'rmse': 5.207774204836251, 'mase': 7.671350866923234, 'smape': 1.5016423762144646, 'mdrae': 8.0424441970849, 'mdape': 7.6700971865189524}\n",
            "Measuring Theils U1: arimax -  0.7149948095302945\n",
            "Evaluation_metrics:  lr -  {'rmse': 2.3474762189191987, 'mase': 3.244498420924605, 'smape': 1.1502681863295179, 'mdrae': 3.408265396961744, 'mdape': 2.254397103691885}\n",
            "Measuring Theils U1: lr -  0.5252602860948301\n",
            "Evaluation_metrics:  ardl -  {'rmse': 3.3446540333629686, 'mase': 4.753167726308884, 'smape': 1.32504763443388, 'mdrae': 5.171970102276333, 'mdape': 5.078084430688968}\n",
            "Measuring Theils U1: ardl -  0.6201313676019334\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 0.9930017299705532, 'mase': 1.253646130343332, 'smape': 0.9925071065225777, 'mdrae': 1.2497603227848375, 'mdape': 0.9297916588523448}\n",
            "Measuring Theils U1: xgboost -  0.35323776742244434\n",
            "Evaluation_metrics:  tft -  {'rmse': 2.112394228371647, 'mase': 2.7650761499405605, 'smape': 1.5292412126485722, 'mdrae': 3.950265761040802, 'mdape': 1.9855782792913135}\n",
            "Measuring Theils U1: tft -  0.8003560917371556\n",
            "Evaluation_metrics:  warimax -  {'rmse': 4.851194615699433, 'mase': 7.185614576420974, 'smape': 1.4873278926899929, 'mdrae': 7.3880355951596295, 'mdape': 7.2132338416357245}\n",
            "Measuring Theils U1: warimax -  0.6989645843101173\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 1.6997079014910703, 'mase': 1.9248904821556483, 'smape': 0.8471877905494606, 'mdrae': 1.745440938336422, 'mdape': 0.9989147895399106}\n",
            "Measuring Theils U1: fewnet -  0.42768239319136137\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "__China: Forecasts Evaluation: FEWNet and baselines:24M (table 06)__"
      ],
      "metadata": {
        "id": "FX4ZKZHB_ViX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual = np.array([4.4131455,5.4205607,5.1803885,4.2711235,3.2467532,2.4118738,2.5069638,1.7576318,2.3875115,\n",
        "1.7304189,0.5415162,-0.4496403,0.2697842,-0.2216312,-0.4227999,0.3219466,0.9226485,1.6443494,\n",
        "1.2401969,1.9125597,0.6415800,0.4613803,1.4328405,2.4772978])\n",
        "rw = np.array([4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51,\n",
        "4.51])\n",
        "rwd = np.array([4.52,\n",
        "4.53,\n",
        "4.53,\n",
        "4.54,\n",
        "4.55,\n",
        "4.55,\n",
        "4.56,\n",
        "4.57,\n",
        "4.57,\n",
        "4.58,\n",
        "4.59,\n",
        "4.59,\n",
        "4.60,\n",
        "4.61,\n",
        "4.61,\n",
        "4.62,\n",
        "4.63,\n",
        "4.63,\n",
        "4.64,\n",
        "4.65,\n",
        "4.66,\n",
        "4.66,\n",
        "4.67,\n",
        "4.68])\n",
        "ar = np.array([4.61,\n",
        "4.67,\n",
        "4.57,\n",
        "4.46,\n",
        "4.32,\n",
        "4.19,\n",
        "4.06,\n",
        "3.94,\n",
        "3.83,\n",
        "3.73,\n",
        "3.64,\n",
        "3.56,\n",
        "3.48,\n",
        "3.42,\n",
        "3.36,\n",
        "3.30,\n",
        "3.25,\n",
        "3.21,\n",
        "3.17,\n",
        "3.13,\n",
        "3.10,\n",
        "3.07,\n",
        "3.04,\n",
        "3.02])\n",
        "msgarch = np.array([4.10,\n",
        "3.78,\n",
        "3.87,\n",
        "3.90,\n",
        "3.86,\n",
        "4.38,\n",
        "3.71,\n",
        "3.47,\n",
        "3.48,\n",
        "3.32,\n",
        "2.78,\n",
        "2.73,\n",
        "3.10,\n",
        "2.87,\n",
        "2.78,\n",
        "2.91,\n",
        "3.11,\n",
        "2.87,\n",
        "3.25,\n",
        "3.12,\n",
        "3.68,\n",
        "3.54,\n",
        "3.81,\n",
        "3.35])\n",
        "deepar = np.array([1.94,\n",
        "1.63,\n",
        "1.88,\n",
        "1.79,\n",
        "1.68,\n",
        "1.49,\n",
        "1.99,\n",
        "1.84,\n",
        "2.20,\n",
        "2.41,\n",
        "2.25,\n",
        "1.79,\n",
        "1.74,\n",
        "2.08,\n",
        "2.01,\n",
        "2.04,\n",
        "2.01,\n",
        "2.03,\n",
        "1.71,\n",
        "1.49,\n",
        "1.60,\n",
        "1.26,\n",
        "1.26,\n",
        "1.38])\n",
        "arnnx = np.array([2.14,\n",
        "3.08,\n",
        "3.58,\n",
        "4.02,\n",
        "3.73,\n",
        "4.22,\n",
        "5.17,\n",
        "5.08,\n",
        "4.37,\n",
        "3.61,\n",
        "3.17,\n",
        "2.10,\n",
        "3.10,\n",
        "2.31,\n",
        "2.23,\n",
        "2.07,\n",
        "1.72,\n",
        "1.76,\n",
        "1.29,\n",
        "1.36,\n",
        "1.22,\n",
        "0.94,\n",
        "1.19,\n",
        "2.20])\n",
        "nbeats = np.array([3.76,\n",
        "4.09,\n",
        "3.80,\n",
        "4.56,\n",
        "4.12,\n",
        "4.41,\n",
        "3.73,\n",
        "4.14,\n",
        "3.41,\n",
        "2.63,\n",
        "2.44,\n",
        "2.18,\n",
        "2.02,\n",
        "1.94,\n",
        "2.04,\n",
        "1.18,\n",
        "1.47,\n",
        "0.54,\n",
        "0.95,\n",
        "0.55,\n",
        "1.10,\n",
        "1.73,\n",
        "1.69,\n",
        "1.76])\n",
        "arfimax = np.array([19.84,\n",
        "19.91,\n",
        "19.90,\n",
        "20.09,\n",
        "20.59,\n",
        "21.34,\n",
        "22.04,\n",
        "22.32,\n",
        "22.08,\n",
        "21.49,\n",
        "20.76,\n",
        "20.16,\n",
        "19.83,\n",
        "19.74,\n",
        "19.74,\n",
        "19.70,\n",
        "19.55,\n",
        "19.32,\n",
        "19.08,\n",
        "18.91,\n",
        "18.88,\n",
        "19.01,\n",
        "19.24,\n",
        "19.47])\n",
        "sarimax = np.array([4.50,\n",
        "4.95,\n",
        "4.30,\n",
        "4.23,\n",
        "4.18,\n",
        "4.06,\n",
        "3.75,\n",
        "3.40,\n",
        "3.85,\n",
        "4.09,\n",
        "4.62,\n",
        "4.64,\n",
        "4.67,\n",
        "4.08,\n",
        "5.26,\n",
        "4.94,\n",
        "5.15,\n",
        "5.22,\n",
        "5.43,\n",
        "5.61,\n",
        "5.40,\n",
        "5.33,\n",
        "5.74,\n",
        "6.10])\n",
        "arimax = np.array([5.41,\n",
        "5.38,\n",
        "5.34,\n",
        "5.18,\n",
        "4.93,\n",
        "4.66,\n",
        "4.45,\n",
        "4.41,\n",
        "4.55,\n",
        "4.83,\n",
        "5.12,\n",
        "5.32,\n",
        "5.39,\n",
        "5.37,\n",
        "5.40,\n",
        "5.56,\n",
        "5.82,\n",
        "6.04,\n",
        "6.12,\n",
        "6.03,\n",
        "5.92,\n",
        "5.98,\n",
        "6.28,\n",
        "6.73])\n",
        "lr = \tnp.array([3.94,\n",
        "3.59,\n",
        "3.22,\n",
        "2.92,\n",
        "3.15,\n",
        "3.50,\n",
        "3.82,\n",
        "3.90,\n",
        "3.75,\n",
        "3.42,\n",
        "3.03,\n",
        "2.67,\n",
        "2.44,\n",
        "2.28,\n",
        "2.10,\n",
        "1.95,\n",
        "1.84,\n",
        "1.69,\n",
        "1.53,\n",
        "1.42,\n",
        "1.43,\n",
        "1.55,\n",
        "1.72,\n",
        "1.90])\n",
        "ardl = np.array([3.83,\n",
        "3.64,\n",
        "3.62,\n",
        "3.91,\n",
        "4.64,\n",
        "5.35,\n",
        "5.68,\n",
        "5.46,\n",
        "4.75,\n",
        "3.88,\n",
        "3.13,\n",
        "2.70,\n",
        "2.58,\n",
        "2.56,\n",
        "2.51,\n",
        "2.19,\n",
        "1.84,\n",
        "1.52,\n",
        "1.23,\n",
        "1.09,\n",
        "1.10,\n",
        "1.26,\n",
        "1.48,\n",
        "1.60])\n",
        "xgboost = np.array([-0.97,\n",
        "-0.98,\n",
        "-0.69,\n",
        "-0.55,\n",
        "-0.46,\n",
        "0.58,\n",
        "1.21,\n",
        "1.84,\n",
        "2.05,\n",
        "2.06,\n",
        "2.30,\n",
        "2.45,\n",
        "2.50,\n",
        "2.80,\n",
        "2.89,\n",
        "2.52,\n",
        "2.44,\n",
        "2.44,\n",
        "2.32,\n",
        "2.13,\n",
        "2.11,\n",
        "2.00,\n",
        "1.87,\n",
        "1.72])\n",
        "tft = \tnp.array([2.11,\n",
        "2.61,\n",
        "0.84,\n",
        "-0.20,\n",
        "1.29,\n",
        "0.81,\n",
        "2.21,\n",
        "1.23,\n",
        "0.39,\n",
        "2.28,\n",
        "1.70,\n",
        "1.64,\n",
        "2.03,\n",
        "2.47,\n",
        "1.42,\n",
        "2.48,\n",
        "2.31,\n",
        "2.47,\n",
        "3.01,\n",
        "2.23,\n",
        "2.98,\n",
        "2.08,\n",
        "1.63,\n",
        "2.19])\n",
        "warimax = np.array([8.96,\n",
        "9.21,\n",
        "9.18,\n",
        "9.39,\n",
        "9.92,\n",
        "10.43,\n",
        "10.89,\n",
        "10.99,\n",
        "10.73,\n",
        "10.21,\n",
        "9.44,\n",
        "8.86,\n",
        "8.62,\n",
        "8.58,\n",
        "8.53,\n",
        "8.42,\n",
        "8.36,\n",
        "8.19,\n",
        "7.94,\n",
        "7.83,\n",
        "7.94,\n",
        "8.11,\n",
        "8.28,\n",
        "8.53])\n",
        "fewnet = np.array([4.10,\n",
        "3.71,\n",
        "3.68,\n",
        "3.25,\n",
        "3.28,\n",
        "3.31,\n",
        "3.58,\n",
        "3.63,\n",
        "3.40,\n",
        "3.39,\n",
        "3.24,\n",
        "2.90,\n",
        "3.20,\n",
        "3.75,\n",
        "3.38,\n",
        "3.42,\n",
        "3.06,\n",
        "2.82,\n",
        "2.58,\n",
        "2.39,\n",
        "2.44,\n",
        "2.16,\n",
        "1.88,\n",
        "2.08])"
      ],
      "metadata": {
        "id": "GqfuHmWb_ZWA"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Generate the values of the different evaluation metrics\n",
        "def evaluate(actual: np.ndarray, predicted: np.ndarray, metrics=('rmse',\n",
        "                                                               'mase',\n",
        "                                                               'smape',\n",
        "                                                               'mdrae',\n",
        "                                                               'mdape'\n",
        "                                                              )):\n",
        " results = {}\n",
        " for name in metrics:\n",
        "     try:\n",
        "         results[name] = METRICS[name](actual, predicted)\n",
        "     except Exception as err:\n",
        "         results[name] = np.nan\n",
        "         print('Unable to compute metric {0}: {1}'.format(name, err))\n",
        " return results\n",
        "\n",
        "\n",
        "print(\"Evaluation_metrics:  rw - \", evaluate(actual,rw))\n",
        "print(\"Measuring Theils U1: rw - \", theils_u1(actual,rw))\n",
        "\n",
        "print(\"Evaluation_metrics:  rwd - \", evaluate(actual,rwd))\n",
        "print(\"Measuring Theils U1: rwd - \", theils_u1(actual,rwd))\n",
        "\n",
        "print(\"Evaluation_metrics:  ar - \", evaluate(actual,ar))\n",
        "print(\"Measuring Theils U1: ar - \", theils_u1(actual,ar))\n",
        "\n",
        "print(\"Evaluation_metrics:  msgarch - \", evaluate(actual,msgarch))\n",
        "print(\"Measuring Theils U1: msgarch - \", theils_u1(actual,msgarch))\n",
        "\n",
        "print(\"Evaluation_metrics:  deepar - \", evaluate(actual,deepar))\n",
        "print(\"Measuring Theils U1: deepar - \", theils_u1(actual,deepar))\n",
        "\n",
        "print(\"Evaluation_metrics:  arnnx - \", evaluate(actual,arnnx))\n",
        "print(\"Measuring Theils U1: arnnx - \", theils_u1(actual,arnnx))\n",
        "\n",
        "print(\"Evaluation_metrics:  nbeats - \", evaluate(actual,nbeats))\n",
        "print(\"Measuring Theils U1: nbeats - \", theils_u1(actual,nbeats))\n",
        "\n",
        "print(\"Evaluation_metrics:  arfimax - \", evaluate(actual,arfimax))\n",
        "print(\"Measuring Theils U1: arfimax - \", theils_u1(actual,arfimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  sarimax - \", evaluate(actual,sarimax))\n",
        "print(\"Measuring Theils U1: sarimax - \", theils_u1(actual,sarimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  arimax - \", evaluate(actual,arimax))\n",
        "print(\"Measuring Theils U1: arimax - \", theils_u1(actual,arimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  lr - \", evaluate(actual,lr))\n",
        "print(\"Measuring Theils U1: lr - \", theils_u1(actual,lr))\n",
        "\n",
        "print(\"Evaluation_metrics:  ardl - \", evaluate(actual,ardl))\n",
        "print(\"Measuring Theils U1: ardl - \", theils_u1(actual,ardl))\n",
        "\n",
        "print(\"Evaluation_metrics:  xgboost - \", evaluate(actual,xgboost))\n",
        "print(\"Measuring Theils U1: xgboost - \", theils_u1(actual,xgboost))\n",
        "\n",
        "print(\"Evaluation_metrics:  tft - \", evaluate(actual,tft))\n",
        "print(\"Measuring Theils U1: tft - \", theils_u1(actual,tft))\n",
        "\n",
        "print(\"Evaluation_metrics:  warimax - \", evaluate(actual,warimax))\n",
        "print(\"Measuring Theils U1: warimax - \", theils_u1(actual,warimax))\n",
        "\n",
        "print(\"Evaluation_metrics:  fewnet - \", evaluate(actual,fewnet))\n",
        "print(\"Measuring Theils U1: fewnet - \", theils_u1(actual,fewnet))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZxDS3D3Z_Z2V",
        "outputId": "af1e451a-6e23-48dc-ffb1-b727f63ef819"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation_metrics:  rw -  {'rmse': 3.1439407682643274, 'mase': 3.9448257634708765, 'smape': 1.012927799354132, 'mdrae': 3.863152898425796, 'mdape': 1.6745157091255096}\n",
            "Measuring Theils U1: rw -  0.4501993507474623\n",
            "Evaluation_metrics:  rwd -  {'rmse': 3.2344600630972637, 'mase': 4.063222997192853, 'smape': 1.0230097447883648, 'mdrae': 4.0713738172202, 'mdape': 1.7312306208491315}\n",
            "Measuring Theils U1: rwd -  0.4574141824064194\n",
            "Evaluation_metrics:  ar -  {'rmse': 2.2342148258440044, 'mase': 2.740339054071565, 'smape': 0.8868119477258164, 'mdrae': 2.6061710518956827, 'mdape': 1.1386035803133057}\n",
            "Measuring Theils U1: ar -  0.36121548118524366\n",
            "Evaluation_metrics:  msgarch -  {'rmse': 2.077109928808413, 'mase': 2.634164752194101, 'smape': 0.8912531877882289, 'mdrae': 2.4191127707752473, 'mdape': 0.9464289840211033}\n",
            "Measuring Theils U1: msgarch -  0.3514165247055698\n",
            "Evaluation_metrics:  deepar -  {'rmse': 1.7126214094007532, 'mase': 1.9496646614711375, 'smape': 0.8142596730977306, 'mdrae': 1.5294793300361682, 'mdape': 0.5706553869612156}\n",
            "Measuring Theils U1: deepar -  0.3973477652772873\n",
            "Evaluation_metrics:  arnnx -  {'rmse': 1.847790515684743, 'mase': 2.1474112171661512, 'smape': 0.7718702953723368, 'mdrae': 2.32333272296037, 'mdape': 0.8472781150615327}\n",
            "Measuring Theils U1: arnnx -  0.3374683930576507\n",
            "Evaluation_metrics:  nbeats -  {'rmse': 1.4271789607839969, 'mase': 1.7477422822327429, 'smape': 0.7738927789737485, 'mdrae': 1.5968369826729163, 'mdape': 0.6324210516510618}\n",
            "Measuring Theils U1: nbeats -  0.2699220012065413\n",
            "Evaluation_metrics:  arfimax -  {'rmse': 18.373736244712756, 'mase': 25.724530374085045, 'smape': 1.6778555053358166, 'mdrae': 26.01966360016198, 'mdape': 11.558931815374017}\n",
            "Measuring Theils U1: arfimax -  0.8121549484629536\n",
            "Evaluation_metrics:  sarimax -  {'rmse': 3.4789244004104423, 'mase': 4.230986426415143, 'smape': 1.019706707171257, 'mdrae': 4.433696305372707, 'mdape': 2.0538747861203457}\n",
            "Measuring Theils U1: sarimax -  0.47969379675574864\n",
            "Evaluation_metrics:  arimax -  {'rmse': 4.040801330854788, 'mase': 5.052556319260356, 'smape': 1.0890851230469272, 'mdrae': 4.98956055997049, 'mdape': 2.4130140155996367}\n",
            "Measuring Theils U1: arimax -  0.5094428006456725\n",
            "Evaluation_metrics:  lr -  {'rmse': 1.5911493767675686, 'mase': 1.888802668978841, 'smape': 0.7478045753025656, 'mdrae': 2.0930929000407765, 'mdape': 0.5472143055632555}\n",
            "Measuring Theils U1: lr -  0.304251494624564\n",
            "Evaluation_metrics:  ardl -  {'rmse': 1.9954808596604365, 'mase': 2.326379279965454, 'smape': 0.7938226289328879, 'mdrae': 2.508307725849224, 'mdape': 0.9918890656959155}\n",
            "Measuring Theils U1: ardl -  0.3448287923280366\n",
            "Evaluation_metrics:  xgboost -  {'rmse': 2.8686858878711616, 'mase': 3.123625961622281, 'smape': 1.151606740581495, 'mdrae': 2.6717714235769336, 'mdape': 1.1374373227947583}\n",
            "Measuring Theils U1: xgboost -  0.6452915067392212\n",
            "Evaluation_metrics:  tft -  {'rmse': 2.053870280313362, 'mase': 2.420584571035997, 'smape': 1.0026740870262085, 'mdrae': 2.3095507102092485, 'mdape': 0.8372500002565613}\n",
            "Measuring Theils U1: tft -  0.45893787362890215\n",
            "Evaluation_metrics:  warimax -  {'rmse': 7.402693953218731, 'mase': 10.165844388194664, 'smape': 1.3865288071448763, 'mdrae': 10.873570267081003, 'mdape': 4.839518269905807}\n",
            "Measuring Theils U1: warimax -  0.6387824328452459\n",
            "Evaluation_metrics:  fewnet -  {'rmse': 2.009707674291087, 'mase': 2.3689195232667446, 'smape': 0.8179678670901619, 'mdrae': 2.4987164571243077, 'mdape': 0.8370135961149536}\n",
            "Measuring Theils U1: fewnet -  0.3566292141790691\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "B7vJvlGTXBR8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}