Added forecast module to compare

config.yaml

@@ -15,18 +15,18 @@ model:
   ]
   time_varying_known_reals: [ "time_idx" ]
   batch_size: 512
-  max_epochs: 100
+  max_epochs: 300
   gpus: 1
   learning_rate: 0.0001
-  hidden_size: 50
+  hidden_size: 100
   dropout: 0.2
-  hidden_continuous_size: 25
+  hidden_continuous_size: 50
   gradient_clip_val: 0.3
   max_prediction_length: 24
   max_encoder_length: 48
   sample: "60min"
   cutoff: 0.70
-  model_path: "/content/model/tft.pt"
+  model_path: "/content/drive/MyDrive/Colab_Notebooks/model/tft.pt"
 
 model_local:
   lags: None
@@ -42,7 +42,7 @@ model_local:
   batch_size: 1024
   max_epochs: 4
   gpus: 0
-  learning_rate: 0.0001
+  learning_rate: 0.01
   hidden_size: 8
   dropout: 0.2
   hidden_continuous_size: 4
@@ -51,4 +51,4 @@ model_local:
   max_encoder_length: 48
   sample: "60min"
   cutoff: 0.70
-  model_path: "model/tft_regressor_test.pt"
+  model_path: "model/temporal_fusion_transformer/tft_local.pt"

tft_predict_colab.py → forecast_tft.py

@@ -1,58 +1,84 @@
 from os import listdir
 from os.path import isfile, join
 import warnings
+import argparse
 
 import pandas as pd
 import matplotlib.pyplot as plt
 
-import torch
-from pytorch_forecasting.metrics import SMAPE, PoissonLoss, QuantileLoss
-from pytorch_forecasting import Baseline, TemporalFusionTransformer, TimeSeriesDataSet
-from pytorch_forecasting.data import GroupNormalizer
-
 from config import load_config
 from load_data import LoadData
 
+import torch
+from pytorch_forecasting import TemporalFusionTransformer, TimeSeriesDataSet
+from pytorch_forecasting.data import GroupNormalizer
+from pytorch_forecasting.metrics import QuantileLoss
+
 warnings.filterwarnings("ignore")
 
-spec = load_config("config.yaml")
-DATA_PATH = spec["general"]["data_path"]
+parser = argparse.ArgumentParser(description="Run Training Pipeline")
+parser.add_argument(
+    "-l",
+    "--local",
+    help="Run local or in colab",
+    action='store_true',
+    default=False,
+)
+args = parser.parse_args()
+
+LOCAL = args.local
+
+if LOCAL:
+    model_key = "model_local"
+    spec = load_config("config.yaml")
+    DATA_PATH = spec["general"]["data_path"]
+    MODEL_PATH = spec[model_key]["model_path"]
+else:
+    model_key = "model"
+    import tensorflow as tf
+    import tensorboard as tb
+    tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
+    spec = load_config("/content/temporal-fusion-transformer/config.yaml")
+    DATA_PATH = "/content/temporal-fusion-transformer/" + spec["general"]["data_path"]
+    MODEL_PATH = "model/temporal_fusion_transformer/tft.pt"
+
 FOLDER_LIST = spec["general"]["folder_list"]
-MODEL_PATH = spec["model"]["model_path"]
-BATCH_SIZE = spec["model"]["batch_size"]
-MAX_EPOCHS = spec["model"]["max_epochs"]
-GPUS = spec["model"]["gpus"]
-LEARNING_RATE = spec["model"]["learning_rate"]
-HIDDEN_SIZE = spec["model"]["hidden_size"]
-DROPOUT = spec["model"]["dropout"]
-HIDDEN_CONTINUOUS_SIZE = spec["model"]["hidden_continuous_size"]
-GRADIENT_CLIP_VAL = spec["model"]["gradient_clip_val"]
-
-lags = spec["model_local"]["lags"]
-sma = spec["model_local"]["sma"]
+BATCH_SIZE = spec[model_key]["batch_size"]
+MAX_EPOCHS = spec[model_key]["max_epochs"]
+GPUS = spec[model_key]["gpus"]
+LEARNING_RATE = spec[model_key]["learning_rate"]
+HIDDEN_SIZE = spec[model_key]["hidden_size"]
+DROPOUT = spec[model_key]["dropout"]
+HIDDEN_CONTINUOUS_SIZE = spec[model_key]["hidden_continuous_size"]
+GRADIENT_CLIP_VAL = spec[model_key]["gradient_clip_val"]
+
+lags = spec[model_key]["lags"]
+sma = spec[model_key]["sma"]
 sma_columns = [f"sma_{sma}" for sma in sma]
 
 if lags != "None":
     lags_columns = [f"(t-{lag})" for lag in range(lags, 0, -1)]
 
     time_varying_known_reals = (
-            spec["model_local"]["time_varying_known_reals"] +
+            spec[model_key]["time_varying_known_reals"] +
             lags_columns +
             sma_columns
     )
 if lags == "None":
     lags = None
     time_varying_known_reals = (
-            spec["model_local"]["time_varying_known_reals"] +
+            spec[model_key]["time_varying_known_reals"] +
             sma_columns
     )
 
-time_varying_known_categoricals = spec["model"]["time_varying_known_categoricals"]
+else:
+    time_varying_known_reals = spec[model_key]["time_varying_known_reals"]
 
-max_prediction_length = spec["model"]["max_prediction_length"]
-max_encoder_length = spec["model"]["max_encoder_length"]
-sample = spec["model"]["sample"]
-cutoff = spec["model"]["cutoff"]
+time_varying_known_categoricals = spec[model_key]["time_varying_known_categoricals"]
+max_prediction_length = spec[model_key]["max_prediction_length"]
+max_encoder_length = spec[model_key]["max_encoder_length"]
+sample = spec[model_key]["sample"]
+cutoff = spec[model_key]["cutoff"]
 
 train_data, test_data = LoadData(
     data_path=DATA_PATH,
@@ -67,7 +93,7 @@
     train_data,
     time_idx="time_idx",
     target="value",
-    group_ids=["dataset", "id"],
+    group_ids=["id"],
     min_encoder_length=max_encoder_length // 2,
     max_encoder_length=max_encoder_length,
     min_prediction_length=1,
@@ -78,12 +104,12 @@
     time_varying_unknown_categoricals=[],
     time_varying_unknown_reals=["value"],
     target_normalizer=GroupNormalizer(
-        groups=["dataset", "id"], transformation="softplus"
+        groups=["id"], transformation="softplus"
     ),
     add_relative_time_idx=True,
     add_target_scales=True,
     add_encoder_length=True,
-    allow_missing_timesteps=True,
+
 )
 
 model = TemporalFusionTransformer.from_dataset(
@@ -97,13 +123,12 @@
     loss=QuantileLoss(),
     log_interval=10,
     reduce_on_plateau_patience=4,
-)
+    )
 
-model.load_state_dict(torch.load("model/tft_regressor_test.pt"))
+model.load_state_dict(torch.load(MODEL_PATH))
 
 for folder in FOLDER_LIST:
 
-    print(folder)
     folder_path = f"{DATA_PATH}/{folder}"
     file_list = [
         f for f in listdir(folder_path) if isfile(join(folder_path, f))
@@ -119,7 +144,6 @@
 
         for start in range(0, 80, 1):
 
-            # start = 0
             test_data_df = df[start:(start + max_encoder_length)]
             y_obs = df[(start + max_encoder_length): (start + max_encoder_length + max_prediction_length)]
 

tft_pipeline_colab.py → initial_tests/tft_pipeline_colab.py

@@ -53,7 +53,6 @@
     time_varying_known_reals = (spec["model"]["time_varying_known_reals"])
 
 time_varying_known_categoricals = spec["model"]["time_varying_known_categoricals"]
-
 max_prediction_length = spec["model"]["max_prediction_length"]
 max_encoder_length = spec["model"]["max_encoder_length"]
 sample = spec["model"]["sample"]
@@ -87,11 +86,9 @@
         target_normalizer=GroupNormalizer(
             groups=["id"], transformation="softplus"
         ),
-        lags={"sma_12": [1, 24, 48]},
         add_relative_time_idx=True,
         add_target_scales=True,
         add_encoder_length=True,
-
     )
     # create validation set (predict=True) which means to predict the
     # last max_prediction_length points in time for each series

tft_predict_local.py → initial_tests/tft_predict_local.py

@@ -83,7 +83,6 @@
     add_relative_time_idx=True,
     add_target_scales=True,
     add_encoder_length=True,
-    allow_missing_timesteps=True,
 )
 
 model = TemporalFusionTransformer.from_dataset(

n_beats.py → train_nbeats.py

@@ -58,7 +58,7 @@
         time_idx="time_idx",
         target="value",
         # categorical_encoders={"id": NaNLabelEncoder().fit(train_data.id)},
-        group_ids=["id"],
+        group_ids=["dataset", "id"],
         time_varying_unknown_reals=["value"],
         max_encoder_length=max_encoder_length,
         max_prediction_length=max_prediction_length,
@@ -70,7 +70,7 @@
     validation = TimeSeriesDataSet.from_dataset(training, train_data, min_prediction_idx=training_cutoff + 1)
 
     # create dataloaders for model
-    batch_size = 512
+    batch_size = 1024
     train_dataloader = training.to_dataloader(train=True, batch_size=batch_size, num_workers=0)
     val_dataloader = validation.to_dataloader(train=False, batch_size=batch_size, num_workers=0)
 
@@ -79,7 +79,7 @@
 
     early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=10, verbose=False, mode="min")
     trainer = pl.Trainer(
-        max_epochs=5,
+        max_epochs=3,
         gpus=0,
         weights_summary="top",
         gradient_clip_val=0.01,

tft_pipeline_local.py → train_tft.py

@@ -1,4 +1,5 @@
 import warnings
+import argparse
 
 from config import load_config
 from load_data import LoadData
@@ -8,24 +9,36 @@
 from pytorch_lightning.loggers import TensorBoardLogger
 import torch
 
-from pytorch_forecasting import Baseline, TemporalFusionTransformer, TimeSeriesDataSet
+from pytorch_forecasting import TemporalFusionTransformer, TimeSeriesDataSet
 from pytorch_forecasting.data import GroupNormalizer
 from pytorch_forecasting.metrics import QuantileLoss
 
-LOCAL = True
+warnings.filterwarnings("ignore")
+
+parser = argparse.ArgumentParser(description="Run Training Pipeline")
+parser.add_argument(
+    "-l",
+    "--local",
+    help="Run local or in colab",
+    action='store_true',
+    default=False,
+)
+args = parser.parse_args()
+
+LOCAL = args.local
 
 if LOCAL:
     model_key = "model_local"
+    spec = load_config("config.yaml")
+    DATA_PATH = spec["general"]["data_path"]
 else:
     model_key = "model"
     import tensorflow as tf
     import tensorboard as tb
     tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
+    spec = load_config("/content/temporal-fusion-transformer/config.yaml")
+    DATA_PATH = "/content/temporal-fusion-transformer/" + spec["general"]["data_path"]
 
-warnings.filterwarnings("ignore")
-
-spec = load_config("config.yaml")
-DATA_PATH = spec["general"]["data_path"]
 FOLDER_LIST = spec["general"]["folder_list"]
 MODEL_PATH = spec[model_key]["model_path"]
 BATCH_SIZE = spec[model_key]["batch_size"]
@@ -56,8 +69,10 @@
             sma_columns
     )
 
-time_varying_known_categoricals = spec[model_key]["time_varying_known_categoricals"]
+else:
+    time_varying_known_reals = spec[model_key]["time_varying_known_reals"]
 
+time_varying_known_categoricals = spec[model_key]["time_varying_known_categoricals"]
 max_prediction_length = spec[model_key]["max_prediction_length"]
 max_encoder_length = spec[model_key]["max_encoder_length"]
 sample = spec[model_key]["sample"]
@@ -78,7 +93,7 @@
         train_data,
         time_idx="time_idx",
         target="value",
-        group_ids=["dataset", "id"],
+        group_ids=["id"],
         min_encoder_length=max_encoder_length // 2,
         max_encoder_length=max_encoder_length,
         min_prediction_length=1,
@@ -89,14 +104,16 @@
         time_varying_unknown_categoricals=[],
         time_varying_unknown_reals=["value"],
         target_normalizer=GroupNormalizer(
-            groups=["dataset", "id"], transformation="softplus"
+            groups=["id"], transformation="softplus"
         ),
         add_relative_time_idx=True,
         add_target_scales=True,
         add_encoder_length=True,
 
     )
 
+    pl.seed_everything(42)
+
     # create validation set (predict=True) which means to predict the
     # last max_prediction_length points in time for each series
     validation = TimeSeriesDataSet.from_dataset(
@@ -117,15 +134,6 @@
         num_workers=0
     )
 
-    # calculate baseline mean absolute error, i.e. predict next value as the
-    # last available value from the history
-    actuals = torch.cat([y for x, (y, weight) in iter(val_dataloader)])
-    baseline_predictions = Baseline().predict(val_dataloader)
-    (actuals - baseline_predictions).abs().mean().item()
-
-    # configure network and trainer
-    pl.seed_everything(42)
-
     # configure network and trainer
     early_stop_callback = EarlyStopping(
         monitor="val_loss",
@@ -134,6 +142,7 @@
         verbose=False,
         mode="min"
     )
+
     lr_logger = LearningRateMonitor()
     logger = TensorBoardLogger("lightning_logs")
 
@@ -155,7 +164,7 @@
         attention_head_size=1,
         dropout=DROPOUT,
         hidden_continuous_size=HIDDEN_CONTINUOUS_SIZE,
-        output_size=7,  # 7 quantiles by default
+        output_size=7,
         loss=QuantileLoss(),
         log_interval=10,
         reduce_on_plateau_patience=4,
@@ -168,4 +177,4 @@
         val_dataloaders=val_dataloader,
     )
 
-    torch.save(tft.state_dict(), MODEL_PATH)
+    torch.save(tft.state_dict(), MODEL_PATH)