Add PF and multi-scale RoPE

README.md

@@ -165,7 +165,7 @@ The validation set will be saved as DATASET_NAME_eval.
 python -m uni2ts.data.builder.simple ETTh1 dataset/ETT-small/ETTh1.csv --date_offset '2017-10-23 23:00:00'
 ```
 
-3. Finally, we can simply run the fine-tuning script with the appropriate [training](cli/conf/origin/finetune/data/etth1.yaml) and [validation](cli/conf/origin/finetune/val_data/etth1.yaml) data configuration files.
+3. Finally, we can simply run the fine-tuning script with the appropriate [training](cli/conf/lsf-setup/origin/finetune/data/etth1.yaml) and [validation](cli/conf/lsf-setup/origin/finetune/val_data/etth1.yaml) data configuration files.
 ```shell
 python -m cli.finetune \
   -cp conf/lsf \
@@ -177,7 +177,7 @@ python -m cli.finetune \
 
 ### Evaluation
 
-The evaluation script can be used to calculate evaluation metrics such as MSE, MASE, CRPS, and so on (see the [configuration file](cli/conf/origin/eval/default.yaml)). 
+The evaluation script can be used to calculate evaluation metrics such as MSE, MASE, CRPS, and so on (see the [configuration file](cli/conf/lsf-setup/origin/eval/default.yaml)). 
 
 Given a test split (see previous section on processing datasets), we can run the following command to evaluate it:
 ```shell
@@ -189,15 +189,15 @@ python -m cli.eval \
   data=etth1_test
 ```
 
-Alternatively, we provide access to popular datasets, and can be toggled via the [data configurations](cli/conf/origin/eval/data).
+Alternatively, we provide access to popular datasets, and can be toggled via the [data configurations](cli/conf/lsf-setup/origin/eval/data).
 As an example, say we want to perform evaluation, again on the ETTh1 dataset from the popular [Long Sequence Forecasting benchmark](https://github.com/thuml/Time-Series-Library).
 We first need to download the pre-processed datasets and put them in the correct directory, by setting up the TSLib repository and following the instructions.
 Then, assign the dataset directory to the `LSF_PATH` environment variable:
 ```shell
 echo "LSF_PATH=PATH_TO_TSLIB/dataset" >> .env
 ```
 
-Thereafter, simply run the following script with the predefined [Hydra config file](cli/conf/origin/eval/data/lsf_test.yaml):
+Thereafter, simply run the following script with the predefined [Hydra config file](cli/conf/lsf-setup/origin/eval/data/lsf_test.yaml):
 ```shell
 python -m cli.eval \ 
   run_name=example_eval_2 \

cli/conf/lsf/eval/default.yaml → cli/conf/lsf-setup/lsf/eval/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/lsf/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+    dir: outputs/lsf-setup/lsf/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
 defaults:
   - model: ???
   - data: ???

cli/conf/lsf/finetune/default.yaml → cli/conf/lsf-setup/lsf/finetune/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/lsf/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
+    dir: outputs/lsf-setup/lsf/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
 defaults:
   - model: ???
   - data: ???

cli/conf/origin/eval/default.yaml → cli/conf/lsf-setup/lsf_point/eval/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/origin/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+    dir: outputs/lsf-setup/lsf_point/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
 defaults:
   - model: ???
   - data: ???

cli/conf/lsf_point/finetune/default.yaml → cli/conf/lsf-setup/lsf_point/finetune/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/lsf_point/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
+    dir: outputs/lsf-setup/lsf_point/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
 defaults:
   - model: ???
   - data: ???

cli/conf/lsf-setup/multi_scale/eval/default.yaml

@@ -0,0 +1,24 @@
+hydra:
+  run:
+    dir: outputs/lsf-setup/multi_scale/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+defaults:
+  - model: ???
+  - data: ???
+  - _self_
+exp_name: ???
+metrics:
+  - _target_: gluonts.ev.metrics.MSE
+  - _target_: uni2ts.eval_util.metrics.MedianMSE
+  - _target_: gluonts.ev.metrics.MAE
+  - _target_: gluonts.ev.metrics.MASE
+  - _target_: gluonts.ev.metrics.MAPE
+  - _target_: gluonts.ev.metrics.SMAPE
+  - _target_: gluonts.ev.metrics.MSIS
+  - _target_: gluonts.ev.metrics.RMSE
+  - _target_: gluonts.ev.metrics.NRMSE
+  - _target_: gluonts.ev.metrics.ND
+  - _target_: gluonts.ev.metrics.MeanWeightedSumQuantileLoss
+    quantile_levels: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+batch_size: 512  # 16
+min_batch_size: 16
+device: auto

cli/conf/multi_scale/finetune/default.yaml → cli/conf/lsf-setup/multi_scale/finetune/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/multi_scale/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
+    dir: outputs/lsf-setup/multi_scale/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
 defaults:
   - model: ???
   - data: ???

cli/conf/lsf_point/eval/default.yaml → cli/conf/lsf-setup/origin/eval/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/lsf_point/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+    dir: outputs/lsf-setup/origin/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
 defaults:
   - model: ???
   - data: ???

cli/conf/lsf-setup/origin/finetune/default.yaml

@@ -0,0 +1,86 @@
+hydra:
+  run:
+    dir: outputs/lsf-setup/origin/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
+defaults:
+  - model: ???
+  - data: ???
+  - val_data: null
+  - _self_
+exp_name: ???
+run_name: ???
+seed: 0
+tf32: true
+compile: false  # set to mode: default, reduce-overhead, max-autotune
+ckpt_path: null
+trainer:
+  _target_: lightning.Trainer
+  accelerator: auto
+  strategy: auto
+  devices: auto
+  num_nodes: 1
+  precision: 32
+  logger:
+      _target_: lightning.pytorch.loggers.TensorBoardLogger
+      save_dir: ${hydra:runtime.output_dir}
+      name: logs
+  callbacks:
+    - _target_: lightning.pytorch.callbacks.LearningRateMonitor
+      logging_interval: epoch
+    - _target_: lightning.pytorch.callbacks.ModelCheckpoint
+      dirpath: ${hydra:runtime.output_dir}/checkpoints
+      monitor: val/PackedNLLLoss
+      save_weights_only: true
+      mode: min
+      save_top_k: 2  # Qz: Sometimes the 1st validation gets anomalous results. Discard that ckpt, and use the 2nd one.
+      every_n_epochs: 1
+    - _target_: uni2ts.callbacks.earlystop.WarmupEarlyStopping  # lightning.pytorch.callbacks.EarlyStopping
+      monitor: val/PackedNLLLoss
+      min_delta: 0.0
+      patience: 30
+      mode: min
+      strict: false
+      verbose: true
+      warmup_steps: 1
+  max_epochs: 1000
+  enable_progress_bar: true
+  accumulate_grad_batches: 1
+  gradient_clip_val: 1.0
+  gradient_clip_algorithm: norm
+train_dataloader:
+  _target_: uni2ts.data.loader.DataLoader
+  batch_size: 32
+  batch_size_factor: 2.0
+  cycle: true
+  num_batches_per_epoch: 10
+  shuffle: true
+  num_workers: 11
+  collate_fn:
+    _target_: uni2ts.data.loader.PackCollate
+    max_length: ${model.module_kwargs.max_seq_len}
+    seq_fields: ${cls_getattr:${model._target_},seq_fields}
+    pad_func_map: ${cls_getattr:${model._target_},pad_func_map}
+  pin_memory: true
+  drop_last: false
+  fill_last: false
+  worker_init_fn: null
+  prefetch_factor: 2
+  persistent_workers: true
+val_dataloader:
+  _target_: uni2ts.data.loader.DataLoader
+  batch_size: 32
+  batch_size_factor: 2.0
+  cycle: false
+  num_batches_per_epoch: null
+  shuffle: false
+  num_workers: 11
+  collate_fn:
+    _target_: uni2ts.data.loader.PackCollate
+    max_length: ${model.module_kwargs.max_seq_len}
+    seq_fields: ${cls_getattr:${model._target_},seq_fields}
+    pad_func_map: ${cls_getattr:${model._target_},pad_func_map}
+  pin_memory: false
+  drop_last: false
+  fill_last: true
+  worker_init_fn: null
+  prefetch_factor: 2
+  persistent_workers: true

cli/conf/lsf-setup/seasonal_naive/eval/default.yaml

@@ -0,0 +1,24 @@
+hydra:
+  run:
+    dir: outputs/lsf-setup/seasonal_naive/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+defaults:
+  - model: ???
+  - data: ???
+  - _self_
+exp_name: ???
+metrics:
+  - _target_: gluonts.ev.metrics.MSE
+  - _target_: uni2ts.eval_util.metrics.MedianMSE
+  - _target_: gluonts.ev.metrics.MAE
+  - _target_: gluonts.ev.metrics.MASE
+  - _target_: gluonts.ev.metrics.MAPE
+  - _target_: gluonts.ev.metrics.SMAPE
+  - _target_: gluonts.ev.metrics.MSIS
+  - _target_: gluonts.ev.metrics.RMSE
+  - _target_: gluonts.ev.metrics.NRMSE
+  - _target_: gluonts.ev.metrics.ND
+  - _target_: gluonts.ev.metrics.MeanWeightedSumQuantileLoss
+    quantile_levels: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+batch_size: 512
+min_batch_size: 16
+device: auto

cli/conf/seasonal_naive/finetune/default.yaml → cli/conf/lsf-setup/seasonal_naive/finetune/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/seasonal_naive/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
+    dir: outputs/lsf-setup/seasonal_naive/finetune/${hydra:runtime.choices.model}/${exp_name}/${model.finetune_pattern}/${hydra:runtime.choices.data}/${run_name}
 defaults:
   - model: ???
   - data: ???

cli/conf/pf-setup/origin/eval/data/gluonts_test.yaml

@@ -0,0 +1,4 @@
+_target_: uni2ts.eval_util.data.get_gluonts_test_dataset
+dataset_name: ???
+prediction_length: null
+mode: S

cli/conf/pf-setup/origin/eval/data/gluonts_val.yaml

@@ -0,0 +1,4 @@
+_target_: uni2ts.eval_util.data.get_gluonts_val_dataset
+dataset_name: ???
+prediction_length: null
+mode: S

cli/conf/seasonal_naive/eval/default.yaml → cli/conf/pf-setup/origin/eval/default.yaml

@@ -1,6 +1,6 @@
 hydra:
   run:
-    dir: outputs/seasonal_naive/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
+    dir: outputs/pf-setup/origin/eval/${hydra:runtime.choices.model}/${exp_name}/${data.dataset_name}/${data.mode}/cl${model.context_length}_pl${data.prediction_length}
 defaults:
   - model: ???
   - data: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.0_R_base.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.0_R_large.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-large
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.0_R_small.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-small
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.1_R_base.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.1-R-base
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.1_R_large.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.1-R-large
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_1.1_R_small.yaml

@@ -0,0 +1,7 @@
+_target_: uni2ts.model.moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.1-R-small
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/eval/model/moirai_lightning_ckpt.yaml

@@ -0,0 +1,6 @@
+_target_: uni2ts.model.moirai.MoiraiForecast.load_from_checkpoint
+checkpoint_path: ...
+pretrained_checkpoint_path: null
+num_samples: 100
+patch_size: ???
+context_length: ???

cli/conf/pf-setup/origin/finetune/data/electricity.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: electricity

cli/conf/pf-setup/origin/finetune/data/etth1.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: ETTh1

cli/conf/pf-setup/origin/finetune/data/etth2.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: ETTh2

cli/conf/pf-setup/origin/finetune/data/ettm1.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: ETTm1

cli/conf/pf-setup/origin/finetune/data/ettm2.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: ETTm2

cli/conf/pf-setup/origin/finetune/data/weather.yaml

@@ -0,0 +1,2 @@
+_target_: uni2ts.data.builder.simple.SimpleDatasetBuilder
+dataset: weather

cli/conf/pf-setup/origin/finetune/model/moirai_1.0_R_base.yaml

@@ -0,0 +1,33 @@
+# load a pretrained checkpoint from huggingface hub
+_target_: uni2ts.model.moirai.MoiraiFinetune
+module:
+  _target_: uni2ts.model.moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base
+module_kwargs:
+  _target_: builtins.dict
+  distr_output:
+    _target_: uni2ts.distribution.MixtureOutput
+    components:
+      - _target_: uni2ts.distribution.StudentTOutput
+      - _target_: uni2ts.distribution.NormalFixedScaleOutput
+      - _target_: uni2ts.distribution.NegativeBinomialOutput
+      - _target_: uni2ts.distribution.LogNormalOutput
+  d_model: 768
+  num_layers: 12
+  patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
+  max_seq_len: 512
+  attn_dropout_p: 0.0
+  dropout_p: 0.0
+  scaling: true
+min_patches: 2
+min_mask_ratio: 0.15
+max_mask_ratio: 0.5
+max_dim: 128
+loss_func:
+  _target_: uni2ts.loss.packed.PackedNLLLoss
+lr: 1e-3
+weight_decay: 1e-1
+beta1: 0.9
+beta2: 0.98
+num_training_steps: ${mul:${trainer.max_epochs},${train_dataloader.num_batches_per_epoch}}
+num_warmup_steps: 0