moirai 1.0 config

zqiao11 · Nov 18, 2024 · 3bca23c · 3bca23c
1 parent dcbbefa
commit 3bca23c
Show file tree

Hide file tree

Showing 68 changed files with 536 additions and 324 deletions.
diff --git a/cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_small.yaml b/cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_small.yaml
@@ -0,0 +1,41 @@
+# load a pretrained checkpoint from huggingface hub
+_target_: uni2ts.model.lsf_moirai.MoiraiFinetune
+module:
+  _target_: uni2ts.model.lsf_moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-small
+module_kwargs:
+  _target_: builtins.dict
+  distr_output:
+    _target_: uni2ts.distribution.MixtureOutput
+    components:
+      - _target_: uni2ts.distribution.StudentTOutput
+      - _target_: uni2ts.distribution.NormalFixedScaleOutput
+      - _target_: uni2ts.distribution.NegativeBinomialOutput
+      - _target_: uni2ts.distribution.LogNormalOutput
+  d_model: 384
+  num_layers: 6
+  patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
+  max_seq_len: 512
+  attn_dropout_p: 0.0
+  dropout_p: 0.0
+  scaling: true
+min_patches: 2
+min_mask_ratio: 0.15
+max_mask_ratio: 0.5
+max_dim: 128
+loss_func:
+  _target_: uni2ts.loss.packed.PackedNLLLoss
+val_metric:
+  - _target_: uni2ts.loss.packed.PackedMSELoss
+  - _target_: uni2ts.loss.packed.PackedNRMSELoss
+    normalize: absolute_target_squared
+lr: 5e-7  #  On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs
+weight_decay: 1e-1
+beta1: 0.9
+beta2: 0.98
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full
diff --git a/cli/conf/lsf-setup/multi_scale/eval/model/moirai_1.0_R_small.yaml b/cli/conf/lsf-setup/multi_scale/eval/model/moirai_1.0_R_small.yaml
@@ -0,0 +1,8 @@
+_target_: uni2ts.model.multi_scale_moirai.MoiraiForecast
+module:
+  _target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-small
+num_samples: 100
+patch_size: ???
+context_length: ???
+num_new_scales: 2
diff --git a/cli/conf/lsf-setup/multi_scale/finetune/model/moirai_1.0_R_small.yaml b/cli/conf/lsf-setup/multi_scale/finetune/model/moirai_1.0_R_small.yaml
@@ -0,0 +1,43 @@
+# load a pretrained checkpoint from huggingface hub
+_target_: uni2ts.model.multi_scale_moirai.MoiraiFinetune
+module:
+  _target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-small
+module_kwargs:
+  _target_: builtins.dict
+  distr_output:
+    _target_: uni2ts.distribution.MixtureOutput
+    components:
+      - _target_: uni2ts.distribution.StudentTOutput
+      - _target_: uni2ts.distribution.NormalFixedScaleOutput
+      - _target_: uni2ts.distribution.NegativeBinomialOutput
+      - _target_: uni2ts.distribution.LogNormalOutput
+  d_model: 384
+  num_layers: 6
+  patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
+  max_seq_len: 512
+  attn_dropout_p: 0.0
+  dropout_p: 0.0
+  scaling: true
+min_patches: 2
+min_mask_ratio: 0.15
+max_mask_ratio: 0.5
+max_dim: 128
+loss_func:
+  _target_: uni2ts.loss.packed.PackedNLLLoss
+val_metric:
+  - _target_: uni2ts.loss.packed.PackedMSELoss
+  - _target_: uni2ts.loss.packed.PackedNRMSELoss
+    normalize: absolute_target_squared
+lr: 5e-7  #  On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs
+weight_decay: 1e-1
+beta1: 0.9
+beta2: 0.98
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full
+num_new_scales: 3
+ds_factor: 2
diff --git a/cli/conf/pf-setup/pf/finetune/model/moirai_1.0_R_small.yaml b/cli/conf/pf-setup/pf/finetune/model/moirai_1.0_R_small.yaml
@@ -0,0 +1,41 @@
+# load a pretrained checkpoint from huggingface hub
+_target_: uni2ts.model.lsf_moirai.MoiraiFinetune
+module:
+  _target_: uni2ts.model.lsf_moirai.MoiraiModule.from_pretrained
+  pretrained_model_name_or_path: Salesforce/moirai-1.0-R-small
+module_kwargs:
+  _target_: builtins.dict
+  distr_output:
+    _target_: uni2ts.distribution.MixtureOutput
+    components:
+      - _target_: uni2ts.distribution.StudentTOutput
+      - _target_: uni2ts.distribution.NormalFixedScaleOutput
+      - _target_: uni2ts.distribution.NegativeBinomialOutput
+      - _target_: uni2ts.distribution.LogNormalOutput
+  d_model: 384
+  num_layers: 6
+  patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
+  max_seq_len: 512
+  attn_dropout_p: 0.0
+  dropout_p: 0.0
+  scaling: true
+min_patches: 2
+min_mask_ratio: 0.15
+max_mask_ratio: 0.5
+max_dim: 128
+loss_func:
+  _target_: uni2ts.loss.packed.PackedNLLLoss
+val_metric:
+  - _target_: uni2ts.loss.packed.PackedMSELoss
+  - _target_: uni2ts.loss.packed.PackedNRMSELoss
+    normalize: absolute_target_squared
+lr: 5e-7
+weight_decay: 1e-1
+beta1: 0.9
+beta2: 0.98
+num_training_steps: null
+num_warmup_steps: 0
+patch_size: null
+context_length: null
+prediction_length: null
+finetune_pattern: full
diff --git a/project/lsf-setup/lsf/eval/small/electricity.sh b/project/lsf-setup/lsf/eval/small/electricity.sh
@@ -3,17 +3,20 @@
 export HYDRA_FULL_ERROR=1
 export CUDA_VISIBLE_DEVICES=0
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=electricity
+cl=5000
+ps=64
+mode=S
 
 
 cpp1=''
 cpp2=''
 cpp3=''
 cpp4=''
+
 index=1
 for pl in 96 192 336 720; do
   case $index in
@@ -30,12 +33,12 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=electricity \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl
 

diff --git a/project/lsf-setup/lsf/eval/small/etth1.sh b/project/lsf-setup/lsf/eval/small/etth1.sh
@@ -3,16 +3,19 @@
 export HYDRA_FULL_ERROR=1
 export CUDA_VISIBLE_DEVICES=0
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=ETTh1
+cl=5000
+ps=64
+mode=S
+
 
-cpp1='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth1/cl512_pl96/checkpoints/epoch_8-step_495.ckpt'
-cpp2='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth1/cl512_pl192/checkpoints/epoch_4-step_275.ckpt'
-cpp3='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth1/cl512_pl336/checkpoints/epoch_1-step_108.ckpt'
-cpp4='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth1/cl512_pl720/checkpoints/epoch_0-step_51.ckpt'
+cpp1=''
+cpp2=''
+cpp3=''
+cpp4=''
 
 index=1
 for pl in 96 192 336 720; do
@@ -30,12 +33,12 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=ETTh1 \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl
 

diff --git a/project/lsf-setup/lsf/eval/small/etth2.sh b/project/lsf-setup/lsf/eval/small/etth2.sh
@@ -3,17 +3,19 @@
 export HYDRA_FULL_ERROR=1
 export CUDA_VISIBLE_DEVICES=0
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=ETTh2
+cl=3000
+ps=64
+mode=S
 
 
-cpp1='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl96/checkpoints/epoch_2-step_114.ckpt'
-cpp2='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl192/checkpoints/epoch_2-step_114.ckpt'
-cpp3='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl336/checkpoints/epoch_3-step_148.ckpt'
-cpp4='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl720/checkpoints/epoch_0-step_34.ckpt'
+cpp1=''
+cpp2=''
+cpp3=''
+cpp4=''
 
 index=1
 for pl in 96 192 336 720; do
@@ -31,12 +33,12 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=ETTh2 \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl
 

diff --git a/project/lsf-setup/lsf/eval/small/ettm1.sh b/project/lsf-setup/lsf/eval/small/ettm1.sh
@@ -1,19 +1,21 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=1
+export CUDA_VISIBLE_DEVICES=0
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=ETTm1
+cl=4000
+ps=128
+mode=S
 
 
-cpp1='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/ettm1/cl3000_pl96/checkpoints/epoch_5-step_1296.ckpt'
-cpp2='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/ettm1/cl3000_pl192/checkpoints/epoch_1-step_430.ckpt'
-cpp3='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/ettm1/cl3000_pl336/checkpoints/epoch_1-step_428.ckpt'
-cpp4='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/ettm1/cl3000_pl720/checkpoints/epoch_0-step_211.ckpt'
+cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl96/checkpoints/epoch_5-step_1254.ckpt'
+#cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl192/checkpoints/epoch_2-step_624.ckpt'
+#cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl336/checkpoints/epoch_1-step_414.ckpt'
+#cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl720/checkpoints/epoch_0-step_204.ckpt'
 
 index=1
 for pl in 96 192 336 720; do
@@ -31,12 +33,12 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=128 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=ETTm1 \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl
 

diff --git a/project/lsf-setup/lsf/eval/small/ettm2.sh b/project/lsf-setup/lsf/eval/small/ettm2.sh
@@ -1,20 +1,21 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=1
+export CUDA_VISIBLE_DEVICES=0
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=ETTm2
+cl=3000
+ps=64
+mode=S
 
 
-cpp1='./outputs/lsf_point/finetune/moirai_1.1_R_small/lsf/2_fc_head_ctx_dp0/ettm2/cl3000_pl96/checkpoints/epoch_11-step_2592.ckpt'
-cpp2='./outputs/lsf_point/finetune/moirai_1.1_R_small/lsf/2_fc_head_ctx_dp0/ettm2/cl3000_pl192/checkpoints/epoch_6-step_1505.ckpt'
-cpp3='./outputs/lsf_point/finetune/moirai_1.1_R_small/lsf/2_fc_head_ctx_dp0/ettm2/cl3000_pl336/checkpoints/epoch_4-step_1070.ckpt'
-cpp4='./outputs/lsf_point/finetune/moirai_1.1_R_small/lsf/2_fc_head_ctx_dp0/ettm2/cl3000_pl720/checkpoints/epoch_3-step_844.ckpt'
-
+cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl96/checkpoints/epoch_12-step_2808.ckpt'
+cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl192/checkpoints/epoch_4-step_1075.ckpt'
+cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl336/checkpoints/epoch_2-step_642.ckpt'
+cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl720/checkpoints/epoch_1-step_422.ckpt'
 
 index=1
 for pl in 96 192 336 720; do
@@ -32,12 +33,12 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=ETTm2 \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl
 

diff --git a/project/lsf-setup/lsf/eval/small/weather.sh b/project/lsf-setup/lsf/eval/small/weather.sh
@@ -3,20 +3,22 @@
 export HYDRA_FULL_ERROR=1
 export CUDA_VISIBLE_DEVICES=1
 
-mode=S
 cp=conf/lsf-setup/lsf/eval
 exp_name=lsf
-cl=3000
 model=moirai_lightning_ckpt
+data=weather
+cl=2000
+ps=128
+mode=S
 
 
-cpp1='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/weather/cl3000_pl96/checkpoints/epoch_14-step_10395.ckpt'
-cpp2='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/weather/cl3000_pl192/checkpoints/epoch_11-step_8304.ckpt'
-cpp3='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/weather/cl3000_pl336/checkpoints/epoch_6-step_4823.ckpt'
-cpp4='./outputs/lsf/finetune/moirai_1.1_R_small/lsf/full/weather/cl3000_pl720/checkpoints/epoch_3-step_2724.ckpt'
+cpp1=''
+cpp2=''
+cpp3=''
+cpp4=''
 
 index=1
-for pl in 96 192 336 ; do  # 720
+for pl in 96 192 336 720 ; do
   case $index in
     1) cpp=$cpp1 ;;
     2) cpp=$cpp2 ;;
@@ -31,12 +33,12 @@ for pl in 96 192 336 ; do  # 720
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=128 \
+    model.patch_size=$ps \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
     data=lsf_test \
-    data.dataset_name=weather \
+    data.dataset_name=$data \
     data.mode=$mode \
     data.prediction_length=$pl