diff --git a/cli/conf/lsf-setup/lsf/finetune/default.yaml b/cli/conf/lsf-setup/lsf/finetune/default.yaml index 1ea168e..76e0e4b 100644 --- a/cli/conf/lsf-setup/lsf/finetune/default.yaml +++ b/cli/conf/lsf-setup/lsf/finetune/default.yaml @@ -33,6 +33,9 @@ trainer: mode: min save_top_k: 1 # Qz: Sometimes the 1st validation gets anomalous results. Discard that ckpt, and use the 2nd one. every_n_epochs: 1 + - _target_: lightning.pytorch.callbacks.ModelCheckpoint + dirpath: ${hydra:runtime.output_dir}/checkpoints + save_weights_only: true - _target_: lightning.pytorch.callbacks.EarlyStopping # uni2ts.callbacks.earlystop.WarmupEarlyStopping monitor: val/PackedNLLLoss min_delta: 0.0 diff --git a/cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_base.yaml b/cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_base.yaml new file mode 100644 index 0000000..3272270 --- /dev/null +++ b/cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_base.yaml @@ -0,0 +1,41 @@ +# load a pretrained checkpoint from huggingface hub +_target_: uni2ts.model.lsf_moirai.MoiraiFinetune +module: + _target_: uni2ts.model.lsf_moirai.MoiraiModule.from_pretrained + pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base +module_kwargs: + _target_: builtins.dict + distr_output: + _target_: uni2ts.distribution.MixtureOutput + components: + - _target_: uni2ts.distribution.StudentTOutput + - _target_: uni2ts.distribution.NormalFixedScaleOutput + - _target_: uni2ts.distribution.NegativeBinomialOutput + - _target_: uni2ts.distribution.LogNormalOutput + d_model: 768 + num_layers: 12 + patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]} + max_seq_len: 512 + attn_dropout_p: 0.0 + dropout_p: 0.0 + scaling: true +min_patches: 2 +min_mask_ratio: 0.15 +max_mask_ratio: 0.5 +max_dim: 128 +loss_func: + _target_: uni2ts.loss.packed.PackedNLLLoss +val_metric: + - _target_: uni2ts.loss.packed.PackedMSELoss + - _target_: uni2ts.loss.packed.PackedNRMSELoss + normalize: absolute_target_squared +lr: 5e-7 # On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs +weight_decay: 1e-1 +beta1: 0.9 +beta2: 0.98 +num_training_steps: null +num_warmup_steps: 0 +patch_size: null +context_length: null +prediction_length: null +finetune_pattern: full diff --git a/cli/conf/lsf-setup/multi_scale/eval/model/moirai_1.0_R_base.yaml b/cli/conf/lsf-setup/multi_scale/eval/model/moirai_1.0_R_base.yaml new file mode 100644 index 0000000..a5d1981 --- /dev/null +++ b/cli/conf/lsf-setup/multi_scale/eval/model/moirai_1.0_R_base.yaml @@ -0,0 +1,8 @@ +_target_: uni2ts.model.multi_scale_moirai.MoiraiForecast +module: + _target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained + pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base +num_samples: 100 +patch_size: ??? +context_length: ??? +num_new_scales: 3 \ No newline at end of file diff --git a/cli/conf/lsf-setup/multi_scale/finetune/model/moirai_1.0_R_base.yaml b/cli/conf/lsf-setup/multi_scale/finetune/model/moirai_1.0_R_base.yaml new file mode 100644 index 0000000..b8bb63b --- /dev/null +++ b/cli/conf/lsf-setup/multi_scale/finetune/model/moirai_1.0_R_base.yaml @@ -0,0 +1,43 @@ +# load a pretrained checkpoint from huggingface hub +_target_: uni2ts.model.multi_scale_moirai.MoiraiFinetune +module: + _target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained + pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base +module_kwargs: + _target_: builtins.dict + distr_output: + _target_: uni2ts.distribution.MixtureOutput + components: + - _target_: uni2ts.distribution.StudentTOutput + - _target_: uni2ts.distribution.NormalFixedScaleOutput + - _target_: uni2ts.distribution.NegativeBinomialOutput + - _target_: uni2ts.distribution.LogNormalOutput + d_model: 768 + num_layers: 12 + patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]} + max_seq_len: 512 + attn_dropout_p: 0.0 + dropout_p: 0.0 + scaling: true +min_patches: 2 +min_mask_ratio: 0.15 +max_mask_ratio: 0.5 +max_dim: 128 +loss_func: + _target_: uni2ts.loss.packed.PackedNLLLoss +val_metric: + - _target_: uni2ts.loss.packed.PackedMSELoss + - _target_: uni2ts.loss.packed.PackedNRMSELoss + normalize: absolute_target_squared +lr: 5e-7 # On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs +weight_decay: 1e-1 +beta1: 0.9 +beta2: 0.98 +num_training_steps: null +num_warmup_steps: 0 +patch_size: null +context_length: null +prediction_length: null +finetune_pattern: full +num_new_scales: 3 +ds_factor: 2 diff --git a/cli/train.py b/cli/train.py index 2270d1d..1bd32df 100644 --- a/cli/train.py +++ b/cli/train.py @@ -139,8 +139,8 @@ def main(cfg: DictConfig): trainer: L.Trainer = instantiate(cfg.trainer) # '=' in ckpt name make it cannot be directly loaded with hydra. Change it to '_'. - trainer.callbacks[1].CHECKPOINT_EQUALS_CHAR = "_" - trainer.callbacks[2].CHECKPOINT_EQUALS_CHAR = "_" + trainer.callbacks[-1].CHECKPOINT_EQUALS_CHAR = "_" + trainer.callbacks[-2].CHECKPOINT_EQUALS_CHAR = "_" train_dataset: Dataset = instantiate(cfg.data).load_dataset( model.train_transform_map diff --git a/project/lsf-setup/lsf/eval/base/electricity.sh b/project/lsf-setup/lsf/eval/base/electricity.sh new file mode 100644 index 0000000..947c57a --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/electricity.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=electricity +cl=5000 +ps=32 +mode=S + + +cpp1='' +cpp2='' +cpp3='' +cpp4='' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done + diff --git a/project/lsf-setup/lsf/eval/base/etth1.sh b/project/lsf-setup/lsf/eval/base/etth1.sh new file mode 100644 index 0000000..6e110d4 --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/etth1.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=ETTh1 +cl=5000 +ps=64 +mode=S + + +cpp1='' +cpp2='' +cpp3='' +cpp4='' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/eval/base/etth2.sh b/project/lsf-setup/lsf/eval/base/etth2.sh new file mode 100644 index 0000000..c966e00 --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/etth2.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=ETTh2 +cl=5000 +ps=64 +mode=S + + +cpp1='' +cpp2='' +cpp3='' +cpp4='' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/lsf/eval/base/ettm1.sh b/project/lsf-setup/lsf/eval/base/ettm1.sh new file mode 100644 index 0000000..0a14e2e --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/ettm1.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=ETTm1 +cl=5000 +ps=64 +mode=S + + +cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl96/checkpoints/epoch_5-step_1254.ckpt' +#cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl192/checkpoints/epoch_2-step_624.ckpt' +#cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl336/checkpoints/epoch_1-step_414.ckpt' +#cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl720/checkpoints/epoch_0-step_204.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/lsf/eval/base/ettm2.sh b/project/lsf-setup/lsf/eval/base/ettm2.sh new file mode 100644 index 0000000..5cbc0be --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/ettm2.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=ETTm2 +cl=5000 +ps=128 +mode=S + + +cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl96/checkpoints/epoch_12-step_2808.ckpt' +cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl192/checkpoints/epoch_4-step_1075.ckpt' +cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl336/checkpoints/epoch_2-step_642.ckpt' +cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl720/checkpoints/epoch_1-step_422.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/lsf/eval/base/run_multi.sh b/project/lsf-setup/lsf/eval/base/run_multi.sh new file mode 100644 index 0000000..b0ddfb5 --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/run_multi.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +bash project/lsf/eval/base/etth1.sh +bash project/lsf/eval/base/etth2.sh +bash project/lsf/eval/base/ettm1.sh +bash project/lsf/eval/base/ettm2.sh +bash project/lsf/eval/base/weather.sh + diff --git a/project/lsf-setup/lsf/eval/base/weather.sh b/project/lsf-setup/lsf/eval/base/weather.sh new file mode 100644 index 0000000..05d2ed8 --- /dev/null +++ b/project/lsf-setup/lsf/eval/base/weather.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +cp=conf/lsf-setup/lsf/eval +exp_name=lsf +model=moirai_lightning_ckpt +data=weather +cl=5000 +ps=128 +mode=S + + +cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl96/checkpoints/epoch_22-step_16422.ckpt' +cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl192/checkpoints/epoch_17-step_12816.ckpt' +cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl336/checkpoints/epoch_11-step_8508.ckpt' +cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl720/checkpoints/epoch_8-step_6309.ckpt' + +index=1 +for pl in 96 192 336 720 ; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=$ps \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=$data \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/lsf/eval/small/weather.sh b/project/lsf-setup/lsf/eval/small/weather.sh index b4e01c0..03c2c56 100644 --- a/project/lsf-setup/lsf/eval/small/weather.sh +++ b/project/lsf-setup/lsf/eval/small/weather.sh @@ -1,7 +1,7 @@ #!/bin/bash export HYDRA_FULL_ERROR=1 -export CUDA_VISIBLE_DEVICES=1 +export CUDA_VISIBLE_DEVICES=0 cp=conf/lsf-setup/lsf/eval exp_name=lsf @@ -12,10 +12,10 @@ ps=128 mode=S -cpp1='' -cpp2='' -cpp3='' -cpp4='' +cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl96/checkpoints/epoch_22-step_16422.ckpt' +cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl192/checkpoints/epoch_17-step_12816.ckpt' +cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl336/checkpoints/epoch_11-step_8508.ckpt' +cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/weather/S/cl2000_pl720/checkpoints/epoch_8-step_6309.ckpt' index=1 for pl in 96 192 336 720 ; do diff --git a/project/lsf-setup/lsf/finetune/base/electricity.sh b/project/lsf-setup/lsf/finetune/base/electricity.sh new file mode 100644 index 0000000..53bf91d --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/electricity.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=electricity +cl=5000 +ps=32 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} \ + model.lr=5e-6 +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/etth1.sh b/project/lsf-setup/lsf/finetune/base/etth1.sh new file mode 100644 index 0000000..71e8440 --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/etth1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=etth1 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/etth2.sh b/project/lsf-setup/lsf/finetune/base/etth2.sh new file mode 100644 index 0000000..3db0f13 --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/etth2.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=etth2 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/ettm1.sh b/project/lsf-setup/lsf/finetune/base/ettm1.sh new file mode 100644 index 0000000..755149d --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/ettm1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=ettm1 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/ettm2.sh b/project/lsf-setup/lsf/finetune/base/ettm2.sh new file mode 100644 index 0000000..a116de1 --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/ettm2.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=ettm2 +cl=5000 +ps=128 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/run_multi.sh b/project/lsf-setup/lsf/finetune/base/run_multi.sh new file mode 100644 index 0000000..daa39e4 --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/run_multi.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +bash project/lsf/finetune/base/etth1.sh +bash project/lsf/finetune/base/etth2.sh +bash project/lsf/finetune/base/ettm1.sh +bash project/lsf/finetune/base/ettm2.sh +bash project/lsf/finetune/base/weather.sh \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/base/weather.sh b/project/lsf-setup/lsf/finetune/base/weather.sh new file mode 100644 index 0000000..e98c9a9 --- /dev/null +++ b/project/lsf-setup/lsf/finetune/base/weather.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_base +cp=conf/lsf-setup/lsf/finetune +exp_name=lsf +data=weather +cl=5000 +ps=128 +mode=S # M +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/lsf/finetune/small/electricity.sh b/project/lsf-setup/lsf/finetune/small/electricity.sh index d57c1b6..5549c60 100644 --- a/project/lsf-setup/lsf/finetune/small/electricity.sh +++ b/project/lsf-setup/lsf/finetune/small/electricity.sh @@ -12,7 +12,7 @@ mode=S ft_pattern=full -for pl in 96 192 336 720; do +for pl in 192 336 720; do # 96 python -m cli.train \ -cp $cp \ exp_name=$exp_name \ diff --git a/project/lsf-setup/lsf/finetune/small/weather.sh b/project/lsf-setup/lsf/finetune/small/weather.sh index 9db8ddc..6d681d9 100644 --- a/project/lsf-setup/lsf/finetune/small/weather.sh +++ b/project/lsf-setup/lsf/finetune/small/weather.sh @@ -1,6 +1,6 @@ #!/bin/bash -export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=2; model=moirai_1.0_R_small cp=conf/lsf-setup/lsf/finetune @@ -8,11 +8,11 @@ exp_name=lsf data=weather cl=2000 ps=128 -mode=M +mode=S # M ft_pattern=full -for pl in 96 192 336 720; do +for pl in 720; do # 96 192 336 python -m cli.train \ -cp $cp \ exp_name=$exp_name \ diff --git a/project/lsf-setup/multi_scale/eval/base/electricity.sh b/project/lsf-setup/multi_scale/eval/base/electricity.sh new file mode 100644 index 0000000..8481db7 --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/electricity.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + + +cpp1='.-mm/outputs_old/lsf/moirai_1.0_R_small/lsf/electricity/cl3000_pl96/checkpoints/epoch_998-step_199800.ckpt' +cpp2='.-mm/outputs/lsf/moirai_1.0_R_small/lsf/electricity/cl3000_pl192/checkpoints/epoch_748-step_149800.ckpt' +cpp3='.-mm/outputs/lsf/moirai_1.0_R_small/lsf/electricity/cl3000_pl336/checkpoints/epoch_600-step_120200.ckpt' +cpp4='.-mm/outputs/lsf/moirai_1.0_R_small/lsf/electricity/cl3000_pl720/checkpoints/epoch_385-step_77200.ckpt' +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=32 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=electricity \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done + diff --git a/project/lsf-setup/multi_scale/eval/base/etth1.sh b/project/lsf-setup/multi_scale/eval/base/etth1.sh new file mode 100644 index 0000000..989b309 --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/etth1.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + +cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl96/checkpoints/epoch_2-step_228.ckpt' +cpp2='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl192/checkpoints/epoch_1-step_150.ckpt' +cpp3='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl336/checkpoints/epoch_1-step_146.ckpt' +cpp4='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl720/checkpoints/epoch_0-step_68.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=64 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=ETTh1 \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/eval/base/etth2.sh b/project/lsf-setup/multi_scale/eval/base/etth2.sh new file mode 100644 index 0000000..38a509b --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/etth2.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + +cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl96/checkpoints/epoch_0-step_76.ckpt' +cpp2='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl192/checkpoints/epoch_1-step_150.ckpt' +cpp3='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl336/checkpoints/epoch_1-step_146.ckpt' +cpp4='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl720/checkpoints/epoch_0-step_68.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=64 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=ETTh2 \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/multi_scale/eval/base/ettm1.sh b/project/lsf-setup/multi_scale/eval/base/ettm1.sh new file mode 100644 index 0000000..1472c36 --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/ettm1.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=0 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl96/checkpoints/epoch_2-step_1251.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl192/checkpoints/epoch_0-step_416.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl336/checkpoints/epoch_0-step_414.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl720/checkpoints/epoch_0-step_408.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=64 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=ETTm1 \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/multi_scale/eval/base/ettm2.sh b/project/lsf-setup/multi_scale/eval/base/ettm2.sh new file mode 100644 index 0000000..53a197a --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/ettm2.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=3 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl96/checkpoints/epoch_4-step_2155.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl192/checkpoints/epoch_2-step_1287.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl336/checkpoints/epoch_0-step_427.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl720/checkpoints/epoch_0-step_422.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=128 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=ETTm2 \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done diff --git a/project/lsf-setup/multi_scale/eval/base/run_multi.sh b/project/lsf-setup/multi_scale/eval/base/run_multi.sh new file mode 100644 index 0000000..6fc18a1 --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/run_multi.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +bash project/multi_scale/eval/small/etth1.sh +bash project/multi_scale/eval/small/etth2.sh +bash project/multi_scale/eval/small/ettm1.sh +bash project/multi_scale/eval/small/ettm2.sh +bash project/multi_scale/eval/small/weather.sh + diff --git a/project/lsf-setup/multi_scale/eval/base/weather.sh b/project/lsf-setup/multi_scale/eval/base/weather.sh new file mode 100644 index 0000000..f14958b --- /dev/null +++ b/project/lsf-setup/multi_scale/eval/base/weather.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=2 + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=lsf +cl=5000 +model=moirai_lightning_ckpt + +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl96/checkpoints/epoch_10-step_15708.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl192/checkpoints/epoch_7-step_11392.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl336/checkpoints/epoch_4-step_7090.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl720/checkpoints/epoch_2-step_4206.ckpt' + +index=1 +for pl in 96 192 336 720; do + case $index in + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; + esac + + pretrained_model=$(echo $cpp | cut -d'/' -f6) + ft_pattern=$(echo $cpp | cut -d'/' -f8) + + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name/$pretrained_model/$ft_pattern \ + model=$model \ + model.patch_size=128 \ + model.context_length=$cl \ + model.checkpoint_path=$cpp \ + model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \ + data=lsf_test \ + data.dataset_name=weather \ + data.mode=$mode \ + data.prediction_length=$pl + + index=$((index+1)) +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/eval/small/electricity.sh b/project/lsf-setup/multi_scale/eval/small/electricity.sh index 7147559..c26c07f 100644 --- a/project/lsf-setup/multi_scale/eval/small/electricity.sh +++ b/project/lsf-setup/multi_scale/eval/small/electricity.sh @@ -6,7 +6,7 @@ export CUDA_VISIBLE_DEVICES=0 mode=S cp=conf/lsf-setup/multi_scale/eval exp_name=lsf -cl=3000 +cl=5000 model=moirai_lightning_ckpt diff --git a/project/lsf-setup/multi_scale/eval/small/etth1.sh b/project/lsf-setup/multi_scale/eval/small/etth1.sh index debdae0..989b309 100644 --- a/project/lsf-setup/multi_scale/eval/small/etth1.sh +++ b/project/lsf-setup/multi_scale/eval/small/etth1.sh @@ -6,7 +6,7 @@ export CUDA_VISIBLE_DEVICES=0 mode=S cp=conf/lsf-setup/multi_scale/eval exp_name=lsf -cl=3000 +cl=5000 model=moirai_lightning_ckpt cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl96/checkpoints/epoch_2-step_228.ckpt' diff --git a/project/lsf-setup/multi_scale/eval/small/ettm1.sh b/project/lsf-setup/multi_scale/eval/small/ettm1.sh index b5cb524..66c080a 100644 --- a/project/lsf-setup/multi_scale/eval/small/ettm1.sh +++ b/project/lsf-setup/multi_scale/eval/small/ettm1.sh @@ -9,21 +9,18 @@ exp_name=lsf cl=4000 model=moirai_lightning_ckpt -#cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm1/S/cl4000_pl96/checkpoints/epoch_3-step_1668.ckpt' -#cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm2/S/cl3000_pl192/checkpoints/epoch_2-step_1287.ckpt' -cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm1/S/cl4000_pl336/checkpoints/epoch_3-step_1656.ckpt' -cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm1/S/cl4000_pl720/checkpoints/epoch_3-step_1632.ckpt' +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl96/checkpoints/epoch_2-step_1251.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl192/checkpoints/epoch_0-step_416.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl336/checkpoints/epoch_0-step_414.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/lsf/full/ettm1/S/cl4000_pl720/checkpoints/epoch_0-step_408.ckpt' index=1 -for pl in 336 720; do # 96 192 +for pl in 96 192 336 720; do case $index in - 1) cpp=$cpp3 ;; - 2) cpp=$cpp4 ;; - -# 1) cpp=$cpp1 ;; -# 2) cpp=$cpp2 ;; -# 3) cpp=$cpp3 ;; -# 4) cpp=$cpp4 ;; + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; esac pretrained_model=$(echo $cpp | cut -d'/' -f6) diff --git a/project/lsf-setup/multi_scale/eval/small/ettm2.sh b/project/lsf-setup/multi_scale/eval/small/ettm2.sh index 903ff35..1dec675 100644 --- a/project/lsf-setup/multi_scale/eval/small/ettm2.sh +++ b/project/lsf-setup/multi_scale/eval/small/ettm2.sh @@ -9,22 +9,18 @@ exp_name=lsf cl=3000 model=moirai_lightning_ckpt -#cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm2/S/cl3000_pl96/checkpoints/epoch_5-step_2586.ckpt' -cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm2/S/cl3000_pl192/checkpoints/epoch_2-step_1287.ckpt' -cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm2/S/cl3000_pl336/checkpoints/epoch_3-step_1708.ckpt' -cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/lsf/full/ettm2/S/cl4000_pl720/checkpoints/epoch_3-step_1688.ckpt' +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl96/checkpoints/epoch_4-step_2155.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl192/checkpoints/epoch_2-step_1287.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl336/checkpoints/epoch_0-step_427.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.1_R_small/ms_qkv_1.1_ctx3000/full/ettm2/S/cl3000_pl720/checkpoints/epoch_0-step_422.ckpt' index=1 -for pl in 192 336 720; do # 96 +for pl in 96 192 336 720; do case $index in - 1) cpp=$cpp2 ;; - 2) cpp=$cpp3 ;; - 3) cpp=$cpp4 ;; - -# 1) cpp=$cpp1 ;; -# 2) cpp=$cpp2 ;; -# 3) cpp=$cpp3 ;; -# 4) cpp=$cpp4 ;; + 1) cpp=$cpp1 ;; + 2) cpp=$cpp2 ;; + 3) cpp=$cpp3 ;; + 4) cpp=$cpp4 ;; esac pretrained_model=$(echo $cpp | cut -d'/' -f6) diff --git a/project/lsf-setup/multi_scale/eval/small/weather.sh b/project/lsf-setup/multi_scale/eval/small/weather.sh index 73ac538..f35b95b 100644 --- a/project/lsf-setup/multi_scale/eval/small/weather.sh +++ b/project/lsf-setup/multi_scale/eval/small/weather.sh @@ -1,18 +1,18 @@ #!/bin/bash export HYDRA_FULL_ERROR=1 -export CUDA_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=2 mode=S cp=conf/lsf-setup/multi_scale/eval exp_name=lsf -cl=3000 +cl=2000 model=moirai_lightning_ckpt -cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/weather/cl3000_pl96/checkpoints/epoch_2-step_300.ckpt' -cpp2='./outputs/origin/moirai_1.1_R_small/lsf/full/weather/cl3000_pl192/checkpoints/epoch_2-step_300.ckpt' -cpp3='./outputs/origin/moirai_1.1_R_small/lsf/full/weather/cl3000_pl336/checkpoints/epoch_2-step_300.ckpt' -cpp4='./outputs/origin/moirai_1.1_R_small/lsf/full/weather/cl3000_pl720/checkpoints/epoch_2-step_300.ckpt' +cpp1='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl96/checkpoints/epoch_10-step_15708.ckpt' +cpp2='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl192/checkpoints/epoch_7-step_11392.ckpt' +cpp3='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl336/checkpoints/epoch_4-step_7090.ckpt' +cpp4='./outputs/lsf-setup/multi_scale/finetune/moirai_1.0_R_small/ms_qkv_1.0/full/weather/S/cl2000_pl720/checkpoints/epoch_2-step_4206.ckpt' index=1 for pl in 96 192 336 720; do diff --git a/project/lsf-setup/multi_scale/finetune/base/electricity.sh b/project/lsf-setup/multi_scale/finetune/base/electricity.sh new file mode 100644 index 0000000..d5de251 --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/electricity.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=lsf +data=electricity +cl=5000 +ps=32 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} \ + model.lr=5e-6 +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/etth1.sh b/project/lsf-setup/multi_scale/finetune/base/etth1.sh new file mode 100644 index 0000000..fcb7900 --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/etth1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=1; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=lsf +data=etth1 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/etth2.sh b/project/lsf-setup/multi_scale/finetune/base/etth2.sh new file mode 100644 index 0000000..6190f38 --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/etth2.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=1; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=lsf +data=etth2 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/ettm1.sh b/project/lsf-setup/multi_scale/finetune/base/ettm1.sh new file mode 100644 index 0000000..5697517 --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/ettm1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=ms_qkv_1.0 +data=ettm1 +cl=5000 +ps=64 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/ettm2.sh b/project/lsf-setup/multi_scale/finetune/base/ettm2.sh new file mode 100644 index 0000000..6979a8b --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/ettm2.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=3; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=ms_qkv_1.0 +data=ettm2 +cl=5000 +ps=128 +mode=S +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/run_multi.sh b/project/lsf-setup/multi_scale/finetune/base/run_multi.sh new file mode 100644 index 0000000..bc8e2be --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/run_multi.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +bash project/multi_scale/finetune/small/etth1.sh +bash project/multi_scale/finetune/small/etth2.sh +#bash project/multi_scale/finetune/small/ettm1.sh +#bash project/multi_scale/finetune/small/ettm2.sh +#bash project/multi_scale/finetune/small/weather.sh \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/base/weather.sh b/project/lsf-setup/multi_scale/finetune/base/weather.sh new file mode 100644 index 0000000..7694b2d --- /dev/null +++ b/project/lsf-setup/multi_scale/finetune/base/weather.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +model=moirai_1.0_R_small +cp=conf/lsf-setup/multi_scale/finetune +exp_name=ms_qkv_1.0 +data=weather +cl=5000 +ps=128 +mode=S # M +ft_pattern=full + + +for pl in 96 192 336 720; do + python -m cli.train \ + -cp $cp \ + exp_name=$exp_name \ + run_name=cl${cl}_pl${pl} \ + model=$model \ + model.patch_size=${ps} \ + model.context_length=$cl \ + model.prediction_length=$pl \ + model.finetune_pattern=$ft_pattern \ + data=${data} \ + data.patch_size=${ps} \ + data.context_length=$cl \ + data.prediction_length=$pl \ + data.mode=${mode} \ + val_data=${data} \ + val_data.patch_size=${ps} \ + val_data.context_length=$cl \ + val_data.prediction_length=$pl \ + val_data.mode=${mode} +done \ No newline at end of file diff --git a/project/lsf-setup/multi_scale/finetune/small/etth1.sh b/project/lsf-setup/multi_scale/finetune/small/etth1.sh index cedd949..fcb7900 100644 --- a/project/lsf-setup/multi_scale/finetune/small/etth1.sh +++ b/project/lsf-setup/multi_scale/finetune/small/etth1.sh @@ -8,7 +8,7 @@ exp_name=lsf data=etth1 cl=5000 ps=64 -mode=M +mode=S ft_pattern=full diff --git a/project/lsf-setup/multi_scale/finetune/small/etth2.sh b/project/lsf-setup/multi_scale/finetune/small/etth2.sh index c2baada..b81cffb 100644 --- a/project/lsf-setup/multi_scale/finetune/small/etth2.sh +++ b/project/lsf-setup/multi_scale/finetune/small/etth2.sh @@ -8,7 +8,7 @@ exp_name=lsf data=etth2 cl=3000 ps=64 -mode=M +mode=S ft_pattern=full diff --git a/project/lsf-setup/multi_scale/finetune/small/ettm1.sh b/project/lsf-setup/multi_scale/finetune/small/ettm1.sh index 30204fc..ba8f05e 100644 --- a/project/lsf-setup/multi_scale/finetune/small/ettm1.sh +++ b/project/lsf-setup/multi_scale/finetune/small/ettm1.sh @@ -4,7 +4,7 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; model=moirai_1.0_R_small cp=conf/lsf-setup/multi_scale/finetune -exp_name=lsf +exp_name=ms_qkv_1.0 data=ettm1 cl=4000 ps=128 diff --git a/project/lsf-setup/multi_scale/finetune/small/ettm2.sh b/project/lsf-setup/multi_scale/finetune/small/ettm2.sh index 66ff9cb..b429aa2 100644 --- a/project/lsf-setup/multi_scale/finetune/small/ettm2.sh +++ b/project/lsf-setup/multi_scale/finetune/small/ettm2.sh @@ -4,7 +4,7 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=3; model=moirai_1.0_R_small cp=conf/lsf-setup/multi_scale/finetune -exp_name=lsf +exp_name=ms_qkv_1.0 data=ettm2 cl=3000 ps=64 diff --git a/project/lsf-setup/multi_scale/finetune/small/weather.sh b/project/lsf-setup/multi_scale/finetune/small/weather.sh index 95e2cc7..1c07990 100644 --- a/project/lsf-setup/multi_scale/finetune/small/weather.sh +++ b/project/lsf-setup/multi_scale/finetune/small/weather.sh @@ -4,11 +4,11 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; model=moirai_1.0_R_small cp=conf/lsf-setup/multi_scale/finetune -exp_name=lsf +exp_name=ms_qkv_1.0 data=weather cl=2000 ps=128 -mode=M +mode=S # M ft_pattern=full diff --git a/project/lsf-setup/multi_scale/zero_shot/lsf_base.sh b/project/lsf-setup/multi_scale/zero_shot/lsf_base.sh new file mode 100644 index 0000000..6ada258 --- /dev/null +++ b/project/lsf-setup/multi_scale/zero_shot/lsf_base.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +mode=S +cp=conf/lsf-setup/multi_scale/eval +exp_name=ms_direct_zero_shot +model=moirai_1.0_R_base + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTh1 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTh2 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTm1 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=128 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTm2 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=32 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=electricity \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=128 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=weather \ + data.mode=$mode \ + data.prediction_length=$pl +done diff --git a/project/lsf-setup/multi_scale/zero_shot/lsf_small.sh b/project/lsf-setup/multi_scale/zero_shot/lsf_small.sh index 48f2b9e..7321152 100644 --- a/project/lsf-setup/multi_scale/zero_shot/lsf_small.sh +++ b/project/lsf-setup/multi_scale/zero_shot/lsf_small.sh @@ -1,11 +1,11 @@ #!/bin/bash -export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=2; mode=S cp=conf/lsf-setup/multi_scale/eval -exp_name=lsf_zero_shot -model=moirai_1.0_R_small +exp_name=ms_direct_zero_shot +model=moirai_1.1_R_small #for pl in 96 192 336 720; do @@ -36,21 +36,21 @@ model=moirai_1.0_R_small #done -for pl in 96 192 336 720; do - python -m cli.eval \ - -cp $cp \ - exp_name=$exp_name \ - model=$model \ - model.patch_size=128 \ - model.context_length=4000 \ - data=lsf_test \ - data.dataset_name=ETTm1 \ - data.mode=S \ - data.prediction_length=$pl -done +#for pl in 96 192 336 720; do +# python -m cli.eval \ +# -cp $cp \ +# exp_name=$exp_name \ +# model=$model \ +# model.patch_size=128 \ +# model.context_length=4000 \ +# data=lsf_test \ +# data.dataset_name=ETTm1 \ +# data.mode=S \ +# data.prediction_length=$pl +#done -for pl in 96 192 336 720; do +for pl in 96 ; do # 192 336 720 python -m cli.eval \ -cp $cp \ exp_name=$exp_name \ diff --git a/project/lsf-setup/origin/zero_shot/lsf_base.sh b/project/lsf-setup/origin/zero_shot/lsf_base.sh new file mode 100644 index 0000000..53acf44 --- /dev/null +++ b/project/lsf-setup/origin/zero_shot/lsf_base.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0; + +mode=S +cp=conf/lsf-setup/origin/eval +exp_name=lsf_zero_shot +model=moirai_1.0_R_base + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTh1 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTh2 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=64 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTm1 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=128 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=ETTm2 \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=32 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=electricity \ + data.mode=$mode \ + data.prediction_length=$pl +done + + +for pl in 96 192 336 720; do + python -m cli.eval \ + -cp $cp \ + exp_name=$exp_name \ + model=$model \ + model.patch_size=128 \ + model.context_length=5000 \ + data=lsf_test \ + data.dataset_name=weather \ + data.mode=$mode \ + data.prediction_length=$pl +done diff --git a/src/uni2ts/model/multi_scale_moirai/finetune.py b/src/uni2ts/model/multi_scale_moirai/finetune.py index 24b23eb..b7105e8 100644 --- a/src/uni2ts/model/multi_scale_moirai/finetune.py +++ b/src/uni2ts/model/multi_scale_moirai/finetune.py @@ -132,6 +132,7 @@ def __init__( self.token_idx_per_scale = self._get_token_idx_per_scale() def post_init(self): + for layer in self.module.encoder.layers: # Check if the layer has an attribute named `self_attn` and if it is an instance of GroupedQueryAttention if hasattr(layer, 'self_attn') and isinstance(layer.self_attn, GroupedQueryAttention): diff --git a/src/uni2ts/model/multi_scale_moirai/forecast.py b/src/uni2ts/model/multi_scale_moirai/forecast.py index 86252c9..5f1869e 100644 --- a/src/uni2ts/model/multi_scale_moirai/forecast.py +++ b/src/uni2ts/model/multi_scale_moirai/forecast.py @@ -128,6 +128,8 @@ def __init__( def post_init(self): + + for layer in self.module.encoder.layers: # Check if the layer has an attribute named `self_attn` and if it is an instance of GroupedQueryAttention if hasattr(layer, 'self_attn') and isinstance(layer.self_attn, GroupedQueryAttention): diff --git a/src/uni2ts/module/multi_scale/attention.py b/src/uni2ts/module/multi_scale/attention.py index abb4009..c71f731 100644 --- a/src/uni2ts/module/multi_scale/attention.py +++ b/src/uni2ts/module/multi_scale/attention.py @@ -353,10 +353,10 @@ def forward( index = index_by_variate[scale + 1] query_scale = init_query[..., index, :] # (bs, num_patch_new_scale, dim) query[..., index, :] = self.query_adapt_weight[scale] * query_scale + self.query_adapt_bias[scale] - + key_scale = init_key[..., index, :] # (bs, num_patch_new_scale, dim) key[..., index, :] = self.key_adapt_weight[scale] * key_scale + self.key_adapt_bias[scale] - + value_scale = init_value[..., index, :] # (bs, num_patch_new_scale, dim) value[..., index, :] = self.value_adapt_weight[scale] * value_scale + self.value_adapt_bias[scale]