Skip to content

Commit

Permalink
Add config and script for moirai-1.0-base
Browse files Browse the repository at this point in the history
  • Loading branch information
zqiao11 committed Nov 20, 2024
1 parent 1810c19 commit 3998641
Show file tree
Hide file tree
Showing 52 changed files with 1,331 additions and 68 deletions.
3 changes: 3 additions & 0 deletions cli/conf/lsf-setup/lsf/finetune/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ trainer:
mode: min
save_top_k: 1 # Qz: Sometimes the 1st validation gets anomalous results. Discard that ckpt, and use the 2nd one.
every_n_epochs: 1
- _target_: lightning.pytorch.callbacks.ModelCheckpoint
dirpath: ${hydra:runtime.output_dir}/checkpoints
save_weights_only: true
- _target_: lightning.pytorch.callbacks.EarlyStopping # uni2ts.callbacks.earlystop.WarmupEarlyStopping
monitor: val/PackedNLLLoss
min_delta: 0.0
Expand Down
41 changes: 41 additions & 0 deletions cli/conf/lsf-setup/lsf/finetune/model/moirai_1.0_R_base.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# load a pretrained checkpoint from huggingface hub
_target_: uni2ts.model.lsf_moirai.MoiraiFinetune
module:
_target_: uni2ts.model.lsf_moirai.MoiraiModule.from_pretrained
pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base
module_kwargs:
_target_: builtins.dict
distr_output:
_target_: uni2ts.distribution.MixtureOutput
components:
- _target_: uni2ts.distribution.StudentTOutput
- _target_: uni2ts.distribution.NormalFixedScaleOutput
- _target_: uni2ts.distribution.NegativeBinomialOutput
- _target_: uni2ts.distribution.LogNormalOutput
d_model: 768
num_layers: 12
patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
max_seq_len: 512
attn_dropout_p: 0.0
dropout_p: 0.0
scaling: true
min_patches: 2
min_mask_ratio: 0.15
max_mask_ratio: 0.5
max_dim: 128
loss_func:
_target_: uni2ts.loss.packed.PackedNLLLoss
val_metric:
- _target_: uni2ts.loss.packed.PackedMSELoss
- _target_: uni2ts.loss.packed.PackedNRMSELoss
normalize: absolute_target_squared
lr: 5e-7 # On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs
weight_decay: 1e-1
beta1: 0.9
beta2: 0.98
num_training_steps: null
num_warmup_steps: 0
patch_size: null
context_length: null
prediction_length: null
finetune_pattern: full
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
_target_: uni2ts.model.multi_scale_moirai.MoiraiForecast
module:
_target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained
pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base
num_samples: 100
patch_size: ???
context_length: ???
num_new_scales: 3
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# load a pretrained checkpoint from huggingface hub
_target_: uni2ts.model.multi_scale_moirai.MoiraiFinetune
module:
_target_: uni2ts.model.multi_scale_moirai.MoiraiModule.from_pretrained
pretrained_model_name_or_path: Salesforce/moirai-1.0-R-base
module_kwargs:
_target_: builtins.dict
distr_output:
_target_: uni2ts.distribution.MixtureOutput
components:
- _target_: uni2ts.distribution.StudentTOutput
- _target_: uni2ts.distribution.NormalFixedScaleOutput
- _target_: uni2ts.distribution.NegativeBinomialOutput
- _target_: uni2ts.distribution.LogNormalOutput
d_model: 768
num_layers: 12
patch_sizes: ${as_tuple:[8, 16, 32, 64, 128]}
max_seq_len: 512
attn_dropout_p: 0.0
dropout_p: 0.0
scaling: true
min_patches: 2
min_mask_ratio: 0.15
max_mask_ratio: 0.5
max_dim: 128
loss_func:
_target_: uni2ts.loss.packed.PackedNLLLoss
val_metric:
- _target_: uni2ts.loss.packed.PackedMSELoss
- _target_: uni2ts.loss.packed.PackedNRMSELoss
normalize: absolute_target_squared
lr: 5e-7 # On ETT dataset, using 1e-6/5e-7 converge within 1-2 epochs. 1e-7 converge in tens of epochs
weight_decay: 1e-1
beta1: 0.9
beta2: 0.98
num_training_steps: null
num_warmup_steps: 0
patch_size: null
context_length: null
prediction_length: null
finetune_pattern: full
num_new_scales: 3
ds_factor: 2
4 changes: 2 additions & 2 deletions cli/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ def main(cfg: DictConfig):
trainer: L.Trainer = instantiate(cfg.trainer)

# '=' in ckpt name make it cannot be directly loaded with hydra. Change it to '_'.
trainer.callbacks[1].CHECKPOINT_EQUALS_CHAR = "_"
trainer.callbacks[2].CHECKPOINT_EQUALS_CHAR = "_"
trainer.callbacks[-1].CHECKPOINT_EQUALS_CHAR = "_"
trainer.callbacks[-2].CHECKPOINT_EQUALS_CHAR = "_"

train_dataset: Dataset = instantiate(cfg.data).load_dataset(
model.train_transform_map
Expand Down
47 changes: 47 additions & 0 deletions project/lsf-setup/lsf/eval/base/electricity.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/bash

export HYDRA_FULL_ERROR=1
export CUDA_VISIBLE_DEVICES=0

cp=conf/lsf-setup/lsf/eval
exp_name=lsf
model=moirai_lightning_ckpt
data=electricity
cl=5000
ps=32
mode=S


cpp1=''
cpp2=''
cpp3=''
cpp4=''

index=1
for pl in 96 192 336 720; do
case $index in
1) cpp=$cpp1 ;;
2) cpp=$cpp2 ;;
3) cpp=$cpp3 ;;
4) cpp=$cpp4 ;;
esac

pretrained_model=$(echo $cpp | cut -d'/' -f6)
ft_pattern=$(echo $cpp | cut -d'/' -f8)

python -m cli.eval \
-cp $cp \
exp_name=$exp_name/$pretrained_model/$ft_pattern \
model=$model \
model.patch_size=$ps \
model.context_length=$cl \
model.checkpoint_path=$cpp \
model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
data=lsf_test \
data.dataset_name=$data \
data.mode=$mode \
data.prediction_length=$pl

index=$((index+1))
done

46 changes: 46 additions & 0 deletions project/lsf-setup/lsf/eval/base/etth1.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

export HYDRA_FULL_ERROR=1
export CUDA_VISIBLE_DEVICES=0

cp=conf/lsf-setup/lsf/eval
exp_name=lsf
model=moirai_lightning_ckpt
data=ETTh1
cl=5000
ps=64
mode=S


cpp1=''
cpp2=''
cpp3=''
cpp4=''

index=1
for pl in 96 192 336 720; do
case $index in
1) cpp=$cpp1 ;;
2) cpp=$cpp2 ;;
3) cpp=$cpp3 ;;
4) cpp=$cpp4 ;;
esac

pretrained_model=$(echo $cpp | cut -d'/' -f6)
ft_pattern=$(echo $cpp | cut -d'/' -f8)

python -m cli.eval \
-cp $cp \
exp_name=$exp_name/$pretrained_model/$ft_pattern \
model=$model \
model.patch_size=$ps \
model.context_length=$cl \
model.checkpoint_path=$cpp \
model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
data=lsf_test \
data.dataset_name=$data \
data.mode=$mode \
data.prediction_length=$pl

index=$((index+1))
done
46 changes: 46 additions & 0 deletions project/lsf-setup/lsf/eval/base/etth2.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

export HYDRA_FULL_ERROR=1
export CUDA_VISIBLE_DEVICES=0

cp=conf/lsf-setup/lsf/eval
exp_name=lsf
model=moirai_lightning_ckpt
data=ETTh2
cl=5000
ps=64
mode=S


cpp1=''
cpp2=''
cpp3=''
cpp4=''

index=1
for pl in 96 192 336 720; do
case $index in
1) cpp=$cpp1 ;;
2) cpp=$cpp2 ;;
3) cpp=$cpp3 ;;
4) cpp=$cpp4 ;;
esac

pretrained_model=$(echo $cpp | cut -d'/' -f6)
ft_pattern=$(echo $cpp | cut -d'/' -f8)

python -m cli.eval \
-cp $cp \
exp_name=$exp_name/$pretrained_model/$ft_pattern \
model=$model \
model.patch_size=$ps \
model.context_length=$cl \
model.checkpoint_path=$cpp \
model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
data=lsf_test \
data.dataset_name=$data \
data.mode=$mode \
data.prediction_length=$pl

index=$((index+1))
done
46 changes: 46 additions & 0 deletions project/lsf-setup/lsf/eval/base/ettm1.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

export HYDRA_FULL_ERROR=1
export CUDA_VISIBLE_DEVICES=0

cp=conf/lsf-setup/lsf/eval
exp_name=lsf
model=moirai_lightning_ckpt
data=ETTm1
cl=5000
ps=64
mode=S


cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl96/checkpoints/epoch_5-step_1254.ckpt'
#cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl192/checkpoints/epoch_2-step_624.ckpt'
#cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl336/checkpoints/epoch_1-step_414.ckpt'
#cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm1/cl4000_pl720/checkpoints/epoch_0-step_204.ckpt'

index=1
for pl in 96 192 336 720; do
case $index in
1) cpp=$cpp1 ;;
2) cpp=$cpp2 ;;
3) cpp=$cpp3 ;;
4) cpp=$cpp4 ;;
esac

pretrained_model=$(echo $cpp | cut -d'/' -f6)
ft_pattern=$(echo $cpp | cut -d'/' -f8)

python -m cli.eval \
-cp $cp \
exp_name=$exp_name/$pretrained_model/$ft_pattern \
model=$model \
model.patch_size=$ps \
model.context_length=$cl \
model.checkpoint_path=$cpp \
model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
data=lsf_test \
data.dataset_name=$data \
data.mode=$mode \
data.prediction_length=$pl

index=$((index+1))
done
46 changes: 46 additions & 0 deletions project/lsf-setup/lsf/eval/base/ettm2.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

export HYDRA_FULL_ERROR=1
export CUDA_VISIBLE_DEVICES=0

cp=conf/lsf-setup/lsf/eval
exp_name=lsf
model=moirai_lightning_ckpt
data=ETTm2
cl=5000
ps=128
mode=S


cpp1='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl96/checkpoints/epoch_12-step_2808.ckpt'
cpp2='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl192/checkpoints/epoch_4-step_1075.ckpt'
cpp3='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl336/checkpoints/epoch_2-step_642.ckpt'
cpp4='./outputs/lsf-setup/lsf/finetune/moirai_1.0_R_small/lsf/full/ettm2/cl3000_pl720/checkpoints/epoch_1-step_422.ckpt'

index=1
for pl in 96 192 336 720; do
case $index in
1) cpp=$cpp1 ;;
2) cpp=$cpp2 ;;
3) cpp=$cpp3 ;;
4) cpp=$cpp4 ;;
esac

pretrained_model=$(echo $cpp | cut -d'/' -f6)
ft_pattern=$(echo $cpp | cut -d'/' -f8)

python -m cli.eval \
-cp $cp \
exp_name=$exp_name/$pretrained_model/$ft_pattern \
model=$model \
model.patch_size=$ps \
model.context_length=$cl \
model.checkpoint_path=$cpp \
model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \
data=lsf_test \
data.dataset_name=$data \
data.mode=$mode \
data.prediction_length=$pl

index=$((index+1))
done
8 changes: 8 additions & 0 deletions project/lsf-setup/lsf/eval/base/run_multi.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

bash project/lsf/eval/base/etth1.sh
bash project/lsf/eval/base/etth2.sh
bash project/lsf/eval/base/ettm1.sh
bash project/lsf/eval/base/ettm2.sh
bash project/lsf/eval/base/weather.sh

Loading

0 comments on commit 3998641

Please sign in to comment.