Sample-conditioned weighting loss per scale

zqiao11 · Jan 18, 2025 · defb30c · defb30c
1 parent a9fec12
commit defb30c
Show file tree

Hide file tree

Showing 17 changed files with 323 additions and 163 deletions.
diff --git a/project/lsf/multi_scale/eval/small/etth1.sh b/project/lsf/multi_scale/eval/small/etth1.sh
@@ -1,20 +1,20 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=0
+export CUDA_VISIBLE_DEVICES=3
 
 mode=S
 cp=conf/lsf/multi_scale/eval
-cl=5000
+cl=500
 model=moirai_lightning_ckpt
 
 cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl96/checkpoints/epoch_2-step_228.ckpt'
-cpp2='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl192/checkpoints/epoch_1-step_150.ckpt'
-cpp3='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl336/checkpoints/epoch_1-step_146.ckpt'
-cpp4='./outputs/origin/moirai_1.1_R_small/lsf/full/etth1/cl3000_pl720/checkpoints/epoch_0-step_68.ckpt'
+cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t0.1/freeze_ffn/etth1/S/cl500_pl192/checkpoints/epoch_2-step_327.ckpt'
+cpp3='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t0.1/freeze_ffn/etth1/S/cl500_pl336/checkpoints/epoch_1-step_214.ckpt'
+cpp4='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t0.1/freeze_ffn/etth1/S/cl500_pl720/checkpoints/epoch_1-step_204.ckpt'
 
-index=1
-for pl in 96 192 336 720; do
+index=2
+for pl in 192 336 720; do  # 96
   case $index in
     1) cpp=$cpp1 ;;
     2) cpp=$cpp2 ;;
@@ -30,7 +30,7 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=32 \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \

diff --git a/project/lsf/multi_scale/eval/small/etth2.sh b/project/lsf/multi_scale/eval/small/etth2.sh
@@ -5,16 +5,16 @@ export CUDA_VISIBLE_DEVICES=0
 
 mode=S
 cp=conf/lsf/multi_scale/eval
-cl=3000
+cl=500
 model=moirai_lightning_ckpt
 
-cpp1='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl96/checkpoints/epoch_0-step_76.ckpt'
-cpp2='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl192/checkpoints/epoch_1-step_150.ckpt'
-cpp3='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl336/checkpoints/epoch_1-step_146.ckpt'
-cpp4='./outputs/origin/moirai_1.1_R_small/lsf/full/etth2/cl3000_pl720/checkpoints/epoch_0-step_68.ckpt'
+cpp1=''
+cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t1/freeze_ffn/etth2/S/cl500_pl192/checkpoints/epoch_27-step_3052.ckpt'
+cpp3='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t1/freeze_ffn/etth2/S/cl500_pl336/checkpoints/epoch_20-step_2247.ckpt'
+cpp4='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_lr5e-6_t1/freeze_ffn/etth2/S/cl500_pl720/checkpoints/epoch_28-step_2958.ckpt'
 
-index=1
-for pl in 96 192 336 720; do
+index=2
+for pl in 192 336 720; do  # 96
   case $index in
     1) cpp=$cpp1 ;;
     2) cpp=$cpp2 ;;
@@ -30,7 +30,7 @@ for pl in 96 192 336 720; do
     -cp $cp \
     exp_name=$exp_name/$pretrained_model/$ft_pattern  \
     model=$model \
-    model.patch_size=64 \
+    model.patch_size=32 \
     model.context_length=$cl \
     model.checkpoint_path=$cpp \
     model.pretrained_checkpoint_path=ckpt/$pretrained_model.ckpt \

diff --git a/project/lsf/multi_scale/eval/small/ettm1.sh b/project/lsf/multi_scale/eval/small/ettm1.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=3
+export CUDA_VISIBLE_DEVICES=2
 
 mode=S
 cp=conf/lsf/multi_scale/eval
 cl=4000
 model=moirai_lightning_ckpt
 
-cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/ettm1/S/cl4000_pl96/checkpoints/epoch_17-step_7506.ckpt'
-cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/ettm1/S/cl4000_pl192/checkpoints/epoch_29-step_12480.ckpt'
-cpp3=''
-cpp4=''
+cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid/freeze_ffn/ettm1/S/cl4000_pl96/checkpoints/epoch_1-step_834.ckpt'
+cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid/freeze_ffn/ettm1/S/cl4000_pl192/checkpoints/epoch_0-step_416.ckpt'
+cpp3='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid/freeze_ffn/ettm1/S/cl4000_pl336/checkpoints/epoch_0-step_414.ckpt'
+cpp4='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid/freeze_ffn/ettm1/S/cl4000_pl720/checkpoints/epoch_0-step_408.ckpt'
 
 index=1
 for pl in 96 192 336 720; do

diff --git a/project/lsf/multi_scale/eval/small/ettm2.sh b/project/lsf/multi_scale/eval/small/ettm2.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=1
+export CUDA_VISIBLE_DEVICES=2
 
 mode=S
 cp=conf/lsf/multi_scale/eval
 cl=3000
 model=moirai_lightning_ckpt
 
-cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/ettm2/S/cl3000_pl96/checkpoints/epoch_39-step_17240.ckpt'
-cpp2=''
-cpp3=''
-cpp4=''
+cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_t01/freeze_ffn/ettm2/S/cl3000_pl96/checkpoints/epoch_0-step_431.ckpt'
+cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_t01/freeze_ffn/ettm2/S/cl3000_pl192/checkpoints/epoch_0-step_429.ckpt'
+cpp3='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_t01/freeze_ffn/ettm2/S/cl3000_pl336/checkpoints/epoch_0-step_427.ckpt'
+cpp4='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_t01/freeze_ffn/ettm2/S/cl3000_pl720/checkpoints/epoch_0-step_422.ckpt'
 
 index=1
 for pl in 96 192 336 720; do

diff --git a/project/lsf/multi_scale/eval/small/weather.sh b/project/lsf/multi_scale/eval/small/weather.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
 
 export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=3
+export CUDA_VISIBLE_DEVICES=2
 
 mode=S
 cp=conf/lsf/multi_scale/eval
 cl=2000
 model=moirai_lightning_ckpt
 
-cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/weather/S/cl2000_pl96/checkpoints/epoch_9-step_14280.ckpt'
-cpp2='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/weather/S/cl2000_pl192/checkpoints/epoch_10-step_15664.ckpt'
-cpp3='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/weather/S/cl2000_pl336/checkpoints/epoch_3-step_5672.ckpt'
-cpp4='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_1e-5/freeze_ffn/weather/S/cl2000_pl720/checkpoints/epoch_2-step_4206.ckpt'
+cpp1='./outputs/lsf/multi_scale/finetune/moirai_1.0_R_small/weighted_loss_mfc_tid_t1/freeze_ffn/weather/S/cl2000_pl96/checkpoints/epoch_0-step_1428.ckpt'
+cpp2=''
+cpp3=''
+cpp4=''
 
 index=1
 for pl in 96 192 336 720; do

diff --git a/project/lsf/multi_scale/finetune/small/electricity.sh b/project/lsf/multi_scale/finetune/small/electricity.sh
@@ -4,7 +4,7 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=data_weight_lr1e-2_valScaled
 data=electricity
 cl=5000
 ps=64
@@ -33,10 +33,18 @@ for pl in 96 192 336 720; do
   val_data.prediction_length=$pl \
   val_data.mode=${mode} \
   model.lr=1e-5 \
-  model.scale_weight_lr=1e-5 \
-  model.temperature=10 \
+  model.scale_weight_lr=1e-2 \
   trainer.max_epochs=3 \
   trainer.callbacks.1.save_last=true \
   train_dataloader.batch_size=256 \
   trainer.callbacks.'2'.patience=1
+
+
+#  model.lr=1e-5 \
+#  model.scale_weight_lr=1e-5 \
+#  model.temperature=10 \
+#  trainer.max_epochs=3 \
+#  trainer.callbacks.1.save_last=true \
+#  train_dataloader.batch_size=256 \
+#  trainer.callbacks.'2'.patience=1
 done
diff --git a/project/lsf/multi_scale/finetune/small/etth1.sh b/project/lsf/multi_scale/finetune/small/etth1.sh
@@ -4,15 +4,15 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=1;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=weighted_loss_mfc_tid_lr5e-6_t0.1
 data=etth1
-cl=5000
-ps=64
+cl=500
+ps=32
 mode=S
 ft_pattern=freeze_ffn
 
 
-for pl in 96 192 336 720; do
+for pl in 96; do  #  192 336 720
   python -m cli.train \
   -cp $cp \
   exp_name=$exp_name \
@@ -32,7 +32,7 @@ for pl in 96 192 336 720; do
   val_data.context_length=$cl \
   val_data.prediction_length=$pl \
   val_data.mode=${mode} \
-  model.lr=1e-5 \
+  model.lr=5e-6 \
   model.scale_weight_lr=1e-5 \
-  model.temperature=10
+  model.temperature=0.1
 done
diff --git a/project/lsf/multi_scale/finetune/small/etth2.sh b/project/lsf/multi_scale/finetune/small/etth2.sh
@@ -4,10 +4,10 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=1;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=weighted_loss_mfc_tid_lr5e-6_t1
 data=etth2
-cl=3000
-ps=64
+cl=500
+ps=32
 mode=S
 ft_pattern=freeze_ffn
 
@@ -32,7 +32,7 @@ for pl in 96 192 336 720; do
   val_data.context_length=$cl \
   val_data.prediction_length=$pl \
   val_data.mode=${mode} \
-  model.lr=1e-5 \
+  model.lr=5e-6 \
   model.scale_weight_lr=1e-5 \
-  model.temperature=10
+  model.temperature=1
 done
diff --git a/project/lsf/multi_scale/finetune/small/ettm1.sh b/project/lsf/multi_scale/finetune/small/ettm1.sh
@@ -4,7 +4,7 @@ export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=2;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=weighted_loss_mfc_tid
 data=ettm1
 cl=4000
 ps=128
@@ -34,5 +34,5 @@ for pl in 96 192 336 720; do
   val_data.mode=${mode} \
   model.lr=1e-5 \
   model.scale_weight_lr=1e-5 \
-  model.temperature=1
+  model.temperature=0.1
 done
diff --git a/project/lsf/multi_scale/finetune/small/ettm2.sh b/project/lsf/multi_scale/finetune/small/ettm2.sh
@@ -1,18 +1,18 @@
 #!/bin/bash
 
-export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=1;
+export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=2;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=data_weight_lr1e-2_valScaled
 data=ettm2
 cl=3000
 ps=64
 mode=S
 ft_pattern=freeze_ffn
 
 
-for pl in 96 192 336 720; do  #
+for pl in 336 720; do  # 96 192
   python -m cli.train \
   -cp $cp \
   exp_name=$exp_name \
@@ -32,7 +32,10 @@ for pl in 96 192 336 720; do  #
   val_data.context_length=$cl \
   val_data.prediction_length=$pl \
   val_data.mode=${mode} \
-  model.lr=1e-5 \
-  model.scale_weight_lr=1e-5 \
-  model.temperature=10
+  model.lr=5e-6 \
+  model.scale_weight_lr=1e-2
+
+#  model.lr=1e-5 \
+#  model.scale_weight_lr=1e-5 \
+#  model.temperature=0.1
 done
diff --git a/project/lsf/multi_scale/finetune/small/weather.sh b/project/lsf/multi_scale/finetune/small/weather.sh
@@ -1,18 +1,18 @@
 #!/bin/bash
 
-export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=3;
+export HYDRA_FULL_ERROR=1; export CUDA_VISIBLE_DEVICES=0;
 
 model=moirai_1.0_R_small
 cp=conf/lsf/multi_scale/finetune
-exp_name=weighted_loss_mfc_1e-5
+exp_name=weighted_loss_mfc_tid_t1
 data=weather
 cl=2000
 ps=128
 mode=S  # M
 ft_pattern=freeze_ffn
 
 
-for pl in 96 192 336 720; do
+for pl in 96; do   # 192 336 720
   python -m cli.train \
   -cp $cp \
   exp_name=$exp_name \
@@ -36,5 +36,5 @@ for pl in 96 192 336 720; do
   trainer.callbacks."2".monitor=val/PackedMSELoss \
   model.lr=1e-5 \
   model.scale_weight_lr=1e-5 \
-  model.temperature=10
+  model.temperature=1
 done
diff --git a/src/uni2ts/loss/packed/_base.py b/src/uni2ts/loss/packed/_base.py
@@ -108,9 +108,9 @@ def reduce_loss(
         nobs = torch.where(nobs == 0, nobs, 1 / nobs).sum()
         loss = safe_div(loss, tobs * nobs)
         if self.return_loss_batchwise:
-            return loss * mask   # (loss * mask).sum(dim=(-1, -2))
+            return loss * mask
         else:
-            return (loss * mask).sum()
+            return (loss * mask).sum()  # (loss * mask).sum(dim=(-1, -2))
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}()"