add job ids in error and output files" (#467)

cta-observatory · May 31, 2024 · 4466a08 · 4466a08
1 parent 36d7497
commit 4466a08
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 10 deletions.
diff --git a/lstmcpipe/scripts/useful_tmp_scripts/clean_corrupted_dl1_dirs_allSky.py b/lstmcpipe/scripts/useful_tmp_scripts/clean_corrupted_dl1_dirs_allSky.py
@@ -41,8 +41,8 @@ def find_test_dirs(config):
 
         node = dir_i.split('/')[-1]
 
-        error_file = f'./clean_dir_{prod_id}_{node}.e'
-        output_file = f'./clean_dir_{prod_id}_{node}.o'
+        error_file = f'./clean_dir_{prod_id}_{node}_%j.e'
+        output_file = f'./clean_dir_{prod_id}_{node}_%j.o'
 
         cmd = f'{source_env} python ./cleandir.py {dir_i}'
 

diff --git a/lstmcpipe/stages/mc_dl2_to_sensitivity.py b/lstmcpipe/stages/mc_dl2_to_sensitivity.py
@@ -92,8 +92,8 @@ def dl2_to_sensitivity(input_paths, output, batch_configuration, wait_jobs_dl1_d
     sbatch_dl2_sens = SbatchLstMCStage(
         "dl2_sens",
         wrap_command=cmd_sens,
-        slurm_error=Path(output).parent.joinpath("job_dl2_to_sensitivity.e"),
-        slurm_output=Path(output).parent.joinpath("job_dl2_to_sensitivity.o"),
+        slurm_error=Path(output).parent.joinpath("job_dl2_to_sensitivity_%j.e"),
+        slurm_output=Path(output).parent.joinpath("job_dl2_to_sensitivity_%j.o"),
         slurm_dependencies=wait_jobs_dl1_dl2,
         extra_slurm_options=extra_slurm_options,
         slurm_account=batch_configuration["slurm_account"],

diff --git a/lstmcpipe/stages/mc_merge_dl1.py b/lstmcpipe/stages/mc_merge_dl1.py
@@ -95,8 +95,8 @@ def merge_dl1(
     sbatch_merge_dl1 = SbatchLstMCStage(
         "merge_dl1",
         wrap_command=cmd,
-        slurm_error=Path(output_file).parent.joinpath("merging-output.e"),
-        slurm_output=Path(output_file).parent.joinpath("merging-output.o"),
+        slurm_error=Path(output_file).parent.joinpath("merging-output_%j.e"),
+        slurm_output=Path(output_file).parent.joinpath("merging-output_%j.o"),
         slurm_dependencies=wait_jobs_split,
         extra_slurm_options=extra_slurm_options,
         slurm_account=batch_configuration["slurm_account"],

diff --git a/lstmcpipe/stages/mc_train.py b/lstmcpipe/stages/mc_train.py
@@ -119,8 +119,8 @@ def batch_plot_rf_features(
         sbatch_rf_feat = SbatchLstMCStage(
             "RF_importance",
             wrap_command=cmd,
-            slurm_error=Path(models_dir).joinpath("job_plot_rf_feat_importance.e").resolve().as_posix(),
-            slurm_output=Path(models_dir).joinpath(models_dir, "job_plot_rf_feat_importance.o").resolve().as_posix(),
+            slurm_error=Path(models_dir).joinpath("job_plot_rf_feat_importance_%j.e").resolve().as_posix(),
+            slurm_output=Path(models_dir).joinpath(models_dir, "job_plot_rf_feat_importance_%j.o").resolve().as_posix(),
             slurm_dependencies=train_jobid,
             slurm_account=batch_configuration["slurm_account"],
             source_environment=batch_configuration["source_environment"],
@@ -198,8 +198,8 @@ def train_pipe(
     sbatch_train_pipe = SbatchLstMCStage(
         "train_pipe",
         wrap_command=cmd,
-        slurm_error=Path(models_dir).joinpath("train_job.e").resolve().as_posix(),
-        slurm_output=Path(models_dir).joinpath("train_job.o").resolve().as_posix(),
+        slurm_error=Path(models_dir).joinpath("train_job_%j.e").resolve().as_posix(),
+        slurm_output=Path(models_dir).joinpath("train_job_%j.o").resolve().as_posix(),
         slurm_dependencies=wait_jobs_dl1,
         extra_slurm_options=extra_slurm_options,
         slurm_account=batch_configuration["slurm_account"],