reworked config to start from DL1 and add slurm options to better man…

…age jobs requirements

production_configs/20240828_v0.10.9_GC_dec_min_2924_tuned/README.md

@@ -12,11 +12,21 @@
 
 ## lstchain_tune_nsb command
 
+```
 lstchain_tune_nsb --config standard_lstchain_config.json --input-mc /fefs/aswg/data/mc/DL0/LSTProd2/TrainingDataset/Protons/dec_min_2924/sim_telarray/node_corsika_theta_69.813_az_217.303_/output_v1.4/simtel_corsika_theta_69.813_az_217.303_run10.simtel.gz --input-data /fefs/aswg/data/real/DL1/20220502/v0.9/tailcut84/dl1_LST-1.Run08079.0035.h5 -o tuned_lstchain_config.json
+```
 
 ## lstmcpipe config command
 
-lstmcpipe_generate_config PathConfigAllSkyFull --prod_id 20240828_v0.10.9_GC_dec_min_2924_tuned --dec_list dec_min_2924 
+```
+lstmcpipe_generate_config PathConfigAllSkyFullDL1ab --dec_list dec_min_2924 --prod_id 20240906_v0.10.9_GC_dec_min_2924_tuned  --kwargs source_prod_id=20240131_allsky_v0.10.5_all_dec_base
+```
+
+Then used the script below to add time and nice options to each step.
+
+```
+python add_slurm_options.py lstmcpipe_config_2024-09-06_PathConfigAllSkyFullDL1ab.yaml 
+```
 
 ## Why this production is needed?
 

production_configs/20240828_v0.10.9_GC_dec_min_2924_tuned/add_slurm_options.py

@@ -0,0 +1,39 @@
+import sys
+from ruamel.yaml import YAML
+
+
+def add_slurm_options(config_file):
+    yaml = YAML()
+    yaml.preserve_quotes = True
+    config = yaml.load(open(config_file))
+
+    for stage_name, stage in config['stages'].items():
+        for ii, step in enumerate(stage):
+            slurm_options = step.get('extra_slurm_options', {})
+            if 'time' not in slurm_options:
+                if stage_name == 'r0_to_dl1':
+                    slurm_options['time'] = '10:00:00'
+                elif stage_name == 'train_pipe':
+                    slurm_options['time'] = '02-00:00:00'
+                elif stage_name == 'dl1ab':
+                    slurm_options['time'] = '04:00:00'
+                elif stage_name == 'merge_dl1' and ('GammaDiffuse' in step['input'] or 'Protons' in step['input']):
+                    slurm_options['time'] = '08:00:00'
+                else:
+                    slurm_options['partition'] = 'short'
+            if 'nice' not in slurm_options:
+                slurm_options['nice'] = 10
+            stage[ii]['extra_slurm_options'] = slurm_options
+
+    with open(config_file, 'w') as f:
+        yaml.dump(config, f)
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Add Slurm options to each step in the given YML config file.')
+    parser.add_argument('input', help='Path to the input YML config file')
+
+    args = parser.parse_args()
+
+    add_slurm_options(args.input)