Skip to content

Commit

Permalink
Merge pull request #29 from erin717/master
Browse files Browse the repository at this point in the history 
Add scripts for running LQ calibration
  • Loading branch information
@ggmarshall
ggmarshall authored Dec 5, 2023
2 parents d8da22d + 57b7807 commit d0bfb54
Show file tree
Hide file tree
Showing 4 changed files with 812 additions and 10 deletions.
56 changes: 53 additions & 3 deletions rules/hit.smk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,13 @@ rule build_aoe_calibration:
datatype="cal",
channel="{channel}",
output:
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit")),
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit", "aoe_cal")),
aoe_results=temp(
get_pattern_pars_tmp_channel(setup, "hit", "objects", extension="pkl")
get_pattern_pars_tmp_channel(
setup, "hit", "aoe_cal_objects", extension="pkl"
)
),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit")),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit", "aoe_cal")),
log:
get_pattern_log_channel(setup, "pars_hit_aoe_cal"),
group:
Expand All @@ -115,6 +117,54 @@ rule build_aoe_calibration:
"{input.files}"


# This rule builds the lq calibration using the calibration dsp files
rule build_lq_calibration:
input:
files=os.path.join(
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
),
tcm_filelist=os.path.join(
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-tcm.filelist"
),
ecal_file=get_pattern_pars_tmp_channel(setup, "hit", "aoe_cal"),
eres_file=get_pattern_pars_tmp_channel(
setup, "hit", "aoe_cal_objects", extension="pkl"
),
inplots=get_pattern_plts_tmp_channel(setup, "hit", "aoe_cal"),
params:
timestamp="{timestamp}",
datatype="cal",
channel="{channel}",
output:
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "hit")),
lq_results=temp(
get_pattern_pars_tmp_channel(setup, "hit", "objects", extension="pkl")
),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "hit")),
log:
get_pattern_log_channel(setup, "pars_hit_lq_cal"),
group:
"par-hit"
resources:
runtime=300,
shell:
"{swenv} python3 -B "
f"{workflow.source_path('../scripts/pars_hit_lq.py')} "
"--log {log} "
"--configs {configs} "
"--datatype {params.datatype} "
"--timestamp {params.timestamp} "
"--inplots {input.inplots} "
"--channel {params.channel} "
"--lq_results {output.lq_results} "
"--eres_file {input.eres_file} "
"--hit_pars {output.hit_pars} "
"--plot_file {output.plot_file} "
"--tcm_filelist {input.tcm_filelist} "
"--ecal_file {input.ecal_file} "
"{input.files}"


rule build_pars_hit:
input:
lambda wildcards: read_filelist_pars_cal_channel(wildcards, "hit"),
Expand Down
181 changes: 174 additions & 7 deletions rules/pht.smk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ rule build_per_energy_calibration:
timestamp="{timestamp}",
datatype="cal",
channel="{channel}",
tier = "pht",
tier="pht",
output:
ecal_file=temp(get_pattern_pars_tmp_channel(setup, "pht", "energy_cal")),
results_file=temp(
Expand Down Expand Up @@ -351,6 +351,7 @@ for key, dataset in part.datasets.items():
partition,
key,
tier="pht",
name="aoecal",
)
],
aoe_results=[
Expand All @@ -360,7 +361,7 @@ for key, dataset in part.datasets.items():
partition,
key,
tier="pht",
name="objects",
name="aoecal_objects",
extension="pkl",
)
],
Expand All @@ -371,6 +372,7 @@ for key, dataset in part.datasets.items():
partition,
key,
tier="pht",
name="aoecal",
)
],
log:
Expand All @@ -379,7 +381,7 @@ for key, dataset in part.datasets.items():
partition,
key,
"pht",
name="par_pht",
name="par_pht_aoe",
),
group:
"par-pht"
Expand Down Expand Up @@ -413,7 +415,7 @@ for key, dataset in part.datasets.items():

# Merged energy and a/e supercalibrations to reduce number of rules as they have same inputs/outputs
# This rule builds the a/e calibration using the calibration dsp files for the whole partition
rule build_pht_super_calibrations:
rule build_pht_aoe_calibrations:
input:
files=os.path.join(
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
Expand All @@ -431,11 +433,13 @@ rule build_pht_super_calibrations:
channel="{channel}",
timestamp="{timestamp}",
output:
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht")),
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht", "aoecal")),
aoe_results=temp(
get_pattern_pars_tmp_channel(setup, "pht", "objects", extension="pkl")
get_pattern_pars_tmp_channel(
setup, "pht", "objeaoecal_objectscts", extension="pkl"
)
),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht")),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht", "aoecal")),
log:
get_pattern_log_channel(setup, "pars_pht_aoe_cal"),
group:
Expand Down Expand Up @@ -470,3 +474,166 @@ for key, items in ordered.items():
rule_order_list += [item.name for item in items]
rule_order_list.append(fallback_pht_rule.name)
workflow._ruleorder.add(*rule_order_list) # [::-1]

part_pht_rules = {}
for key, dataset in part.datasets.items():
for partition in dataset.keys():

rule:
input:
files=part.get_filelists(partition, key, "dsp"),
tcm_files=part.get_filelists(partition, key, "tcm"),
ecal_file=part.get_par_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
name="aoecal",
),
eres_file=part.get_par_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
name="aoecal_objects",
extension="pkl",
),
inplots=part.get_plt_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
name="aoecal",
),
params:
datatype="cal",
channel="{channel}",
timestamp=part.get_timestamp(
f"{par_pht_path(setup)}/validity.jsonl", partition, key, tier="pht"
),
output:
hit_pars=[
temp(file)
for file in part.get_par_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
)
],
lq_results=[
temp(file)
for file in part.get_par_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
name="objects",
extension="pkl",
)
],
plot_file=[
temp(file)
for file in part.get_plt_files(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
tier="pht",
)
],
log:
part.get_log_file(
f"{par_pht_path(setup)}/validity.jsonl",
partition,
key,
"pht",
name="par_pht_lq",
),
group:
"par-pht"
resources:
mem_swap=75,
runtime=300,
shell:
"{swenv} python3 -B "
f"{basedir}/../scripts/pars_pht_lqcal.py "
"--log {log} "
"--configs {configs} "
"--datatype {params.datatype} "
"--timestamp {params.timestamp} "
"--inplots {input.inplots} "
"--channel {params.channel} "
"--lq_results {output.lq_results} "
"--eres_file {input.eres_file} "
"--hit_pars {output.hit_pars} "
"--plot_file {output.plot_file} "
"--ecal_file {input.ecal_file} "
"--tcm_filelist {input.tcm_files} "
"--input_files {input.files}"

# fix_name(f"{key}-{partition}")

if key in part_pht_rules:
part_pht_rules[key].append(list(workflow.rules)[-1])
else:
part_pht_rules[key] = [list(workflow.rules)[-1]]


# This rule builds the lq calibration using the calibration dsp files for the whole partition
rule build_pht_lq_calibration:
input:
files=os.path.join(
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-dsp.filelist"
),
tcm_filelist=os.path.join(
filelist_path(setup), "all-{experiment}-{period}-{run}-cal-tcm.filelist"
),
ecal_file=get_pattern_pars_tmp_channel(setup, "pht", "aoecal"),
eres_file=get_pattern_pars_tmp_channel(
setup, "pht", "aoecal_objects", extension="pkl"
),
inplots=get_pattern_plts_tmp_channel(setup, "pht", "aoecal"),
params:
datatype="cal",
channel="{channel}",
timestamp="{timestamp}",
output:
hit_pars=temp(get_pattern_pars_tmp_channel(setup, "pht")),
lq_results=temp(
get_pattern_pars_tmp_channel(setup, "pht", "objects", extension="pkl")
),
plot_file=temp(get_pattern_plts_tmp_channel(setup, "pht")),
log:
get_pattern_log_channel(setup, "pars_pht_lq_cal"),
group:
"par-pht"
resources:
mem_swap=60,
runtime=300,
shell:
"{swenv} python3 -B "
f"{basedir}/./scripts/pars_pht_lqcal.py "
"--log {log} "
"--configs {configs} "
"--datatype {params.datatype} "
"--timestamp {params.timestamp} "
"--inplots {input.inplots} "
"--channel {params.channel} "
"--lq_results {output.lq_results} "
"--eres_file {input.eres_file} "
"--hit_pars {output.hit_pars} "
"--plot_file {output.plot_file} "
"--ecal_file {input.ecal_file} "
"--tcm_filelist {input.tcm_filelist} "
"--input_files {input.files}"


fallback_pht_rule = list(workflow.rules)[-1]

rule_order_list = []
ordered = OrderedDict(part_pht_rules)
ordered.move_to_end("default")
for key, items in ordered.items():
rule_order_list += [item.name for item in items]
rule_order_list.append(fallback_pht_rule.name)
workflow._ruleorder.add(*rule_order_list) # [::-1]
Loading

0 comments on commit d0bfb54

Please sign in to comment.