Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin' into nsb_tuning_test
Browse files Browse the repository at this point in the history
  • Loading branch information
vuillaut committed Sep 3, 2024
2 parents 0c85209 + 53fe232 commit a7ddf5f
Show file tree
Hide file tree
Showing 45 changed files with 30,329 additions and 121 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ jobs:
python-version: ['3.9', '3.10', '3.11']

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

Expand Down Expand Up @@ -63,10 +63,10 @@ jobs:
conda activate ci
pytest --cov=lstmcpipe --cov-report=xml lstmcpipe
- uses: codecov/codecov-action@v1
- uses: codecov/codecov-action@v4

- name: Detect added configs
uses: tj-actions/changed-files@v39
uses: tj-actions/changed-files@v41
id: verify-changed-files
with:
files: |
Expand Down
10 changes: 5 additions & 5 deletions codemeta.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
"license": "https://spdx.org/licenses/MIT",
"codeRepository": "git+https://github.com/cta-observatory/lstmcpipe.git",
"dateCreated": "2019-09-09",
"datePublished": "2024-02-12",
"downloadUrl": "https://github.com/cta-observatory/lstmcpipe/archive/refs/tags/v0.10.4.tar.gz",
"datePublished": "2024-09-02",
"downloadUrl": "https://github.com/cta-observatory/lstmcpipe/archive/refs/tags/v0.10.5.tar.gz",
"issueTracker": "https://github.com/cta-observatory/lstmcpipe/issues",
"name": "lstmcpipe",
"version": "0.10.4",
"softwareVersion": "0.10.4",
"version": "0.10.5",
"softwareVersion": "0.10.5",
"readme": "https://github.com/cta-observatory/lstmcpipe/blob/master/README.rst",
"description": "Scripts to ease the reduction of MC data on the LST cluster at La Palma. With this package, the analysis/creation of R1/DL0/DL1/DL2/IRFs can be orchestrated.",
"applicationCategory": "CTA, LST",
"releaseNotes": "The main change is the introduction of a new PathConfig to train and apply a new model starting from existing DL1b merged files. This allows testing other models hyperparameters and src-dependent analysis starting from existing DL1 files. **Full Changelog**: https://github.com/cta-observatory/lstmcpipe/compare/v0.10.3...v0.10.4",
"releaseNotes": "v0.10.5 introduces the script to generate NSB-grid productions. For bug fixes, see the **full changelog**: https://github.com/cta-observatory/lstmcpipe/compare/v0.10.4...v0.10.5",
"funding": "824064",
"developmentStatus": "active",
"isPartOf": "https://www.cta-observatory.org/",
Expand Down
11 changes: 11 additions & 0 deletions docs/pipeline.rst
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,17 @@ To generate a config for that pipeline, you may run:
lstmcpipe_generate_config PathConfigProd5Trans80 --prod_id whatagreatprod
**IMPORTANT NOTE:** prod5 MC files need the config to set "focal_length_choice": "EQUIVALENT" to be analyzed with ``lstchain >= v0.9``

In the lstchain config, please set:

.. code-block:: json
"source_config": {
"EventSource": {
"focal_length_choice": "EQUIVALENT"
}
}
One can also start back from DL1, applying the dl1ab stage:
Expand Down
2 changes: 1 addition & 1 deletion download_test_data.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

set -eu

TEST_DATA_URL=${TEST_DATA_URL:-https://big-tank.app.tu-dortmund.de/lst-testdata/}
TEST_DATA_URL=${TEST_DATA_URL:-https://cloud.e5.physik.tu-dortmund.de/lst-testdata/}
TEST_DATA_USER=${TEST_DATA_USER:-""}
TEST_DATA_PASSWORD=${TEST_DATA_PASSWORD:-""}

Expand Down
6 changes: 2 additions & 4 deletions environment.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
name: lstmcpipe
channels:
- default
- cta-observatory
- conda-forge
dependencies:
- python>=3.7
- python>=3.9
- setuptools_scm<8.0
- astropy>=4.2
- pip
- pre-commit
- ctapipe~=0.12
- ctapipe
- ruamel.yaml>=0.17
- lstchain
- pyirf
- ctaplot
- scipy~=1.11.4
12 changes: 7 additions & 5 deletions lstmcpipe/config/paths_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,8 +304,10 @@ def path_dict(gamma_part, offset):
'electron_file': self.dl2_output_file('electron'),
},
'output': os.path.join(self.irf_dir(gamma_src_offset=offset), f'irf_{self.prod_id}_{offset}.fits.gz'),
'options': '--point-like' if gamma_part == 'gamma' else '',
'options': '--gh-efficiency 0.7 --theta-containment 0.7 --energy-dependent-gh --energy-dependent-theta '
}
if gamma_part == 'gamma':
d['options'] += ' --point-like'
return d

for gamma_part in ['gamma-diffuse', 'gamma']:
Expand Down Expand Up @@ -533,6 +535,7 @@ def load_pointings(self, join_type='inner'):
Returns
-------
'astropy.table.QTable`
"""
tabs = {}

Expand Down Expand Up @@ -813,18 +816,17 @@ def dl2_to_irfs(self):
paths = []

for pointing in self.pointing_dirs():
paths.append(
{
pp = {
'input': {
'gamma_file': self.dl2_output_file(pointing),
'proton_file': None,
'electron_file': None,
},
'output': os.path.join(self.irf_dir(pointing), f'irf_{self.prod_id}_{pointing}.fits.gz'),
'options': '--point-like',
'options': '--point-like --gh-efficiency 0.7 --theta-containment 0.7 --energy-dependent-gh --energy-dependent-theta ',
'extra_slurm_options': {'mem': '6GB'},
}
)
paths.append(pp)

return paths

Expand Down
51 changes: 4 additions & 47 deletions lstmcpipe/config/pipeline_config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/usr/bin/env python

from ruamel.yaml import YAML
import calendar
import logging

log = logging.getLogger(__name__)
Expand Down Expand Up @@ -33,7 +32,7 @@ def load_config(config_path):

log.info(f'************ - lstMCpipe will be launch using the {config["workflow_kind"]} pipeline:- ************')
log.info(f'\nPROD_ID to be used: {config["prod_id"]}')
log.info("\nStages to be run:\n - " + "\n - ".join(config["stages_to_run"]))
log.info("\nStages to run:\n - " + "\n - ".join(config["stages_to_run"]))

log.info(
"Slurm configuration:"
Expand Down Expand Up @@ -121,61 +120,19 @@ def complete_lstmcpipe_config(loaded_config):
"""
config = loaded_config.copy()

suffix_prod_id = loaded_config.get("prod_id", "v00")
workflow_kind = loaded_config["workflow_kind"]

# TODO ??
# # to locate the source dl1 files
# dl1_reference_id = loaded_config.get("dl1_reference_id")
# # Full path to an observed dl1 file
# dl1_noise_tune_data_run = loaded_config.get("dl1_noise_tune_data_run")
# dl1_noise_tune_mc_run = loaded_config.get("dl1_noise_tune_mc_run")

# Prod_id syntax
t = calendar.datetime.date.today()
year, month, day = f"{t.year:04d}", f"{t.month:02d}", f"{t.day:02d}"
if workflow_kind == "lstchain":

import lstchain

base_prod_id = f"{year}{month}{day}_v{lstchain.__version__}"

elif workflow_kind == "ctapipe":

import ctapipe

base_prod_id = f"{year}{month}{day}_vctapipe{ctapipe.__version__}"

elif workflow_kind == "hiperta": # RTA

# TODO parse version from hiPeRTA module
import lstchain

base_prod_id = f"{year}{month}{day}_vRTA420_v{lstchain.__version__}"

# Create the final config structure to be passed to the pipeline
# 1 - Prod_id
prod_type = f"_{loaded_config['prod_type']}" if 'prod_type' in loaded_config else ''
suffix_id = "{}_{}".format(prod_type, suffix_prod_id)
config["prod_id"] = base_prod_id + suffix_id

# 2 - Parse source environment correctly
# Parse source environment correctly
src_env = (
f"source {loaded_config['source_environment']['source_file']}; "
f"conda activate {loaded_config['source_environment']['conda_env']}; "
)
# 2.1 - Parse slurm user config account
# 1 - Parse slurm user config account
slurm_account = loaded_config.get("slurm_config", {}).get("user_account", "")

# 2.2 - Create a dict for all env configuration and slurm configuration (batch arguments)
# 2 - Create a dict for all env configuration and slurm configuration (batch arguments)
config["batch_config"] = {
"source_environment": src_env,
"slurm_account": slurm_account,
}

# TODO ??
# config["dl1_reference_id"] = dl1_reference_id
# config["dl1_noise_tune_data_run"] = dl1_noise_tune_data_run
# config["dl1_noise_tune_mc_run"] = dl1_noise_tune_mc_run

return config
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,7 @@ def dump_plus_copy_node_to_create_new_table(
astropy_table_to_copy : astropy table to be copied
newparent_pointer : newparent copy_node parameter
newname_pointer : newname copy_node parameter
tmp_name : [str] flag to identify the temportal table and make it unique (necessary when simultaneous reorganizers
are run in the same dir)
tmp_name : [str] flag to identify the temportal table and make it unique (necessary when simultaneous reorganizers are run in the same dir)
overwrite : overwrite parameter of the copy_node method
"""
input_filename = input_filename.split("___")[0]
Expand Down
147 changes: 147 additions & 0 deletions lstmcpipe/scripts/generate_nsb_levels_configs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
import subprocess
import json
import logging
from lstchain.io.config import get_mc_config
import argparse
from datetime import date

BASE_LSTCHAIN_MC_CONFIG = get_mc_config()

# Configure logging
logging.basicConfig(level=logging.INFO) # Set the default log level to INFO
logger = logging.getLogger(__name__)


def build_argparser():
"""
Build the argument parser for the script.
Returns:
argparse.ArgumentParser: The argument parser object.
"""
parser = argparse.ArgumentParser(description="Generate a set of lstchain and lstmcpipe configuration files for different nsb tuning ratios.")
parser.add_argument("--nsb_ratios", "-nsb",
nargs="+", type=float, default=None,
help="List of nsb tuning ratios. If not provided, no NSB tuning is applied.")

parser.add_argument(
"--dec_list",
nargs="+",
help="List of declination values",
default=[
"dec_2276",
"dec_3476",
"dec_4822",
"dec_6166",
# "dec_6166_high_density",
"dec_6676",
"dec_931",
"dec_min_1802",
"dec_min_2924",
"dec_min_413"
],
)

return parser


def lstchain_config_name(nsb_tuning_ratio):
"""
Generate the name of the lstchain configuration file based on the given nsb_tuning_ratio.
Parameters:
nsb_tuning_ratio (float): The nsb tuning ratio.
Returns:
str: The name of the lstchain configuration file.
"""
if nsb_tuning_ratio == 0 or nsb_tuning_ratio is None:
return "lstchain_config.json"
else:
return f"lstchain_config_nsb{nsb_tuning_ratio}.json"


def dump_lstchain_nsb_config(nsb_tuning_ratio):
"""
Dump the lstchain configuration file with the given nsb_tuning_ratio.
Parameters:
nsb_tuning_ratio (float): The nsb tuning ratio.
"""
new_config = BASE_LSTCHAIN_MC_CONFIG.copy()
if nsb_tuning_ratio == 0 or nsb_tuning_ratio is None:
new_config["waveform_nsb_tuning"]["nsb_tuning"] = False
else:
new_config["waveform_nsb_tuning"]["nsb_tuning"] = True
new_config["waveform_nsb_tuning"]["nsb_tuning_ratio"] = nsb_tuning_ratio
json_filename = lstchain_config_name(nsb_tuning_ratio)
with open(json_filename, 'w') as f:
json.dump(new_config, f, indent=4)
logger.info(f"Dumped lstchain configuration file: {json_filename}")


def prod_id(nsb_tuning_ratio):
"""
Generate the prod ID based on the given nsb_tuning_ratio.
Parameters:
nsb_tuning_ratio (float): The nsb tuning ratio.
Returns:
str: The product ID.
"""
return f"{date.today()}_allsky_nsb_tuning_{nsb_tuning_ratio}"


def lstmcpipe_config_filename(nsb_tuning_ratio):
"""
Generate the name of the lstmcpipe configuration file based on the given nsb_tuning_ratio.
Parameters:
nsb_tuning_ratio (float): The nsb tuning ratio.
Returns:
str: The name of the lstmcpipe configuration file.
"""
if nsb_tuning_ratio == 0 or nsb_tuning_ratio is None:
return "lstmcpipe_config.json"
else:
return f"lstmcpipe_config_nsb{nsb_tuning_ratio}.json"


def main():
"""
Dump the lstchain and lstmcpipe configuration files for the given nsb_tuning_ratios.
"""
parser = build_argparser()
args = parser.parse_args()

dec_list = " ".join(args.dec_list)
nsb_tuning_ratios = args.nsb_ratios

if nsb_tuning_ratios is None:
nsb_tuning_ratios = [None]
for nsb_tuning_ratio in nsb_tuning_ratios:
logger.info(f"Working on ratio {nsb_tuning_ratio}")
dump_lstchain_nsb_config(nsb_tuning_ratio)
tmp_lstchain_config = "tmp_lstchain_config.json"
command = [
"lstmcpipe_generate_config",
"PathConfigAllSkyFull",
"--prod_id",
prod_id(nsb_tuning_ratio),
"-o",
lstmcpipe_config_filename(nsb_tuning_ratio),
"--dec_list",
dec_list,
"--lstchain_conf",
tmp_lstchain_config
]
subprocess.run(command, check=True)
# Delete tmp_lstchain_config (the lstchain configs with nsb tuning are already dumped)
subprocess.run(["rm", tmp_lstchain_config], check=True)
logger.info(f"Generated lstmcpipe configuration file: {lstmcpipe_config_filename(nsb_tuning_ratio)}")


if __name__ == "__main__":
main()
Loading

0 comments on commit a7ddf5f

Please sign in to comment.