Run snakemake -n in CI

config-lngs.yaml → .github/dataflow-config-ci.yaml

@@ -1,6 +1,5 @@
 paths:
-  sandbox_path: /data1/shared/l200-p13/sandbox
-  tier_daq: $_/../daq/generated/tier/daq
+  sandbox_path: $_/sandbox
   tier_raw_blind: ""
 
   workflow: $_/workflow
@@ -13,6 +12,7 @@ paths:
   detector_db: $_/inputs/hardware/detectors
 
   tier: $_/generated/tier
+  tier_daq: $_/generated/tier/daq
   tier_raw: $_/generated/tier/raw
   tier_tcm: $_/generated/tier/tcm
   tier_dsp: $_/generated/tier/dsp
@@ -58,8 +58,6 @@ table_format:
   tcm: hardware_tcm_1
 
 execenv:
-  cmd: apptainer exec
-  arg: /data2/public/prodenv/containers/legendexp_legend-base_latest_20241110203225.sif
   env:
     PRODENV: $PRODENV
     HDF5_USE_FILE_LOCKING: "False"
@@ -69,4 +67,4 @@ execenv:
     PYGAMA_FASTMATH: "false"
     DISABLE_TQDM: "True"
 
-legend_metadata_version: v0.5.7
+legend_metadata_version: refactor

.github/release.yml

@@ -0,0 +1,5 @@
+changelog:
+  exclude:
+    authors:
+      - dependabot
+      - pre-commit-ci

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: lgdo
+name: legend-dataflow
 
 on:
   workflow_dispatch:
@@ -16,8 +16,8 @@ env:
   FORCE_COLOR: 3
 
 jobs:
-  build-and-test:
-    name: Test lgdo with Python
+  unit-tests:
+    name: Run legend-dataflow unit tests
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -39,6 +39,56 @@ jobs:
         run: |
           python -m pytest
 
+  run-dataflow:
+    if: github.event.pull_request.head.repo.fork == false
+    name: Run the Snakemake workflow
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+        os: [ubuntu-latest, macos-13]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set LEGEND_METADATA variable
+        run: |
+          echo "LEGEND_METADATA=$GITHUB_WORKSPACE/inputs" >> $GITHUB_ENV
+      - name: Clone legend-metadata
+        uses: actions/checkout@v4
+        with:
+          repository: legend-exp/legend-metadata
+          ref: refactor # REMOVE ME
+          fetch-depth: 0
+          submodules: recursive
+          token: ${{ secrets.CLONE_LEGEND_METADATA }}
+          path: ${{ env.LEGEND_METADATA }}
+
+      - name: Get dependencies and install legend-dataflow
+        run: |
+          python -m pip install --upgrade uv
+          python -m uv pip install --upgrade .[runprod]
+
+      - name: Set the PRODENV variable
+        run: |
+          echo "PRODENV=$(realpath $GITHUB_WORKSPACE/..)" >> $GITHUB_ENV
+
+      - name: test prodenv scripts
+        run: |
+          cp .github/dataflow-config-ci.yaml ./dataflow-config.yaml
+
+      - name: run workflows in dry-run mode
+        run: |
+          snakemake --workflow-profile workflow/profiles/lngs-build-raw -n all-*-daq.gen
+          snakemake --workflow-profile workflow/profiles/lngs-build-raw -n all-*-raw.gen
+          snakemake --workflow-profile workflow/profiles/lngs -n all-*-evt.gen
+          snakemake --workflow-profile workflow/profiles/lngs -n all-*-skm.gen
+
   test-coverage:
     name: Calculate and upload test coverage
     runs-on: ubuntu-latest
@@ -48,7 +98,7 @@ jobs:
           fetch-depth: 2
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Generate Report
         run: |
@@ -57,3 +107,5 @@ jobs:
           python -m pytest --cov=legenddataflow --cov-report=xml
       - name: Upload Coverage to codecov.io
         uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

dataflow-config.yaml

@@ -0,0 +1,70 @@
+paths:
+  sandbox_path: $_/sandbox
+  tier_raw_blind: ""
+
+  workflow: $_/workflow
+
+  metadata: $_/inputs
+  config: $_/inputs/dataprod/config
+  par_overwrite: $_/inputs/dataprod/overrides
+  chan_map: $_/inputs/hardware/configuration
+  detector_status: $_/inputs/datasets
+  detector_db: $_/inputs/hardware/detectors
+
+  tier: $_/generated/tier
+  tier_daq: $_/generated/tier/daq
+  tier_raw: $_/generated/tier/raw
+  tier_tcm: $_/generated/tier/tcm
+  tier_dsp: $_/generated/tier/dsp
+  tier_hit: $_/generated/tier/hit
+  tier_ann: $_/generated/tier/ann
+  tier_evt: $_/generated/tier/evt
+  tier_psp: $_/generated/tier/psp
+  tier_pht: $_/generated/tier/pht
+  tier_pan: $_/generated/tier/pan
+  tier_pet: $_/generated/tier/pet
+  tier_skm: $_/generated/tier/skm
+
+  par: $_/generated/par
+  par_raw: $_/generated/par/raw
+  par_tcm: $_/generated/par/tcm
+  par_dsp: $_/generated/par/dsp
+  par_hit: $_/generated/par/hit
+  par_evt: $_/generated/par/evt
+  par_psp: $_/generated/par/psp
+  par_pht: $_/generated/par/pht
+  par_pet: $_/generated/par/pet
+
+  plt: $_/generated/plt
+  log: $_/generated/log
+
+  tmp_plt: $_/generated/tmp/plt
+  tmp_log: $_/generated/tmp/log
+  tmp_filelists: $_/generated/tmp/filelists
+  tmp_par: $_/generated/tmp/par
+
+  src: $_/software/python/src
+  install: $_/.snakemake/legend-dataflow/venv
+
+table_format:
+  raw: ch{ch:07d}/raw
+  dsp: ch{ch:07d}/dsp
+  psp: ch{ch:07d}/dsp
+  hit: ch{ch:07d}/hit
+  pht: ch{ch:07d}/hit
+  evt: "{grp}/evt"
+  pet: "{grp}/evt"
+  skm: "{grp}/skm"
+  tcm: hardware_tcm_1
+
+execenv:
+  env:
+    PRODENV: $PRODENV
+    HDF5_USE_FILE_LOCKING: "False"
+    LGDO_BOUNDSCHECK: "false"
+    DSPEED_BOUNDSCHECK: "false"
+    PYGAMA_PARALLEL: "false"
+    PYGAMA_FASTMATH: "false"
+    DISABLE_TQDM: "True"
+
+legend_metadata_version: refactor

pyproject.toml

@@ -57,6 +57,7 @@ dependencies = [
     "pylegendmeta==1.2.0a2",
     "legend-pydataobj>=1.11.6",
     "legend-daq2lh5>=1.4",
+    "pip",
 ]
 
 [project.optional-dependencies]

workflow/profiles/default/config.yaml

@@ -1,4 +1,5 @@
 cores: all
+configfile: dataflow-config.yaml
 snakefile: ./workflow/Snakefile
 keep-going: true
 rerun-incomplete: true

workflow/profiles/lngs-build-raw/config.yaml

@@ -1,7 +1,7 @@
 cores: 30
 resources:
   - mem_swap=3500
-configfile: config-lngs.yaml
+configfile: dataflow-config.yaml
 snakefile: ./workflow/Snakefile-build-raw
 keep-going: true
 rerun-incomplete: true

workflow/profiles/lngs/config.yaml

@@ -2,7 +2,7 @@ cores: 125
 restart-times: 2
 resources:
   - mem_swap=3500
-configfile: config-lngs.yaml
+configfile: dataflow-config.yaml
 snakefile: ./workflow/Snakefile
 keep-going: true
 rerun-incomplete: true

workflow/rules/filelist_gen.smk

@@ -128,9 +128,9 @@ def concat_phy_filenames(config, phy_filenames, tier):
     """
     This function concatenates the files from the same run together
     """
-    fn_pattern = patt.get_pattern(config, tier)
+    fn_pattern = get_pattern(config, tier)
     # group files by run
-    sorted_phy_filenames = patt.run_grouper(phy_filenames)
+    sorted_phy_filenames = run_grouper(phy_filenames)
     phy_filenames = []
 
     for run in sorted_phy_filenames: