Merge pull request #2099 from NNPDF/check_newcd

Execute new commondata generation in CI

.github/workflows/check_newcd.yml

@@ -0,0 +1,34 @@
+# A CI script with github workflow to test the new commondata
+name: Test new commondata
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test-commondata:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Install NNPDF data package 🐍
+        run: pip install ./nnpdf_data/'[filter]'
+      - name: Run the filters 📦
+        shell: bash -l {0}
+        run: |
+          here=$PWD
+          readarray -d '' array < <(find ./nnpdf_data/nnpdf_data/commondata -name "filter.py" -print0)
+          for datname in "${array[@]}"; do dirpath=${datname%/*}; cd $dirpath; python filter.py || exit $?; cd $here; done
+      - name: Check for modified files 🛎️
+        uses: tj-actions/verify-changed-files@v20
+        id: verify-changed-files
+      - name: List all changed files and fail if any 🛎️
+        if: steps.verify-changed-files.outputs.files_changed == 'true'
+        env:
+          CHANGED_FILES: ${{ steps.verify-changed-files.outputs.changed_files }}
+        run: |
+          echo "Changed files: $CHANGED_FILES"
+          exit 1

.github/workflows/fitbot.yml

@@ -10,7 +10,7 @@ on:
 env:
   N3FIT_MAXNREP: 20 # total number of replicas to fit
   POSTFIT_NREP: 16 # requested replicas for postfit
-  REFERENCE_SET: NNBOT-da9fed01b-2024-10-23 # reference set for exact results
+  REFERENCE_SET: NNBOT-73468bfff-2024-11-01 # reference set for exact results
   STABLE_REFERENCE_SET: NNBOT-c0f99b7b3-2024-02-28 # reference set for last tag
   CONDA_PY: 312
   PYTHONHASHSEED: "0"

.github/workflows/regenerate_data.yml

@@ -0,0 +1,37 @@
+# CI script with github workflow to regenerate the data
+name: Regenerate Commondata
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  build:
+    if: contains(github.event.pull_request.labels.*.name, 'regenerate-data')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Install NNPDF data package 🐍
+        run: pip install ./nnpdf_data/'[filter]'
+      - name: Run the filters 📦
+        shell: bash -l {0}
+        run: |
+          here=$PWD
+          readarray -d '' array < <(find ./nnpdf_data/nnpdf_data/commondata -name "filter.py" -print0)
+          for datname in "${array[@]}"; do dirpath=${datname%/*}; cd $dirpath; python filter.py || exit $?; cd $here; done
+      - name: Check for modified files 🛎️
+        uses: tj-actions/verify-changed-files@v20
+        id: verify-changed-files
+      - name: Commit the changed files 🛎️
+        if: steps.verify-changed-files.outputs.files_changed == 'true'
+        shell: bash -l {0}
+        run: |
+          git config user.name "Regenerate Commondata"
+          git config user.email "<>"
+          git add nnpdf_data/nnpdf_data/commondata/*
+          git status
+          git commit -m "Automatically regenerated commondata from PR ${{ github.event.number }}, branch ${{ github.event.pull_request.head.ref }}."
+          git push origin "${{ github.event.pull_request.head.ref }}"

nnpdf_data/nnpdf_data/commondata/ATHENA_NC_105GEV_EP/data.yaml

@@ -1,124 +1,124 @@
 data_central:
-- 0.0030232492667256476
-- 0.002555546843002419
-- 0.0057601463999422215
-- 0.002034588868637695
-- 0.004558456269590243
-- 0.008532582308076646
-- 0.0017279908038945128
-- 0.0033703961980815203
-- 0.006451277170582645
-- 0.010466932984758447
-- 0.0013265331758707064
-- 0.0025590501051209586
-- 0.004795570413674596
-- 0.007735791204031044
-- 0.012224836425224874
-- 0.0010846226765217652
-- 0.0020756968013515716
-- 0.00342249795999119
-- 0.005817355761132647
-- 0.008906418989800935
-- 0.013295759775211337
-- 0.0007582303616102364
-- 0.0014982502854730636
-- 0.002529556455526126
-- 0.004164636624241837
-- 0.006414662909968195
-- 0.010045125193224206
-- 0.014795141156436245
-- 0.0005484376931462637
-- 0.0011719339995933972
-- 0.0019621855490396457
-- 0.0031074112503655248
-- 0.004775355943674676
-- 0.007139891622980082
-- 0.011098209407376257
-- 0.016710878682833046
-- 0.0008882210489093715
-- 0.0015473672218736204
-- 0.002395998266516214
-- 0.0035801901290965304
-- 0.005469990877244589
-- 0.008553882831584961
-- 0.013045288348511439
-- 0.019741213268127285
-- 0.0011170555572520596
-- 0.001732180172703438
-- 0.0029422565201887834
-- 0.0045222497096678695
-- 0.006781903696060579
-- 0.010664131110250361
-- 0.016369598420864023
-- 0.025067783739548977
-- 0.0015564819085647536
-- 0.002393155719935957
-- 0.0034183817956831186
-- 0.0055710852657673655
-- 0.008880682302329709
-- 0.013546789665405769
-- 0.02257514160897749
-- 0.03567920543503588
-- 0.0020013377586011573
-- 0.0031760568752588225
-- 0.004826183395098619
-- 0.007801274867260488
-- 0.01188377690593458
-- 0.0196364088034342
-- 0.03213925202754847
-- 0.051712309848685876
-- 0.0025897154812055444
-- 0.004529742794075346
-- 0.0071350779994061135
-- 0.010636358498966613
-- 0.01793554991512172
-- 0.028207549466742785
-- 0.04751995403767628
-- 0.078399292226318
-- 0.003797811136504016
-- 0.006106445193975674
-- 0.008956449990185172
-- 0.015409760938497722
-- 0.024702970460839312
-- 0.04276543913975987
-- 0.07121694430149718
-- 0.11587349188568133
-- 0.005325119381510992
-- 0.008739860667078495
-- 0.013811984097920824
-- 0.022547749486510168
-- 0.036418678415793396
-- 0.061092464889775336
-- 0.10120334768295079
-- 0.16729706489332236
-- 0.007314426129897138
-- 0.012310565849337556
-- 0.01880548673319385
-- 0.03150322613317545
-- 0.050248794571101964
-- 0.08528986456786831
-- 0.14002591134881992
-- 0.22685366247561983
-- 0.010412613343060003
-- 0.01585900025047944
-- 0.02639526782013252
-- 0.041523343993664856
-- 0.06939374238598457
-- 0.10990318552454333
-- 0.18248513590443896
-- 0.29672896478109023
-- 0.013143953495619778
-- 0.019441475337021664
-- 0.03717737735144933
-- 0.056289104982429596
-- 0.08183214001645585
-- 0.1415165136560526
-- 0.22888822225339417
-- 0.35549348132337844
-- 0.008547674684558856
-- 0.03108709490324683
-- 0.03640459562611305
-- 0.07940695861152511
-- 0.11371150089179818
-- 0.17430883944275216
-- 0.2547218362841646
+- 3.02324927e-03
+- 2.55554684e-03
+- 5.76014640e-03
+- 2.03458887e-03
+- 4.55845627e-03
+- 8.53258231e-03
+- 1.72799080e-03
+- 3.37039620e-03
+- 6.45127717e-03
+- 1.04669330e-02
+- 1.32653318e-03
+- 2.55905011e-03
+- 4.79557041e-03
+- 7.73579120e-03
+- 1.22248364e-02
+- 1.08462268e-03
+- 2.07569680e-03
+- 3.42249796e-03
+- 5.81735576e-03
+- 8.90641899e-03
+- 1.32957598e-02
+- 7.58230362e-04
+- 1.49825029e-03
+- 2.52955646e-03
+- 4.16463662e-03
+- 6.41466291e-03
+- 1.00451252e-02
+- 1.47951412e-02
+- 5.48437693e-04
+- 1.17193400e-03
+- 1.96218555e-03
+- 3.10741125e-03
+- 4.77535594e-03
+- 7.13989162e-03
+- 1.10982094e-02
+- 1.67108787e-02
+- 8.88221049e-04
+- 1.54736722e-03
+- 2.39599827e-03
+- 3.58019013e-03
+- 5.46999088e-03
+- 8.55388283e-03
+- 1.30452883e-02
+- 1.97412133e-02
+- 1.11705556e-03
+- 1.73218017e-03
+- 2.94225652e-03
+- 4.52224971e-03
+- 6.78190370e-03
+- 1.06641311e-02
+- 1.63695984e-02
+- 2.50677837e-02
+- 1.55648191e-03
+- 2.39315572e-03
+- 3.41838180e-03
+- 5.57108527e-03
+- 8.88068230e-03
+- 1.35467897e-02
+- 2.25751416e-02
+- 3.56792054e-02
+- 2.00133776e-03
+- 3.17605688e-03
+- 4.82618340e-03
+- 7.80127487e-03
+- 1.18837769e-02
+- 1.96364088e-02
+- 3.21392520e-02
+- 5.17123098e-02
+- 2.58971548e-03
+- 4.52974279e-03
+- 7.13507800e-03
+- 1.06363585e-02
+- 1.79355499e-02
+- 2.82075495e-02
+- 4.75199540e-02
+- 7.83992922e-02
+- 3.79781114e-03
+- 6.10644519e-03
+- 8.95644999e-03
+- 1.54097609e-02
+- 2.47029705e-02
+- 4.27654391e-02
+- 7.12169443e-02
+- 1.15873492e-01
+- 5.32511938e-03
+- 8.73986067e-03
+- 1.38119841e-02
+- 2.25477495e-02
+- 3.64186784e-02
+- 6.10924649e-02
+- 1.01203348e-01
+- 1.67297065e-01
+- 7.31442613e-03
+- 1.23105658e-02
+- 1.88054867e-02
+- 3.15032261e-02
+- 5.02487946e-02
+- 8.52898646e-02
+- 1.40025911e-01
+- 2.26853662e-01
+- 1.04126133e-02
+- 1.58590003e-02
+- 2.63952678e-02
+- 4.15233440e-02
+- 6.93937424e-02
+- 1.09903186e-01
+- 1.82485136e-01
+- 2.96728965e-01
+- 1.31439535e-02
+- 1.94414753e-02
+- 3.71773774e-02
+- 5.62891050e-02
+- 8.18321400e-02
+- 1.41516514e-01
+- 2.28888222e-01
+- 3.55493481e-01
+- 8.54767468e-03
+- 3.10870949e-02
+- 3.64045956e-02
+- 7.94069586e-02
+- 1.13711501e-01
+- 1.74308839e-01
+- 2.54721836e-01

nnpdf_data/nnpdf_data/commondata/ATHENA_NC_105GEV_EP/filter.py

@@ -1,13 +1,17 @@
 from pathlib import Path
 
 import numpy as np
+import yaml
 
-from nnpdf_data.filter_utils.eic_utils import (
+from nnpdf_data.filter_utils.poldata_utils import (
     fluctuate_data,
     read_central_values,
     read_excel,
     write_data,
 )
+from nnpdf_data.filter_utils.utils import prettify_float
+
+yaml.add_representer(float, prettify_float)
 
 np.random.seed(1234567890)