From 93f9cda52e8f3e55ac6d053f0cd3ac7d7586f57f Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Fri, 5 May 2023 17:07:41 -0400 Subject: [PATCH 01/13] #224 Added basic conda build recipe and build script. "conda build ." worked on the workstations, and conda install worked on the thin client. --- build.sh | 4 ++++ meta.yaml | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 build.sh create mode 100644 meta.yaml diff --git a/build.sh b/build.sh new file mode 100644 index 00000000..93e49ee6 --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +autoreconf -i +./configure --prefix=$PREFIX --disable-ocean-model-grid-generator +make -j +make install diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 00000000..a4dfc742 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,22 @@ +package: + name: fre-nctools + version: 2022.02.01 + +source: + path: . + +build: + number: 0 + +requirements: + build: + - libnetcdf + - netcdf-fortran + run: + +test: + commands: + +about: + license: LGPL-3.0 + summary: Tools for manipulating and creating netCDF inputs for FMS managed models From 5f6164808dc6ec3c701a9e7a0f3e2b331ada1b6e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 16:00:09 -0500 Subject: [PATCH 02/13] add build_conda --- .github/workflows/build_conda.yml | 36 +++++++++++++++++++++++++++++++ meta.yaml | 1 + 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/build_conda.yml diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml new file mode 100644 index 00000000..90fbf65f --- /dev/null +++ b/.github/workflows/build_conda.yml @@ -0,0 +1,36 @@ +name: build_conda +on: + pull_request: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + with_mpi: ['','--with-mpi'] + enable_quad_precision: ['', '--enable-quad-precision'] + + container: + image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 + env: + MPI: ${{ matrix.with_mpi }} + QUAD_P: ${{ matrix.enable_quad_precision }} + + steps: + - name: Checkout Files + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Run Conda to Build + run: | + # append the reqd channels + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + + # install conda-build and conda-verify + conda install conda-build conda-verify + + # conda build + conda build . diff --git a/meta.yaml b/meta.yaml index a4dfc742..86a74dcd 100644 --- a/meta.yaml +++ b/meta.yaml @@ -16,6 +16,7 @@ requirements: test: commands: + - make check-expensive about: license: LGPL-3.0 From 5df2f0b8df0d1d9034649579dd39c1f2051608ea Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 16:13:55 -0500 Subject: [PATCH 03/13] disable workflows i am not touching- add jinja to meta.yaml to grab git tag from repo for package building, disable varios combos for build_conda workflow for now --- .github/workflows/build_conda.yml | 14 ++--- .github/workflows/distcheck.yml | 74 +++++++++++++------------- .github/workflows/main.yml | 70 ++++++++++++------------- .github/workflows/main_expensive.yml | 78 ++++++++++++++-------------- meta.yaml | 2 +- 5 files changed, 119 insertions(+), 119 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 90fbf65f..d644dac8 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -6,16 +6,16 @@ on: jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - with_mpi: ['','--with-mpi'] - enable_quad_precision: ['', '--enable-quad-precision'] +# strategy: +# matrix: +# with_mpi: ['','--with-mpi'] +# enable_quad_precision: ['', '--enable-quad-precision'] container: image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 - env: - MPI: ${{ matrix.with_mpi }} - QUAD_P: ${{ matrix.enable_quad_precision }} +# env: +# MPI: ${{ matrix.with_mpi }} +# QUAD_P: ${{ matrix.enable_quad_precision }} steps: - name: Checkout Files diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml index c583b9c0..4229ca95 100644 --- a/.github/workflows/distcheck.yml +++ b/.github/workflows/distcheck.yml @@ -3,40 +3,40 @@ # CI testing for the FRE-NCtools repo, builds and runs unit tests # image dockerfile is maintained here: # https://gitlab.gfdl.noaa.gov/fre/hpc-me -name: FRE-NCtools CI -on: - workflow_run: - workflows: ["FRE-NCtools Check Expensive"] - types: - - completed -jobs: - CI: - runs-on: ubuntu-latest - strategy: - matrix: - with_mpi: ['','--with-mpi'] - enable_quad_precision: ['', '--enable-quad-precision'] - container: - image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 - env: - MPI: ${{ matrix.with_mpi }} - QUAD_P: ${{ matrix.enable_quad_precision }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Configure - run: | - mkdir build && cd build - autoreconf -i ../configure.ac - ../configure $MPI $QUAD_P || cat config.log - - name: Build distribution with check - run: make DISTCHECK_CONFIGURE_FLAGS="$MPI $QUAD_P" -C build -j - - name: Save log file on failure - uses: actions/upload-artifact@v4.4.0 - if: failure() - with: - name: test-suites - path: | - build/tests/test-suite.log +#name: FRE-NCtools CI +#on: +# workflow_run: +# workflows: ["FRE-NCtools Check Expensive"] +# types: +# - completed +#jobs: +# CI: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# with_mpi: ['','--with-mpi'] +# enable_quad_precision: ['', '--enable-quad-precision'] +# container: +# image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 +# env: +# MPI: ${{ matrix.with_mpi }} +# QUAD_P: ${{ matrix.enable_quad_precision }} +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# - name: Configure +# run: | +# mkdir build && cd build +# autoreconf -i ../configure.ac +# ../configure $MPI $QUAD_P || cat config.log +# - name: Build distribution with check +# run: make DISTCHECK_CONFIGURE_FLAGS="$MPI $QUAD_P" -C build -j +# - name: Save log file on failure +# uses: actions/upload-artifact@v4.4.0 +# if: failure() +# with: +# name: test-suites +# path: | +# build/tests/test-suite.log diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60f07dcf..bc8c62ce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,38 +3,38 @@ # CI testing for the FRE-NCtools repo, builds and runs unit tests # image dockerfile is maintained here: # https://gitlab.gfdl.noaa.gov/fre/hpc-me -name: FRE-NCtools CI -on: [push, pull_request] -jobs: - CI: - runs-on: ubuntu-latest - strategy: - matrix: - with_mpi: ['','--with-mpi'] - enable_quad_precision: ['', '--enable-quad-precision'] - container: - image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 - env: - MPI: ${{ matrix.with_mpi }} - QUAD_P: ${{ matrix.enable_quad_precision }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Configure - run: | - mkdir build && cd build - autoreconf -i ../configure.ac - ../configure $MPI $QUAD_P || cat config.log - - name: Build - run: make -C build -j - - name: Run most tests (skip the slow ones) - run: make -C build -j check - - name: Save log file on failure - uses: actions/upload-artifact@v4.4.0 - if: failure() - with: - name: test-suites - path: | - build/tests/test-suite.log +#name: FRE-NCtools CI +#on: [push, pull_request] +#jobs: +# CI: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# with_mpi: ['','--with-mpi'] +# enable_quad_precision: ['', '--enable-quad-precision'] +# container: +# image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 +# env: +# MPI: ${{ matrix.with_mpi }} +# QUAD_P: ${{ matrix.enable_quad_precision }} +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# - name: Configure +# run: | +# mkdir build && cd build +# autoreconf -i ../configure.ac +# ../configure $MPI $QUAD_P || cat config.log +# - name: Build +# run: make -C build -j +# - name: Run most tests (skip the slow ones) +# run: make -C build -j check +# - name: Save log file on failure +# uses: actions/upload-artifact@v4.4.0 +# if: failure() +# with: +# name: test-suites +# path: | +# build/tests/test-suite.log diff --git a/.github/workflows/main_expensive.yml b/.github/workflows/main_expensive.yml index c98c4fca..d6ed8e18 100644 --- a/.github/workflows/main_expensive.yml +++ b/.github/workflows/main_expensive.yml @@ -3,42 +3,42 @@ # CI testing for the FRE-NCtools repo, builds and runs unit tests # image dockerfile is maintained here: # https://gitlab.gfdl.noaa.gov/fre/hpc-me -name: FRE-NCtools Check Expensive -on: - workflow_run: - workflows: ["FRE-NCtools CI"] - types: - - completed -jobs: - CI: - runs-on: ubuntu-latest - strategy: - matrix: - with_mpi: ['','--with-mpi'] - enable_quad_precision: ['', '--enable-quad-precision'] - container: - image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 - env: - MPI: ${{ matrix.with_mpi }} - QUAD_P: ${{ matrix.enable_quad_precision }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Configure - run: | - mkdir build && cd build - autoreconf -i ../configure.ac - ../configure $MPI $QUAD_P || cat config.log - - name: Build - run: make -C build -j - - name: Run most tests (skip the slow ones) - run: make -C build check-very-expensive - - name: Save log file on failure - uses: actions/upload-artifact@v4.4.0 - if: failure() - with: - name: test-suites - path: | - build/tests/test-suite.log +#name: FRE-NCtools Check Expensive +#on: +# workflow_run: +# workflows: ["FRE-NCtools CI"] +# types: +# - completed +#jobs: +# CI: +# runs-on: ubuntu-latest +# strategy: +# matrix: +# with_mpi: ['','--with-mpi'] +# enable_quad_precision: ['', '--enable-quad-precision'] +# container: +# image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 +# env: +# MPI: ${{ matrix.with_mpi }} +# QUAD_P: ${{ matrix.enable_quad_precision }} +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# submodules: recursive +# - name: Configure +# run: | +# mkdir build && cd build +# autoreconf -i ../configure.ac +# ../configure $MPI $QUAD_P || cat config.log +# - name: Build +# run: make -C build -j +# - name: Run most tests (skip the slow ones) +# run: make -C build check-very-expensive +# - name: Save log file on failure +# uses: actions/upload-artifact@v4.4.0 +# if: failure() +# with: +# name: test-suites +# path: | +# build/tests/test-suite.log diff --git a/meta.yaml b/meta.yaml index 86a74dcd..7b2f175a 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,6 +1,6 @@ package: name: fre-nctools - version: 2022.02.01 + version: '{{ environ.get("GIT_DESCRIBE_TAG", data.get("version")) }}' source: path: . From 7ea36da4b7f3b02aa481d488b0a46d37fa516e6d Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 16:16:51 -0500 Subject: [PATCH 04/13] use the fre-cli container with miniconda installed instead --- .github/workflows/build_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index d644dac8..7779bb86 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -12,7 +12,8 @@ jobs: # enable_quad_precision: ['', '--enable-quad-precision'] container: - image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 +# image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 # env: # MPI: ${{ matrix.with_mpi }} # QUAD_P: ${{ matrix.enable_quad_precision }} From 9c80fb8014643ee6457e6773cfe57e8c767bb4da Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 16:20:38 -0500 Subject: [PATCH 05/13] remove git tag jinja2 reference... apparently that needs a setup.py around to work. --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 7b2f175a..a4120596 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,6 +1,6 @@ package: name: fre-nctools - version: '{{ environ.get("GIT_DESCRIBE_TAG", data.get("version")) }}' + version: conda.alpha source: path: . From 9719b46a1bf08352b5640ce6178038e0cea0aa97 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 16:29:19 -0500 Subject: [PATCH 06/13] comment out matrix running for now- add appropriate conda-forge package dependencies --- .github/workflows/build_conda.yml | 8 -------- meta.yaml | 5 +++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 7779bb86..1965b765 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -6,17 +6,9 @@ on: jobs: build: runs-on: ubuntu-latest -# strategy: -# matrix: -# with_mpi: ['','--with-mpi'] -# enable_quad_precision: ['', '--enable-quad-precision'] container: image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 -# image: ghcr.io/noaa-gfdl/fre-nctools-ci-rocky-gnu:14.2.0_v2 -# env: -# MPI: ${{ matrix.with_mpi }} -# QUAD_P: ${{ matrix.enable_quad_precision }} steps: - name: Checkout Files diff --git a/meta.yaml b/meta.yaml index a4120596..e70e723e 100644 --- a/meta.yaml +++ b/meta.yaml @@ -10,8 +10,9 @@ build: requirements: build: - - libnetcdf - - netcdf-fortran + - conda-forge::libnetcdf + - conda-forge::netcdf-fortran + - conda-forge::nco run: test: From f59330dddf85867f08d16dedcfd67fd8780d841c Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 17:16:47 -0500 Subject: [PATCH 07/13] update build reqs- adjust build target for hdf5 library, add openmp --- meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta.yaml b/meta.yaml index e70e723e..4f7184bc 100644 --- a/meta.yaml +++ b/meta.yaml @@ -13,6 +13,8 @@ requirements: - conda-forge::libnetcdf - conda-forge::netcdf-fortran - conda-forge::nco + - conda-forge::hdf5=1.14.*=mpi* + - conda-forge::openmp run: test: From 481a1bdf1102e756822558ac010973671eae2888 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 30 Jan 2025 17:26:50 -0500 Subject: [PATCH 08/13] add tcsh --- meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/meta.yaml b/meta.yaml index 4f7184bc..cfc71f2d 100644 --- a/meta.yaml +++ b/meta.yaml @@ -15,6 +15,7 @@ requirements: - conda-forge::nco - conda-forge::hdf5=1.14.*=mpi* - conda-forge::openmp + - conda-forge::tcsh run: test: From 6298e24e6b16e7ea8b018ea5b75acb84a91a1836 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:25:33 -0500 Subject: [PATCH 09/13] Create split_ncvars.pl.1 --- man/split_ncvars.pl.1 | 1 + 1 file changed, 1 insertion(+) create mode 100644 man/split_ncvars.pl.1 diff --git a/man/split_ncvars.pl.1 b/man/split_ncvars.pl.1 new file mode 100644 index 00000000..4e505f9f --- /dev/null +++ b/man/split_ncvars.pl.1 @@ -0,0 +1 @@ +.so man1/split_ncvars.pl From dde7244436f8ceaa78b2d26e2b2f43e6cc72bddf Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:43:55 -0500 Subject: [PATCH 10/13] Update Makefile.am try making the if WITH_ASCIIDOC filelist line up a bit better with whats currently present --- man/Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/man/Makefile.am b/man/Makefile.am index 72e38103..4b74e9e5 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -44,6 +44,7 @@ EXTRA_DIST = \ river_regrid.txt \ scatter-ncc.txt \ split_ncvars.pl.txt \ + split_ncvars.pl.1 \ timavg.txt \ timavg.csh.1 \ time_average.txt \ @@ -80,6 +81,7 @@ man_MANS += \ plevel.sh.1 \ river_regrid.1 \ scatter-ncc.1 \ + split_ncvars.pl.txt \ split_ncvars.pl.1 \ timavg.1 \ timavg.csh.1 \ From 5c8c20b10c43cf2aa25df3c6b879db415da818ee Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:48:17 -0500 Subject: [PATCH 11/13] Update build.sh limit build make command to two jobs at most --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 93e49ee6..93db9d50 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ autoreconf -i ./configure --prefix=$PREFIX --disable-ocean-model-grid-generator -make -j +make -j 2 make install From 3415a9945467cde379bacc0e64ddaea6bff120e6 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 10 Feb 2025 16:54:17 -0500 Subject: [PATCH 12/13] guard against lack of split_ncvars man page in conda package, at autoconf and automake, adjust make check call --- build.sh | 2 +- meta.yaml | 7 +++++-- src/Makefile.am | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 93db9d50..93e49ee6 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ autoreconf -i ./configure --prefix=$PREFIX --disable-ocean-model-grid-generator -make -j 2 +make -j make install diff --git a/meta.yaml b/meta.yaml index cfc71f2d..7ce08911 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,6 +1,6 @@ package: name: fre-nctools - version: conda.alpha + version: alpha source: path: . @@ -10,17 +10,20 @@ build: requirements: build: + - conda-forge::autoconf + - conda-forge::automake - conda-forge::libnetcdf - conda-forge::netcdf-fortran - conda-forge::nco - conda-forge::hdf5=1.14.*=mpi* - conda-forge::openmp + - conda-forge::mpich - conda-forge::tcsh run: test: commands: - - make check-expensive + - make check RUN_EXPENSIVE_TESTS=yes about: license: LGPL-3.0 diff --git a/src/Makefile.am b/src/Makefile.am index 2aa22217..2b5eb8e9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -213,4 +213,4 @@ split_ncvars.pl: split_ncvars/split_ncvars.pl.in Makefile install-exec-hook: ln $(DESTDIR)$(bindir)/split_ncvars.pl $(DESTDIR)$(bindir)/split_ncvars install-data-hook: - ln $(DESTDIR)$(mandir)/man1/split_ncvars.pl.1 $(DESTDIR)$(mandir)/man1/split_ncvars.1 + ln $(DESTDIR)$(mandir)/man1/split_ncvars.pl.1 $(DESTDIR)$(mandir)/man1/split_ncvars.1 || echo OOPS From c054d7704274beeab42ef3102807c4036340de06 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Mon, 10 Feb 2025 17:15:22 -0500 Subject: [PATCH 13/13] remove non-existant option in configure call. re-autoconfigure in test block of meta.yaml to be able to successfully call tests. add environment.yml... may just remove it --- build.sh | 2 +- environment.yml | 14 ++++++++++++++ meta.yaml | 4 ++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 environment.yml diff --git a/build.sh b/build.sh index 93e49ee6..ba8a0b5b 100644 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ autoreconf -i -./configure --prefix=$PREFIX --disable-ocean-model-grid-generator +./configure --prefix=$PREFIX make -j make install diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..5d93bbe6 --- /dev/null +++ b/environment.yml @@ -0,0 +1,14 @@ +name: fre-nctools +channels: + - conda-forge + - noaa-gfdl +dependencies: + - autoconf + - automake + - libnetcdf + - netcdf-fortran + - nco + - hdf5=1.14.*=mpi* + - openmp + - mpich + - tcsh diff --git a/meta.yaml b/meta.yaml index 7ce08911..ddb008bf 100644 --- a/meta.yaml +++ b/meta.yaml @@ -23,6 +23,10 @@ requirements: test: commands: + - echo 'yay! we made it!' + - pwd && ls + - autoreconf -i + - ./configure --prefix=$PREFIX - make check RUN_EXPENSIVE_TESTS=yes about: