Merge pull request #1953 from jedwards4b/netcdf_integration_v5

netcdf integration update
NCAR · May 18, 2023 · d858d79 · d858d79
2 parents 744bf5c + 4ac2b0e
commit d858d79
Show file tree

Hide file tree

Showing 184 changed files with 5,298 additions and 1,729 deletions.
diff --git a/.github/actions/buildhdf5/action.yml b/.github/actions/buildhdf5/action.yml
@@ -20,7 +20,7 @@ inputs:
     default: False
     required: False
     type: boolean
-    
+
   install_prefix:
     description: 'Install path of hdf5'
     default: ${GITHUB_WORKSPACE}/hdf5

diff --git a/.github/actions/buildmpich/action.yml b/.github/actions/buildmpich/action.yml
@@ -24,7 +24,7 @@ runs:
       run: |
         pushd mpich-src
         ./autogen.sh
-        ./configure --prefix=${{ inputs.install_prefix }} 
+        ./configure --prefix=${{ inputs.install_prefix }}
         make -j4
         make install
         popd
diff --git a/.github/actions/buildnetcdf/action.yml b/.github/actions/buildnetcdf/action.yml
@@ -35,7 +35,7 @@ inputs:
     default: False
     required: False
     type: boolean
-    
+
   install_prefix:
     description: 'Install path of netcdf'
     default: $HOME/netcdf-c

diff --git a/.github/actions/buildopenmpi/action.yml b/.github/actions/buildopenmpi/action.yml
@@ -24,7 +24,7 @@ runs:
       run: |
         pushd openmpi-src
         perl ./autogen.pl
-        ./configure --prefix=${{ inputs.install_prefix }} 
+        ./configure --prefix=${{ inputs.install_prefix }}
         make -j4
         make install
         popd
diff --git a/.github/actions/buildpnetcdf/action.yml b/.github/actions/buildpnetcdf/action.yml
@@ -41,7 +41,7 @@ runs:
         then
           config_opts="${config_opts} --enable-shared=yes"
         fi
-        
+
         config_opts="${config_opts}"
         echo "config_opts=$config_opts"
         autoreconf -i

diff --git a/.github/actions/parallelio_autotools/action.yml b/.github/actions/parallelio_autotools/action.yml
@@ -42,43 +42,43 @@ inputs:
     type: boolean
   with_mpi-serial:
     description: 'Build with mpi-serial library'
-    default: 
+    default:
     required: False
     type: string
   with_pnetcdf:
     description: 'Build with pnetcdf library'
-    default: 
+    default:
     required: False
-    type: string  
+    type: string
   with_netcdf:
     description: 'Build with netcdf library'
     default: /usr
     required: False
-    type: string  
+    type: string
   with_netcdff:
     description: 'Build with netcdff library'
-    default: 
+    default:
     required: False
-    type: string  
+    type: string
   with_valgrind:
     description: 'Build the parallelio valgrind leak check '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   enable_fortran:
     description: 'Build the parallelio Fortran Library '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   extra_cflags:
     description: 'Additional cflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_fflags:
     description: 'Additional fflags to use'
-    default: 
+    default:
     required: False
     type: string
   with_mpiexec:
@@ -91,27 +91,41 @@ inputs:
     default: ${GITHUB_WORKSPACE}/parallelio
     required: False
     type: string
-
+  src_path:
+    description: 'Path to parallelio source'
+    default: ${GITHUB_WORKSPACE}/parallelio-src
+    required: False
+    type: string
+
 runs:
   using: composite
   steps:
-    - uses: actions/checkout@v3
+    - name: Check if already present
+      uses: andstor/file-existence-action@v2
+      with:
+       files: ${{ inputs.src_path }}
+    - name: get parallelio
+      if: ${{ steps.check_files.outputs.files_exists != 'true' }}
+      uses: actions/checkout@v3
       with:
         repository: NCAR/ParallelIO
-        path: parallelio-src
+        path: ${{ inputs.src_path }}
         ref: ${{ inputs.parallelio_version }}
     - id: parallelio-build
       shell: bash
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src
+        cd ${{ inputs.src_path }}
         autoreconf -i
-        config_opts='--prefix=${{ inputs.install_prefix }} --enable-fortran=${{ inputs.enable_fortran }} '
+        config_opts='--prefix=${{ inputs.install_prefix }} '
         if [[ -z "${{ inputs.with_pnetcdf }}" ]];
         then
           config_opts="${config_opts} --disable-pnetcdf "
         fi
+        if [ "${{ inputs.enable_fortran }}" = "true" ];
+        then
+          config_opts="${config_opts} --enable-fortran=yes "
+        fi
         echo "config_opts = ${config_opts} --with-mpiexec=${{ inputs.with_mpiexec }}"
         ./configure ${config_opts} --with-mpiexec="${{ inputs.with_mpiexec }}"
         make -j 4 VERBOSE=1
         make install
-
diff --git a/.github/actions/parallelio_cmake/action.yml b/.github/actions/parallelio_cmake/action.yml
@@ -42,39 +42,39 @@ inputs:
     type: boolean
   with_mpi-serial:
     description: 'Build with mpi-serial library'
-    default: 
+    default:
     required: False
     type: string
   with_valgrind:
     description: 'Build the parallelio valgrind leak check '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   enable_fortran:
     description: 'Build the parallelio Fortran Library '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   extra_cflags:
     description: 'Additional cflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_ldflags:
     description: 'Additional ldflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_fflags:
     description: 'Additional fflags to use'
-    default: 
+    default:
     required: False
     type: string
   mpiexec_flags:
     description: 'extra mpiexec flags'
-    default: 
+    default:
     required: False
     type: string
   netcdf_c_include_dir:

diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml
@@ -17,14 +17,14 @@ jobs:
       CFLAGS: "-std=c99 -Wall -Werror"
       CPPFLAGS: "-I/usr/include -I/usr/lib/x86_64-linux-gnu/netcdf/mpi/include/ "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf_mpi -lpnetcdf"
-      FCFLAGS: "-Wall -Werror -fallow-argument-mismatch"
+      FCFLAGS: "-Wall -Werror -fallow-argument-mismatch -Wno-conversion"
     steps:
     - uses: actions/checkout@v3
     - name: Installs
       run: |
         set -x
-        sudo apt-get update 
-        sudo apt-get install netcdf-bin 
+        sudo apt-get update
+        sudo apt-get install netcdf-bin
         sudo apt-get install libnetcdf-mpi-19
         sudo apt-get install libnetcdf-mpi-dev
         sudo apt-get install pnetcdf-bin
@@ -46,10 +46,11 @@ jobs:
         with_pnetcdf: /usr
         with_mpiexec: 'mpiexec --oversubscribe'
         parallelio_version: ${{ env.GITHUB_SHA }}
+        src_path: ${GITHUB_WORKSPACE}
     - name: make check
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src   
-        make  check
+        cd $GITHUB_WORKSPACE
+        make check
 
 #    - name: Setup tmate session
 #      if: ${{ failure() }}

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
@@ -26,7 +26,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install netcdf-bin libnetcdf-dev doxygen graphviz wget gfortran \
         libjpeg-dev libz-dev openmpi-bin libopenmpi-dev cmake pnetcdf-bin libpnetcdf-dev libnetcdff-dev
-        
+
     - name: cmake build
       uses: ./.github/actions/parallelio_cmake
       with:

diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -16,17 +16,16 @@ jobs:
       FC: mpiifort
       # Versions should match the github tag names
       PNETCDF_VERSION: checkpoint.1.12.3
-      NETCDF_C_VERSION: v4.9.0
+      NETCDF_C_VERSION: v4.9.1
       NETCDF_FORTRAN_VERSION: v4.6.0
       HDF5_VERSION: hdf5_1_12_2
     steps:
     - uses: actions/checkout@v3
     - name: Installs
       run: |
         set -x
-        sudo apt-get update 
-        sudo apt-get install wget
-        sudo apt-get install libjpeg-dev
+        sudo apt-get update
+        sudo apt-get install libcurl4-gnutls-dev
         sudo apt-get install libz-dev
     - name: cache intel compiler
       id: cache-intel
@@ -73,9 +72,9 @@ jobs:
     - name: prep netcdf-c
       run: |
         export PATH=$GITHUB_WORKSPACE/hdf5/bin:$GITHUB_WORKSPACE/netcdf/bin:$PATH
-        export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib  "
+        export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib  -L/usr/lib/x86_64-linux-gnu/ -lcurl"
         export CPPFLAGS="$CPPFLAGS -I$GITHUB_WORKSPACE/hdf5/include -I$GITHUB_WORKSPACE/netcdf/include"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib"
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib:/usr/lib/x86_64-linux-gnu/"
         printenv >> $GITHUB_ENV
     - name: build-netcdf-c
       if: steps.cache-netcdf-c.outputs.cache-hit != 'true'

diff --git a/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml b/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml
@@ -12,17 +12,17 @@ jobs:
       CPPFLAGS: "-I${GITHUB_WORKSPACE}/mpich/include"
       LDFLAGS: "-L${GITHUB_WORKSPACE}/mpich/lib "
       # Note issue https://github.com/NCAR/ParallelIO/issues/1889 netcdf integration currently only works with netcdf 4.7.4
-      NETCDF_C_VERSION: v4.7.4
-      NETCDF_FORTRAN_VERSION: v4.5.4
-      MPICH_VERSION: v4.0.3
+      NETCDF_C_VERSION: v4.9.2
+      NETCDF_FORTRAN_VERSION: v4.6.0
+      MPICH_VERSION: v4.1
       HDF5_VERSION: hdf5_1_12_2
       FFLAGS: "-fallow-argument-mismatch"
       FCFLAGS: "-fallow-argument-mismatch"
     steps:
     - uses: actions/checkout@v3
     - name: Installs
       run: |
-        sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev
+        sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev libcurl4-gnutls-dev
     - name: cache-mpich
       id: cache-mpich
       uses: actions/cache@v3
@@ -65,24 +65,24 @@ jobs:
         export FC=mpifort
         export PATH="${GITHUB_WORKSPACE}/hdf5/bin:${GITHUB_WORKSPACE}/mpich/bin:${GITHUB_WORKSPACE}/netcdf/bin:$PATH"
         export CPPFLAGS="$CPPFLAGS -I${GITHUB_WORKSPACE}/hdf5/include -I${GITHUB_WORKSPACE}/netcdf/include"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${GITHUB_WORKSPACE}/hdf5/lib"
-        export LDFLAGS="$LDFLAGS -L${GITHUB_WORKSPACE}/hdf5/lib  -L${GITHUB_WORKSPACE}/netcdf/lib "
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${GITHUB_WORKSPACE}/hdf5/lib:/usr/lib/x86_64-linux-gnu/"
+        export LDFLAGS="$LDFLAGS -L${GITHUB_WORKSPACE}/hdf5/lib  -L${GITHUB_WORKSPACE}/netcdf/lib -L/usr/lib/x86_64-linux-gnu/ -lcurl"
         printenv >> $GITHUB_ENV
-      
-        
+
+
     - name: Build NetCDF C
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
       uses: ./.github/actions/buildnetcdf
       with:
         netcdf_version: ${{ env.NETCDF_C_VERSION }}
         install_prefix: ${GITHUB_WORKSPACE}/netcdf
 
-    # - name: cache-netcdf-fortran
-    #   id: cache-netcdf-fortran
-    #   uses: actions/cache@v3
-    #   with:
-    #     path: ~/netcdf-fortran
-    #     key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }}
+    - name: cache-netcdf-fortran
+      id: cache-netcdf-fortran
+      uses: actions/cache@v3
+      with:
+        path: ~/netcdf-fortran
+        key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }}
 
     - name: Build NetCDF Fortran
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
@@ -97,13 +97,12 @@ jobs:
       with:
         enable_fortran: True
         enable_netcdf_integration: True
+        src_path: ${GITHUB_WORKSPACE}
         parallelio_version: ${{ env.GITHUB_SHA }}
     - name: make check
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src   
+        cd $GITHUB_WORKSPACE
         make -j check
 #    - name: Setup tmate session
 #      if: ${{ failure() }}
 #      uses: mxschmitt/action-tmate@v3
-
-
diff --git a/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml b/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml
@@ -37,7 +37,7 @@ jobs:
       with:
         install_prefix: $HOME/mpich
         mpich_version: ${{ env.MPICH_VERSION }}
-        
+
     - name: cache-hdf5
       id: cache-hdf5
       uses: actions/cache@v3
@@ -120,23 +120,11 @@ jobs:
       run: |
         cd $GITHUB_WORKSPACE/parallelio-src
         make -j check
-        
+
     - name: autotools build
       run: |
         set -x
         gcc --version
-#        echo 'export PATH=/home/runner/mpich/bin:$PATH' > .bashrc
-#        source .bashrc
-#        export CPPFLAGS: "-I/home/runner/mpich/include -I/home/runner/hdf5/include -I/home/runner/netcdf-c/include -I/home/runner/netcdf-fortran/include -I/home/runner/pnetcdf/include"
-#        export LDFLAGS: "-L/home/runner/mpich/lib -L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib -L/home/runner/netcdf-fortran/lib -L/home/runner/pnetcdf/lib"
-#        export CC=/home/runner/mpich/bin/mpicc
-#        export FC=/home/runner/mpich/bin/mpifort
-#        export CFLAGS="-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan"
-#        export FCFLAGS="$FCFLAGS -g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan"
-#        export LDFLAGS="$LDFLAGS -static-libasan"
-#        export ASAN_OPTIONS="detect_odr_violation=0"
         autoreconf -i
         ./configure --enable-fortran --enable-netcdf-integration
         make -j check
-
-
diff --git a/.github/workflows/netcdf_pnetcdf_openmpi.yml b/.github/workflows/netcdf_pnetcdf_openmpi.yml
@@ -71,9 +71,9 @@ jobs:
         netcdf_version: ${{ env.NETCDF_C_VERSION }}
         install_prefix: ${GITHUB_WORKSPACE}/netcdf
 
-    - name: Setup tmate session
-      if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3
+#    - name: Setup tmate session
+#      if: ${{ failure() }}
+#      uses: mxschmitt/action-tmate@v3
 
     - name: Build NetCDF Fortran
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
@@ -105,7 +105,7 @@ jobs:
     #     parallelio_version: ${{ env.GITHUB_SHA }}
     # - name: make check
     #   run: |
-    #     cd ${GITHUB_WORKSPACE}/parallelio-src   
+    #     cd ${GITHUB_WORKSPACE}/parallelio-src
     #     make -j check
     #     make distclean
     # - name: cmake build