Sync develop changes 8/27 - 9/11 to hdf5_1_14.

.github/workflows/aocc-auto.yml

@@ -74,6 +74,7 @@ jobs:
             --enable-build-mode=${{ inputs.build_mode }} \
             --enable-shared \
             --enable-parallel \
+            --enable-subfiling-vfd \
             LDFLAGS="-L/home/runner/work/hdf5/hdf5/aocc-compiler-4.2.0/lib \
             -L/home/runner/work/hdf5/hdf5/openmpi-4.1.6-install/lib"
 

.github/workflows/aocc-cmake.yml

@@ -71,6 +71,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
           -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF \
           -DHDF5_ENABLE_PARALLEL:BOOL=ON \
+          -DHDF5_ENABLE_SUBFILING_VFD:BOOL=ON \
           -DHDF5_BUILD_CPP_LIB:BOOL=OFF \
           -DLIBAEC_USE_LOCALCONTENT=OFF \
           -DZLIB_USE_LOCALCONTENT=OFF \

.github/workflows/clang-format-check.yml

@@ -1,17 +1,20 @@
 name: clang-format Check
 on:
   pull_request:
+
 permissions:
   contents: read
+
 jobs:
   formatting-check:
     name: Formatting Check
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'skip-ci')"
     steps:
     - uses: actions/[email protected]
+
     - name: Run clang-format style check for C and Java code
-      uses: DoozyX/clang-format-lint-action@v0.17
+      uses: DoozyX/clang-format-lint-action@v0.18.2
       with:
         source: '.'
         extensions: 'c,h,cpp,hpp,java'

.github/workflows/clang-format-fix.yml

@@ -11,8 +11,10 @@ name: clang-format Commit Changes
 on: 
   workflow_dispatch:
   push:
+
 permissions:
   contents: read
+
 jobs:
   formatting-check:
     name: Commit Format Changes
@@ -21,16 +23,18 @@ jobs:
     permissions:
         contents: write # In order to allow EndBug/add-and-commit to commit changes
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - uses: actions/[email protected]
+
     - name: Fix C and Java formatting issues detected by clang-format
-      uses: DoozyX/clang-format-lint-action@d3c7f85989e3b6416265a0d12f8b4a8aa8b0c4ff # v0.13
+      uses: DoozyX/clang-format-lint-action@v0.18.2
       with:
         source: '.'
         extensions: 'c,h,cpp,hpp,java'
         clangFormatVersion: 13
         inplace: True
         style: file
         exclude: './config ./hl/src/H5LTanalyze.c ./hl/src/H5LTparse.c ./hl/src/H5LTparse.h ./src/H5Epubgen.h ./src/H5Einit.h ./src/H5Eterm.h ./src/H5Edefin.h ./src/H5version.h ./src/H5overflow.h'
+
     - uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9.1.4
       with:
           author_name: github-actions

.github/workflows/cmake-ctest.yml

@@ -23,6 +23,12 @@ on:
         required: true
         default: snapshots
     secrets:
+        APPLE_CERTS_BASE64:
+            required: true
+        APPLE_CERTS_BASE64_PASSWD:
+            required: true
+        KEYCHAIN_PASSWD:
+            required: true
         AZURE_TENANT_ID:
             required: true
         AZURE_CLIENT_ID:
@@ -53,13 +59,15 @@ jobs:
         run: |
           if [[ '${{ env.signing_secret }}' == '' ]]
           then
-            SIGN_VAL=$(echo "false")
+            SIGN_VAL=$(echo 'notexists')
           else
-            SIGN_VAL=$(echo "true")
+            SIGN_VAL=$(echo 'exists')
           fi
           echo "BINSIGN=$SIGN_VAL" >> $GITHUB_OUTPUT
         shell: bash
 
+      - run: echo "signing is ${{ steps.set-signing-state.outputs.BINSIGN }}."
+
   build_and_test_win:
   # Windows w/ MSVC + CMake
   #
@@ -114,7 +122,27 @@ jobs:
         run: 7z x ${{ steps.set-file-base.outputs.FILE_BASE }}.zip
         shell: bash
 
+      - name: Install TrustedSigning (Windows)
+        run: |
+            Invoke-WebRequest -Uri https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile .\nuget.exe
+            .\nuget.exe install Microsoft.Windows.SDK.BuildTools -Version 10.0.22621.3233 -x
+            .\nuget.exe install Microsoft.Trusted.Signing.Client -Version 1.0.53 -x
+        shell: pwsh
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/[email protected]
+        with:
+            name: "credentials.json"
+            dir: '${{ steps.set-file-base.outputs.SOURCE_BASE }}'
+            json: '{"Endpoint": "${{ secrets.AZURE_ENDPOINT }}","CodeSigningAccountName": "${{ secrets.AZURE_CODE_SIGNING_NAME }}","CertificateProfileName": "${{ secrets.AZURE_CERT_PROFILE_NAME }}"}'
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+
       - name: Run ctest (Windows)
+        env:
+          BINSIGN: ${{ needs.check-secret.outputs.sign-state }}
+          SIGNTOOLDIR: ${{ github.workspace }}/Microsoft.Windows.SDK.BuildTools/bin/10.0.22621.0/x64
         run: |
           cd "${{ runner.workspace }}/hdf5/${{ steps.set-file-base.outputs.SOURCE_BASE }}"
           cmake --workflow --preset=${{ inputs.preset_name }}-MSVC --fresh
@@ -134,7 +162,7 @@ jobs:
           file-digest: SHA256
           timestamp-rfc3161: http://timestamp.acs.microsoft.com
           timestamp-digest: SHA256
-        if: ${{ needs.check-secret.outputs.sign-state == 'true' }}
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
 
       - name: Publish binary (Windows)
         id: publish-ctest-binary
@@ -323,6 +351,28 @@ jobs:
         with:
           version: "1.9.7"
 
+      - name: Install the Apple certificate and provisioning profile
+        shell: bash
+        env:
+          BUILD_CERTIFICATE_BASE64: ${{ secrets.APPLE_CERTS_BASE64 }}
+          P12_PASSWORD: ${{ secrets.APPLE_CERTS_BASE64_PASSWD }}
+          KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
+        run: |
+            # create variables
+            CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
+            KEYCHAIN_FILE=${{ vars.KEYCHAIN_NAME }}.keychain
+            # import certificate from secrets
+            echo $BUILD_CERTIFICATE_BASE64 | base64 --decode > $CERTIFICATE_PATH
+            security -v create-keychain -p $KEYCHAIN_PASSWD $KEYCHAIN_FILE
+            security -v list-keychain -d user -s $KEYCHAIN_FILE
+            security -v list-keychains
+            security -v set-keychain-settings -lut 21600 $KEYCHAIN_FILE
+            security -v unlock-keychain -p $KEYCHAIN_PASSWD $KEYCHAIN_FILE
+            # import certificate to keychain
+            security -v import $CERTIFICATE_PATH -P $P12_PASSWORD -A -t cert -f pkcs12 -k $KEYCHAIN_FILE
+            security -v set-key-partition-list -S apple-tool:,codesign:,apple: -k $KEYCHAIN_PASSWD $KEYCHAIN_FILE
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+
       - name: Set up JDK 19
         uses: actions/setup-java@v4
         with:
@@ -367,11 +417,100 @@ jobs:
 
       - name: Run ctest (MacOS_latest)
         id: run-ctest
+        env:
+          BINSIGN: ${{ needs.check-secret.outputs.sign-state }}
+          SIGNER: ${{ vars.SIGNER }}
         run: |
           cd "${{ runner.workspace }}/hdf5/${{ steps.set-file-base.outputs.SOURCE_BASE }}"
-          cmake --workflow --preset=${{ inputs.preset_name }}-MACOS-Clang --fresh
+          cmake --workflow --preset=${{ inputs.preset_name }}-macos-Clang --fresh
+        shell: bash
+
+      - name: Sign dmg (MacOS_latest)
+        id: sign-dmg
+        env:
+          KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
+          KEYCHAIN_NAME: ${{ vars.KEYCHAIN_NAME }}
+          SIGNER: ${{ vars.SIGNER }}
+          NOTARY_USER: ${{ vars.NOTARY_USER }}
+          NOTARY_KEY: ${{ vars.NOTARY_KEY }}
+        run: |
+          /usr/bin/codesign --force --timestamp --options runtime --entitlements ${{ runner.workspace }}/hdf5/${{ steps.set-file-base.outputs.SOURCE_BASE }}/config/cmake/distribution.entitlements --verbose=4 --strict --sign ${{ env.SIGNER }} --deep ${{ runner.workspace }}/hdf5/build/${{ inputs.preset_name }}-macos-Clang/*.dmg
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
         shell: bash
 
+      - name: Check dmg timestamp (MacOS_latest)
+        run: |
+          /usr/bin/codesign -dvv ${{ runner.workspace }}/hdf5/build/${{ inputs.preset_name }}-macos-Clang/*.dmg
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: Verify dmg (MacOS_latest)
+        run: |
+          /usr/bin/hdiutil verify ${{ runner.workspace }}/hdf5/build/${{ inputs.preset_name }}-macos-Clang/*.dmg
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: Notarize dmg (MacOS_latest)
+        id: notarize-dmg
+        env:
+          KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
+          KEYCHAIN_NAME: ${{ vars.KEYCHAIN_NAME }}
+          SIGNER: ${{ vars.SIGNER }}
+          NOTARY_USER: ${{ vars.NOTARY_USER }}
+          NOTARY_KEY: ${{ vars.NOTARY_KEY }}
+        run: |
+          jsonout=$(/usr/bin/xcrun notarytool submit --wait --output-format json --apple-id ${{ env.NOTARY_USER }} --password ${{ env.NOTARY_KEY }} --team-id ${{ env.SIGNER }} ${{ runner.workspace }}/hdf5/build/${{ inputs.preset_name }}-macos-Clang/*.dmg)
+          echo "JSONOUT=$jsonout" >> $GITHUB_OUTPUT
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: Get ID token (MacOS_latest)
+        id: get-id-token
+        run: |
+          echo "notary result is ${{ fromJson(steps.notarize-dmg.outputs.JSONOUT) }}"
+          token=${{ fromJson(steps.notarize-dmg.outputs.JSONOUT).id }}
+          echo "ID_TOKEN=$token" >> "$GITHUB_OUTPUT"
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: post notary check (MacOS_latest)
+        id: post-notary
+        env:
+          KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
+          KEYCHAIN_NAME: ${{ vars.KEYCHAIN_NAME }}
+          SIGNER: ${{ vars.SIGNER }}
+          NOTARY_USER: ${{ vars.NOTARY_USER }}
+          NOTARY_KEY: ${{ vars.NOTARY_KEY }}
+        run: |
+          {
+            echo 'NOTARYOUT<<EOF'
+            /usr/bin/xcrun notarytool info --apple-id ${{ env.NOTARY_USER }} --password ${{ env.NOTARY_KEY }} --team-id ${{ env.SIGNER }} ${{ steps.get-id-token.outputs.ID_TOKEN }}
+            echo EOF
+          } >> $GITHUB_OUTPUT
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: Get notary info (MacOS_latest)
+        id: get-notary-info
+        run: |
+          echo "notary info is ${{ steps.post-notary.outputs.NOTARYOUT }}."
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+
+      - name: Staple dmg (MacOS_latest)
+        id: staple-dmg
+        env:
+          KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
+          KEYCHAIN_NAME: ${{ vars.KEYCHAIN_NAME }}
+          SIGNER: ${{ vars.SIGNER }}
+          NOTARY_USER: ${{ vars.NOTARY_USER }}
+          NOTARY_KEY: ${{ vars.NOTARY_KEY }}
+        run: |
+          /usr/bin/xcrun stapler staple ${{ runner.workspace }}/hdf5/build/${{ inputs.preset_name }}-macos-Clang/*.dmg
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+        shell: bash
+        continue-on-error: true
+
       - name: Publish binary (MacOS_latest)
         id: publish-ctest-binary
         run: |
@@ -395,7 +534,7 @@ jobs:
           cp ${{ runner.workspace }}/hdf5/build114/${{ inputs.preset_name }}-Clang/README.md ${{ runner.workspace }}/builddmg/hdf5
           cp ${{ runner.workspace }}/hdf5/build114/${{ inputs.preset_name }}-Clang/*.dmg ${{ runner.workspace }}/builddmg/hdf5
           cd "${{ runner.workspace }}/builddmg"
-          tar -zcvf ${{ steps.set-file-base.outputs.FILE_BASE }}-macos14_clang.arm64.dmg.tar.gz hdf5
+          tar -zcvf ${{ steps.set-file-base.outputs.FILE_BASE }}-macos14_clang.dmg.tar.gz hdf5
         shell: bash
 
       - name: List files in the space (MacOS_latest)
@@ -415,7 +554,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
               name: tgz-macos14_clang-dmg-binary
-              path: ${{ runner.workspace }}/builddmg/${{ steps.set-file-base.outputs.FILE_BASE }}-macos14_clang.arm64.dmg.tar.gz
+              path: ${{ runner.workspace }}/builddmg/${{ steps.set-file-base.outputs.FILE_BASE }}-macos14_clang.dmg.tar.gz
               if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
 
   build_and_test_S3_linux:
@@ -544,11 +683,30 @@ jobs:
         run: 7z x ${{ steps.set-file-base.outputs.FILE_BASE }}.zip
         shell: bash
 
+      - name: Install TrustedSigning (Windows)
+        run: |
+            Invoke-WebRequest -Uri https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile .\nuget.exe
+            .\nuget.exe install Microsoft.Windows.SDK.BuildTools -Version 10.0.22621.3233 -x
+            .\nuget.exe install Microsoft.Trusted.Signing.Client -Version 1.0.53 -x
+        shell: pwsh
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+
+      - name: create-json
+        id: create-json
+        uses: jsdaniell/[email protected]
+        with:
+            name: "credentials.json"
+            dir: '${{ steps.set-file-base.outputs.SOURCE_BASE }}'
+            json: '{"Endpoint": "${{ secrets.AZURE_ENDPOINT }}","CodeSigningAccountName": "${{ secrets.AZURE_CODE_SIGNING_NAME }}","CertificateProfileName": "${{ secrets.AZURE_CERT_PROFILE_NAME }}"}'
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
+
       - name: Run ctest (Windows_intel) with oneapi
         env:
           FC: ${{ steps.setup-fortran.outputs.fc }}
           CC: ${{ steps.setup-fortran.outputs.cc }}
           CXX: ${{ steps.setup-fortran.outputs.cxx }}
+          BINSIGN: ${{ needs.check-secret.outputs.sign-state }}
+          SIGNTOOLDIR: ${{ github.workspace }}/Microsoft.Windows.SDK.BuildTools/bin/10.0.22621.0/x64
         run: |
           cd "${{ runner.workspace }}/hdf5/${{ steps.set-file-base.outputs.SOURCE_BASE }}"
           cmake --workflow --preset=${{ inputs.preset_name }}-win-Intel --fresh
@@ -568,7 +726,7 @@ jobs:
           file-digest: SHA256
           timestamp-rfc3161: http://timestamp.acs.microsoft.com
           timestamp-digest: SHA256
-        if: ${{ needs.check-secret.outputs.sign-state == 'true' }}
+        if: ${{ needs.check-secret.outputs.sign-state == 'exists' }}
 
       - name: Publish binary (Windows_intel)
         id: publish-ctest-binary

.github/workflows/daily-build.yml

@@ -45,6 +45,9 @@ jobs:
       #use_tag: snapshot-1.14
       use_environ: snapshots
     secrets:
+        APPLE_CERTS_BASE64: ${{ secrets.APPLE_CERTS_BASE64 }}
+        APPLE_CERTS_BASE64_PASSWD: ${{ secrets.APPLE_CERTS_BASE64_PASSWD }}
+        KEYCHAIN_PASSWD: ${{ secrets.KEYCHAIN_PASSWD }}
         AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
         AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
         AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}

.github/workflows/julia-auto.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Get Sources
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
 
       - name: Install Dependencies
         shell: bash
@@ -60,7 +60,7 @@ jobs:
           arch: 'x64'
 
       - name: Get julia hdf5 source
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           repository: JuliaIO/HDF5.jl
           path: .

.github/workflows/julia-cmake.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Get Sources
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
 
       - name: Install Dependencies
         shell: bash
@@ -63,7 +63,7 @@ jobs:
           arch: 'x64'
 
       - name: Get julia hdf5 source
-        uses: actions/[email protected].1
+        uses: actions/[email protected].7
         with:
           repository: JuliaIO/HDF5.jl
           path: .

.github/workflows/main-auto-par.yml

@@ -63,6 +63,7 @@ jobs:
             --with-default-api-version=v114 \
             --enable-shared \
             --enable-parallel \
+            --enable-subfiling-vfd \
             --disable-cxx \
             --enable-fortran \
             --disable-java \

.github/workflows/main-auto-spc.yml

@@ -428,7 +428,7 @@ jobs:
             --enable-cxx \
             --disable-fortran \
             --enable-java \
-            --disable-mirror-vfd \
+            --enable-mirror-vfd \
             --enable-direct-vfd \
             --disable-ros3-vfd \
             --with-szlib=yes
@@ -491,7 +491,7 @@ jobs:
             --enable-cxx \
             --disable-fortran \
             --enable-java \
-            --disable-mirror-vfd \
+            --enable-mirror-vfd \
             --enable-direct-vfd \
             --disable-ros3-vfd \
             --with-szlib=yes