From 4d858494a882d01550c604dbf3ee8cba961c84fb Mon Sep 17 00:00:00 2001 From: Armin Samii Date: Wed, 21 Feb 2024 18:06:35 -0500 Subject: [PATCH] also generate golden hash --- .github/actions/gradle-and-sha/action.yml | 9 ++-- .github/actions/zip/action.yml | 5 +-- .../workflows/generate-dependency-hashes.sh | 2 +- .github/workflows/release.yml | 6 ++- .github/workflows/sha-of-zip.sh | 41 +++++++++++++++++++ .github/workflows/sha.sh | 2 +- build.gradle | 8 ++++ 7 files changed, 64 insertions(+), 9 deletions(-) create mode 100755 .github/workflows/sha-of-zip.sh diff --git a/.github/actions/gradle-and-sha/action.yml b/.github/actions/gradle-and-sha/action.yml index 4c0bcd145..108097700 100644 --- a/.github/actions/gradle-and-sha/action.yml +++ b/.github/actions/gradle-and-sha/action.yml @@ -23,9 +23,12 @@ runs: - name: "Rename output file" shell: bash run: | - ls build/*/* mv ${{ inputs.intermediate-filepath }} ${{ inputs.final-filepath }} - - name: "Generate SHA512" + - name: "Generate SHA512. If ZIP, it will also extract to avoid timestamp-mingling." shell: bash - run: ./.github/workflows/sha.sh ${{ runner.os }} ${{ inputs.final-filepath }} 512 + run: | + ./.github/workflows/sha.sh ${{ inputs.final-filepath }} ${{ runner.os }} 512 > ${{ inputs.final-finalpath }}.sha512 + if [ "${{ inputs.intermediate-filepath }}" == "*zip" ]; then + ./.github/workflows/sha-of-zip.sh ${{ inputs.final-filepath }} ${{ runner.os }} 512 > ${{ inputs.final-finalpath }}.golden.sha512 + fi diff --git a/.github/actions/zip/action.yml b/.github/actions/zip/action.yml index 651a93558..b8ab7cce0 100644 --- a/.github/actions/zip/action.yml +++ b/.github/actions/zip/action.yml @@ -1,5 +1,5 @@ name: "Create zip of a directory" -description: "Cross-platform helper: creates SHA512" +description: "Cross-platform helper: creates a zip archive" inputs: input: @@ -20,11 +20,10 @@ runs: run: ${{ inputs.command }} - name: "Generate zip for Linux" - if: runner.os == 'Linux' || runner.os == 'MacOS' shell: bash run: zip -r ${{inputs.zipFilename}} ${{inputs.input}} - name: "Generate zip for Windows" if: runner.os == 'Windows' shell: powershell - run: Compress-Archive -Path ${{inputs.input}} -Destination ${{inputs.zipFilename}} + run: zip -r ${{inputs.zipFilename}} ${{inputs.input}} diff --git a/.github/workflows/generate-dependency-hashes.sh b/.github/workflows/generate-dependency-hashes.sh index cdc743b7d..7bd77d114 100755 --- a/.github/workflows/generate-dependency-hashes.sh +++ b/.github/workflows/generate-dependency-hashes.sh @@ -10,7 +10,7 @@ parentPath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) echo "Filename, SHA-1 Checksum, SHA-256 Checksum, Maven Dependency URL, Direct URL to SHA-1, Direct URL to SHA-256" cd ~/.gradle/caches/modules-2/files-2.1 -for filename in $(find * -type f); do +for filename in $(find * -type f); do # filename is of format, with dot-separated org: # ////-. # friendly URL is of format, with dot-separated org: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index deb67a5d2..1f4b6f406 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,7 +97,9 @@ jobs: - name: "Generate SHA512 for plugins cache" shell: bash - run: ./.github/workflows/sha.sh ${{steps.cachefn.outputs.FILEPATH}} ${{ runner.os }} 512 + run: | + ./.github/workflows/sha.sh ${{steps.cachefn.outputs.FILEPATH}} ${{ runner.os }} 512 > ${{steps.cachefn.outputs.FILEPATH}}.sha512 + # ./.github/workflows/sha-of-zip.sh ${{steps.cachefn.outputs.FILEPATH}} ${{ runner.os }} 512 > ${{steps.cachefn.outputs.FILEPATH}}.golden.sha512 - name: "Prepare keychain" if: matrix.os == 'macOS-latest' @@ -152,10 +154,12 @@ jobs: path: | ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }} ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }}.sha512 + ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }}.golden.sha512 ${{ github.workspace }}/${{ steps.exefn.outputs.FILEPATH }} ${{ github.workspace }}/${{ steps.exefn.outputs.FILEPATH }}.sha512 ${{ github.workspace }}/${{steps.cachefn.outputs.FILEPATH}} ${{ github.workspace }}/${{steps.cachefn.outputs.FILEPATH}}.sha512 + ${{ github.workspace }}/${{steps.cachefn.outputs.FILEPATH}}.golden.sha512 retention-days: 1 - name: "Upload binaries to release" diff --git a/.github/workflows/sha-of-zip.sh b/.github/workflows/sha-of-zip.sh new file mode 100755 index 000000000..57b13dc05 --- /dev/null +++ b/.github/workflows/sha-of-zip.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Unzips the given zip file, then generates a checksum of the zip file that ignores +# the timestamps by extracting it and SHA'ing a file of all SHAs. This is not recursive:- +# zips within the zip will not be extracted, and therefore the timestamps of the "inner" zip +# will be a part of the hash. +# Usage: ./sha-of-zip.sh +set -e + +zipFilepath=$1 +os=$2 +a=$3 + +echo parentpath +parentPath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) + +# Make a temporary directory to extract zip, and a temporary file to hold SHAs +echo temp1 +tempDirectory=$(mktemp -d) +echo temp2 +tempAllChecksumsFile=$(mktemp) +echo temp3 +touch $tempAllChecksumsFile + +# Extract the zip +echo unzipy +unzip -q $zipFilepath -d $tempDirectory 2>/dev/null + +# Get a checksum for each file in the zip +echo tempcd +cd $tempDirectory +echo foreach +for filename in $(find * -type f | sort); do + echo hcheck + checksum=$($parentPath/sha.sh $filename $os $a) + echo sum + echo $checksum >> $tempAllChecksumsFile +done + +# Echo the checksum of the checksums +echo sumitall +echo $($parentPath/sha.sh $tempAllChecksumsFile $os $a) diff --git a/.github/workflows/sha.sh b/.github/workflows/sha.sh index e36b0b3f9..89fe9ba93 100755 --- a/.github/workflows/sha.sh +++ b/.github/workflows/sha.sh @@ -8,7 +8,6 @@ filename=$1 os=$2 a=$3 - if [ $os == 'Windows' ]; then echo $(certutil -hashfile $filename SHA$a | sed -n 2p) elif [ $os == 'Linux' ]; then @@ -16,3 +15,4 @@ elif [ $os == 'Linux' ]; then else echo $(shasum -a $a $filename | cut -f1 -d" ") fi + diff --git a/build.gradle b/build.gradle index cb3cd2a9c..09c2579cb 100644 --- a/build.gradle +++ b/build.gradle @@ -236,3 +236,11 @@ tasks.jlink.doLast { into JLINK_DIR + "/sample_input" } } + +// Ensure reproducible hashes +tasks.withType(AbstractArchiveTask) { + preserveFileTimestamps = false + reproducibleFileOrder = true + dirMode = 0755 + fileMode = 0644 +}