Merge remote-tracking branch 'origin-official/main'

.github/issue_template.md

@@ -3,7 +3,7 @@
 <!--     comply with it, including treating everyone with respect: -->
 <!--     https://github.com/hyperledger/besu/blob/main/CODE_OF_CONDUCT.md -->
 <!--   * Reproduced the issue in the latest version of the software -->
-<!--   * Read the debugging docs: https://besu.hyperledger.org/en/stable/HowTo/Monitor/Logging/ -->
+<!--   * Read the debugging docs: https://besu.hyperledger.org/private-networks/how-to -->
 <!--   * Duplicate Issue check:  https://github.com/search?q=+is%3Aissue+repo%3Ahyperledger/Besu -->
 <!-- Note:  Not all sections will apply to all issue types. -->
 

.github/pull_request_template.md

@@ -1,20 +1,21 @@
+## PR description
+
+## Fixed Issue(s)
+<!-- Please link to fixed issue(s) here using format: fixes #<issue number> -->
+<!-- Example: "fixes #2" -->
+
+
 ### Thanks for sending a pull request! Have you done the following?
 
 - [ ] Checked out our [contribution guidelines](https://github.com/hyperledger/besu/blob/main/CONTRIBUTING.md)?
 - [ ] Considered documentation and added the `doc-change-required` label to this PR [if updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).
 - [ ] Considered the changelog and included an [update if required](https://wiki.hyperledger.org/display/BESU/Changelog).
 - [ ] For database changes (e.g. KeyValueSegmentIdentifier) considered compatibility and performed forwards and backwards compatibility tests
 
-### Most advanced CI tests are deferred until PR approval, but you could:
+### Locally, you can run these tests to catch failures early:
 
-- [ ] locally run all unit tests via: `./gradlew build`
-- [ ] locally run all acceptance tests via: `./gradlew acceptanceTest`
-- [ ] locally run all integration tests via: `./gradlew integrationTest`
-- [ ] locally run all reference tests via: `./gradlew ethereum:referenceTests:referenceTests`
+- [ ] unit tests: `./gradlew build`
+- [ ] acceptance tests: `./gradlew acceptanceTest`
+- [ ] integration tests: `./gradlew integrationTest`
+- [ ] reference tests: `./gradlew ethereum:referenceTests:referenceTests`
 
-
-## PR description
-
-## Fixed Issue(s)
-<!-- Please link to fixed issue(s) here using format: fixes #<issue number> -->
-<!-- Example: "fixes #2" -->

.github/workflows/acceptance-tests.yml

@@ -1,7 +1,7 @@
 name: acceptance-tests
 on:
   workflow_dispatch:
-  pull_request_target:
+  pull_request:
     branches:
       - main
       - release-*
@@ -11,8 +11,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
-  total-runners: 16
+  GRADLE_OPTS: "-Xmx6g"
+  total-runners: 10
 
 jobs:
   acceptanceTestEthereum:
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+        runner_index: [0,1,2,3,4,5,6,7,8,9]
     steps:
       - name: Checkout Repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -35,54 +35,69 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
-      - name: get acceptance test report
+      - name: Install required packages
+        run:  sudo apt-get install -y xmlstarlet
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: List unit tests
+        run: ./gradlew acceptanceTestNotPrivacy --test-dry-run -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+      - name: Extract current test list
+        run: mkdir tmp; find . -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v '/testsuite/@name' '{}'; echo ' acceptanceTestNotPrivacy'" | tee tmp/currentTests.list
+      - name: get acceptance test reports
         uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
         with:
           branch: main
           name_is_regexp: true
           name: 'acceptance-node-\d*\d-test-results'
           path: tmp/junit-xml-reports-downloaded
           if_no_artifact_found: true
-      - name: setup gradle
-        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
-        with:
-          cache-disabled: true
       - name: Split tests
-        id: split-tests
-        uses: r7kamura/split-tests-by-timings@9322bd292d9423e2bc5a65bec548901801341e3f
-        with:
-          reports: tmp/junit-xml-reports-downloaded
-          glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
-          total: ${{env.total-runners}}
-          index: ${{ matrix.runner_index }}
-      - name: write out test list
-        run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
+        run: .github/workflows/splitTestsByTime.sh tmp/junit-xml-reports-downloaded "tmp/junit-xml-reports-downloaded/acceptance-node-.*-test-results" "TEST-" ${{env.total-runners}} ${{ matrix.runner_index }} > testList.txt
       - name: format gradle args
-        #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
-        #then drop file extension, then insert --tests option between each.
-        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g'  > gradleArgs.txt
+        # we do not need the module task here
+        run:  cat testList.txt | cut -f 2- -d ' ' | tee gradleArgs.txt
+      - name: Upload Timing
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
+        with:
+          name: acceptance-tests-timing
+          path: 'tmp/timing.tsv'
+      - name: Upload Lists
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
+        with:
+          name: acceptance-tests-lists
+          path: 'tmp/*.list'
+      - name: Upload gradle test tasks
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        with:
+          name: test-args-${{ matrix.runner_index }}.txt
+          path: '*.txt'
       - name: run acceptance tests
         run: ./gradlew acceptanceTestNotPrivacy `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
-      - name: cleanup tempfiles
-        run: rm testList.txt gradleArgs.txt
       - name: Upload Acceptance Test Results
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
         with:
           name: acceptance-node-${{matrix.runner_index}}-test-results
           path: 'acceptance-tests/tests/build/test-results/**/TEST-*.xml'
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@5f47764eec0e1c1f19f40c8e60a5ba47e47015c5
-        if: (success() || failure()) # always run even if the build step fails
-        with:
-          report_paths: 'acceptance-tests/tests/build/test-results/**/TEST-*.xml'
-          annotate_only: true
   accepttests-passed:
     name: "accepttests-passed"
     runs-on: ubuntu-22.04
     needs: [ acceptanceTestEthereum ]
     permissions:
       checks: write
       statuses: write
+    if: always()
     steps:
-      - name: consolidation
-        run: echo "consolidating statuses"
+      # Fail if any `needs` job was not a success.
+      # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow.
+      - name: Fail on workflow error
+        run: exit 1
+        if: >-
+          ${{
+            contains(needs.*.result, 'failure')
+            || contains(needs.*.result, 'cancelled')
+            || contains(needs.*.result, 'skipped')
+          }}

.github/workflows/artifacts.yml

@@ -13,6 +13,9 @@ jobs:
     runs-on: ubuntu-22.04
     permissions:
       contents: write
+    outputs:
+      tarSha: ${{steps.hashes.outputs.tarSha}}
+      zipSha: ${{steps.hashes.outputs.zipSha}}
     steps:
       - name: checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -27,30 +30,32 @@ jobs:
           cache-disabled: true
       - name: assemble release
         run:
-          ./gradlew -Prelease.releaseVersion=${{github.ref_name}} -Pversion=${{github.ref_name}} assemble
+          ./gradlew -Prelease.releaseVersion=${{github.event.release.name}} -Pversion=${{github.event.release.name}} assemble
       - name: hashes
         id: hashes
         run: |
           cd build/distributions
+          echo "zipSha=$(shasum -a 256 besu*.zip)"
+          echo "tarSha=$(shasum -a 256 besu*.tar.gz)"
           echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT
           echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT
       - name: upload tarball
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
         with:
           path: 'build/distributions/besu*.tar.gz'
-          name: besu-${{ github.ref_name }}.tar.gz
+          name: besu-${{ github.event.release.name }}.tar.gz
           compression-level: 0
       - name: upload zipfile
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
         with:
           path: 'build/distributions/besu*.zip'
-          name: besu-${{ github.ref_name }}.zip
+          name: besu-${{ github.event.release.name }}.zip
           compression-level: 0
+
   testWindows:
     runs-on: windows-2022
     needs: artifacts
     timeout-minutes: 10
-    if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Set up Java
         uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
@@ -73,7 +78,9 @@ jobs:
           besu\bin\besu.bat --version
   publish:
     runs-on: ubuntu-22.04
-    needs: testWindows
+    needs: [artifacts, testWindows]
+    permissions:
+      contents: write
     steps:
       - name: Download archives
         uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe
@@ -89,5 +96,25 @@ jobs:
             build/distributions/besu*.tar.gz
             build/distributions/besu*.zip
           body: |
-            ${{steps.hashes.outputs.tarSha}}
-            ${{steps.hashes.outputs.zipSha}}
+            ${{needs.artifacts.outputs.tarSha}}
+            ${{needs.artifacts.outputs.zipSha}}
+  artifactoryPublish:
+    runs-on: ubuntu-22.04
+    needs: artifacts
+    steps:
+      - name: checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up JDK 17
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: Artifactory Publish
+        env:
+          ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }}
+          ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY_TOKEN }}
+        run: ./gradlew -Prelease.releaseVersion=${{ github.event.release.name }} -Pversion=${{github.event.release.name}} artifactoryPublish