Merge branch 'aptos-labs:main' into feat/pypi_ci

.github/actions/file-change-determinator/action.yaml

@@ -13,4 +13,5 @@ runs:
       continue-on-error: true # Avoid skipping any checks if this job fails (see: https://github.com/fkirc/skip-duplicate-actions/issues/301)
       uses: fkirc/skip-duplicate-actions@v5
       with:
+        skip_after_successful_duplicate: false # Don't skip if the action is a duplicate (this may cause false positives)
         paths_ignore: '["**/*.md", "developer-docs-site/**"]'

.github/actions/general-lints/action.yaml

@@ -10,6 +10,8 @@ runs:
   steps:
     # Checkout the repository
     - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0 # get all the history because cargo xtest --change-since origin/main requires it.
 
     # Install shellcheck and run it on the dev_setup.sh script
     - name: Run shell lints

.github/linters/semgrep/pull-request-target-code-checkout.yaml

@@ -0,0 +1,59 @@
+rules:
+  - id: pull-request-target-code-checkout
+    languages:
+      - yaml
+    message: This GitHub Actions workflow file uses `pull_request_target` and checks
+      out code from the incoming pull request. When using `pull_request_target`,
+      the Action runs in the context of the target repository, which includes
+      access to all repository secrets. Please ensure you have `permission-check`
+      enabled for the jobs that check out code. Please see
+      https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      for additional mitigations.
+    metadata:
+      category: security
+      owasp:
+        - A01:2021 - Broken Access Control
+      cwe:
+        - "CWE-913: Improper Control of Dynamically-Managed Code Resources"
+      references:
+        - https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+        - https://github.com/justinsteven/advisories/blob/master/2021_github_actions_checkspelling_token_leak_via_advice_symlink.md
+      technology:
+        - github-actions
+      subcategory:
+        - audit
+      likelihood: MEDIUM
+      impact: LOW
+      confidence: MEDIUM
+      license: Commons Clause License Condition v1.0[LGPL-2.1-only]
+      vulnerability_class:
+        - Code Injection
+    patterns:
+      - pattern-either:
+          - pattern-inside: |
+              on:
+                ...
+                pull_request_target: ...
+                ...
+              ...
+          - pattern-inside: |
+              on: [..., pull_request_target, ...]
+              ...
+          - pattern-inside: |
+              on: pull_request_target
+              ...
+      - pattern-inside: |
+          jobs:
+            ...
+            $JOBNAME:
+              ...
+      - pattern-not-inside: |
+          needs: [permission-check]
+          ...
+      - pattern: |
+          ...
+          uses: "$ACTION"
+      - metavariable-regex:
+          metavariable: $ACTION
+          regex: actions/checkout@.*
+    severity: WARNING

.github/workflows/cargo-metadata-upload.yaml

@@ -0,0 +1,26 @@
+name: Target Determinator Cargo Metadata Uploader
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    paths:
+      - ".github/workflows/cargo-metadata-upload.yaml"
+permissions:
+  contents: read
+  id-token: write
+jobs:
+  cargo-metadata:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
+      - id: auth
+        uses: "google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033" # pin@v
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+      - uses: 'google-github-actions/setup-gcloud@v1'
+      - shell: bash
+        run: |
+          cargo metadata --all-features | gsutil cp - gs://aptos-core-cargo-metadata-public/metadata-${{ github.sha }}.json

.github/workflows/check-sdk-examples.yaml

@@ -1,6 +1,7 @@
 name: "Check SDK examples"
 on:
   pull_request:
+    types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
   push:
     branches:
       - devnet
@@ -15,6 +16,7 @@ jobs:
   # whereas we run the test-sdk-confirm-client-generated-publish against a node
   # built from the same commit and run as part of that CI job.
   run-examples:
+    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     runs-on: ubuntu-latest
     env:
       APTOS_NODE_URL: https://fullnode.devnet.aptoslabs.com
@@ -44,6 +46,7 @@ jobs:
           command: cd ./ecosystem/typescript/sdk/examples/javascript && pnpm install && pnpm test
 
   run-python-examples:
+    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     runs-on: ubuntu-latest
     env:
       APTOS_NODE_URL: https://fullnode.devnet.aptoslabs.com/v1

.github/workflows/cli-e2e-tests.yaml

@@ -11,7 +11,14 @@ on:
         required: true
         type: string
         description: Use this to override the git SHA1, branch name (e.g. devnet) or tag
+      SKIP_JOB:
+        required: false
+        default: false
+        type: boolean
+        description: Set to true to skip this job. Useful for PRs that don't require this workflow.
 
+# TODO: should we migrate this to a composite action, so that we can skip it
+# at the call site, and don't need to wrap each step in an if statement?
 jobs:
   # Run the Aptos CLI examples. We run the CLI on this commit / PR against a
   # local testnet using the devnet, testnet, and mainnet branches. This way
@@ -24,10 +31,12 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@v3
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           ref: ${{ inputs.GIT_SHA }}
 
       - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
           GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
@@ -37,11 +46,13 @@ jobs:
           GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
 
       - uses: ./.github/actions/python-setup
+        if: ${{ !inputs.SKIP_JOB }}
         with:
           pyproject_directory: crates/aptos/e2e
 
       # Run CLI tests against local testnet built from devnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: devnet-tests
         with:
           max_attempts: 5
@@ -50,6 +61,7 @@ jobs:
 
       # Run CLI tests against local testnet built from testnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: testnet-tests
         with:
           max_attempts: 5
@@ -58,13 +70,18 @@ jobs:
 
       # Run CLI tests against local testnet built from mainnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
+        if: ${{ !inputs.SKIP_JOB }}
         name: mainnet-tests
         with:
           max_attempts: 5
           timeout_minutes: 20
           command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network mainnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-mainnet
 
       - name: Print local testnet logs on failure
-        if: ${{ failure() }}
+        if: ${{ !inputs.SKIP_JOB && failure() }}
         working-directory: docker/compose/validator-testnet
         run: docker logs aptos-tools-devnet && docker logs aptos-tools-testnet && docker logs aptos-tools-mainnet
+
+      # Print out whether the job was skipped.
+      - run: echo "Skipping CLI E2E tests!"
+        if: ${{ inputs.SKIP_JOB }}

.github/workflows/codeql-analysis.yaml

@@ -1,9 +1,8 @@
 name: "CodeQL"
 
 on:
-  schedule:
-    # Every day at 10:15am UTC aka 3:15am PT
-    - cron: "15 10 * * *"
+  # Allow triggering manually
+  workflow_dispatch:
 
 jobs:
   analyze:

.github/workflows/coverage.yaml

@@ -2,11 +2,6 @@ name: Test Coverage
 on:
   workflow_dispatch:
   workflow_call:
-  schedule:
-    # every day at 9am PST
-    - cron: "0 16 * * *"
-  pull_request:
-    types: [ labeled, opened, synchronize, reopened ]
 
 env:
   CARGO_INCREMENTAL: "0"

.github/workflows/docker-build-test.yaml

@@ -109,6 +109,18 @@ jobs:
       targetCacheId: ${{ env.TARGET_CACHE_ID }}
       targetRegistry: ${{ env.TARGET_REGISTRY }}
 
+  # This job determines which files were changed
+  file_change_determinator:
+    runs-on: ubuntu-latest
+    outputs:
+      only_docs_changed: ${{ steps.determine_file_changes.outputs.only_docs_changed }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run the file change determinator
+        id: determine_file_changes
+        uses: ./.github/actions/file-change-determinator
+
+  # This is a PR required job.
   rust-images:
     needs: [permission-check, determine-docker-build-metadata]
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-docker-rust-build.yaml@main
@@ -182,8 +194,9 @@ jobs:
       BUILD_ADDL_TESTING_IMAGES: true
       TARGET_REGISTRY: ${{ needs.determine-docker-build-metadata.outputs.targetRegistry }}
 
+  # This is a PR required job.
   node-api-compatibility-tests:
-    needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
+    needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images"
     if: |
       (
         github.event_name == 'push' ||
@@ -196,9 +209,11 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
+  # This is a PR required job.
   cli-e2e-tests:
-    needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
+    needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images"
     if: |
       (
         github.event_name == 'push' ||
@@ -211,20 +226,20 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   indexer-grpc-e2e-tests:
     needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
     if: |
-      (github.event_name == 'push' && github.ref_name != 'main') ||
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
-      github.event.pull_request.auto_merge != null ||
       contains(github.event.pull_request.body, '#e2e')
     uses: aptos-labs/aptos-core/.github/workflows/docker-indexer-grpc-test.yaml@main
     secrets: inherit
     with:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
 
+  # This is a PR required job.
   forge-e2e-test:
     needs:
       - permission-check
@@ -234,6 +249,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -254,8 +270,9 @@ jobs:
       # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered
       # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
       FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
-  # Run e2e compat test against testnet branch
+  # Run e2e compat test against testnet branch. This is a PR required job.
   forge-compat-test:
     needs:
       - permission-check
@@ -265,6 +282,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -282,8 +300,9 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 300
       COMMENT_HEADER: forge-compat
       FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
-  # Run forge framework upgradability test
+  # Run forge framework upgradability test. This is a PR required job.
   forge-framework-upgrade-test:
     needs:
       - permission-check
@@ -293,6 +312,7 @@ jobs:
       - rust-images-failpoints
       - rust-images-performance
       - rust-images-consensus-only-perf-test
+      - file_change_determinator
     if: |
       !failure() && !cancelled() && needs.permission-check.result == 'success' && (
         (github.event_name == 'push' && github.ref_name != 'main') ||
@@ -310,6 +330,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 300
       COMMENT_HEADER: forge-framework-upgrade
       FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
   forge-consensus-only-perf-test:
     needs:

.github/workflows/execution-performance.yaml

@@ -2,11 +2,20 @@ name: "execution-performance"
 on:
   workflow_dispatch:
   pull_request:
+    types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
   schedule:
     - cron: "0 12 * * *" # This runs every day at 12pm UTC.
 
 jobs:
   execution-performance:
+    if: | # Only run on each PR once an appropriate event occurs
+      (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'schedule' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null) ||
+        contains(github.event.pull_request.body, '#e2e'
+      )
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-execution-performance.yaml@main
     secrets: inherit
     with:

.github/workflows/faucet-tests.yaml

@@ -25,6 +25,7 @@ jobs:
           comment-not-permitted: Sorry, you don't have permission to trigger this workflow.
 
   run-tests-devnet:
+    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     needs: [permission-check]
     runs-on: high-perf-docker
     steps:
@@ -43,6 +44,7 @@ jobs:
           GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
 
   run-tests-testnet:
+    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     needs: [permission-check]
     runs-on: high-perf-docker
     permissions:

.github/workflows/find-packages-with-undeclared-feature-dependencies.yaml

@@ -1,9 +1,6 @@
 name: "Find Packages with undeclared feature dependencies"
 on:
   workflow_dispatch:
-  schedule:
-    # every day at 3am PST
-    - cron: "0 10 * * *"
 
 jobs:
   find-packages-with-undeclared-feature-dependencies: