From 491c51c4345627e3eb9a393b215dba1cb431511a Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 25 Nov 2024 15:47:38 +1300 Subject: [PATCH 01/46] merge test 1 --- temp.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 temp.txt diff --git a/temp.txt b/temp.txt new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/temp.txt @@ -0,0 +1 @@ +test \ No newline at end of file From 4655e8a24a338a5c966deed03c37cb9cbd489829 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 25 Nov 2024 15:53:14 +1300 Subject: [PATCH 02/46] update pr --- temp.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temp.txt b/temp.txt index 30d74d2..d606037 100644 --- a/temp.txt +++ b/temp.txt @@ -1 +1 @@ -test \ No newline at end of file +test2 \ No newline at end of file From 855d7356d1e59f23f2d329b801cdc3737655a567 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 25 Nov 2024 16:08:17 +1300 Subject: [PATCH 03/46] disable jobs --- .github/workflows/book.yml | 56 ++++++------- .github/workflows/test-suite.yml | 130 +------------------------------ 2 files changed, 29 insertions(+), 157 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 031a88b..86b0020 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,34 +1,34 @@ -name: mdbook +# name: mdbook -on: - push: - branches: - - unstable +# on: +# push: +# branches: +# - unstable -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true -jobs: - build-and-upload-to-s3: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 +# jobs: +# build-and-upload-to-s3: +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 - - name: Setup mdBook - uses: peaceiris/actions-mdbook@v1 - with: - mdbook-version: 'latest' +# - name: Setup mdBook +# uses: peaceiris/actions-mdbook@v1 +# with: +# mdbook-version: 'latest' - - run: mdbook build - working-directory: book +# - run: mdbook build +# working-directory: book - - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83 - with: - args: --follow-symlinks --delete - env: - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'ap-southeast-2' - SOURCE_DIR: 'book/book' +# - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83 +# with: +# args: --follow-symlinks --delete +# env: +# AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# AWS_REGION: 'ap-southeast-2' +# SOURCE_DIR: 'book/book' diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 56ba5a4..cf44911 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -59,132 +59,4 @@ jobs: steps: - name: Check that the pull request is not targeting the stable branch run: test ${{ github.base_ref }} != "stable" - release-tests-ubuntu: - name: release-tests-ubuntu - needs: [check-labels] - if: needs.check-labels.outputs.skip_ci != 'true' - # Use self-hosted runners only on the sigp repo. - runs-on: ubuntu-latest #${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - bins: cargo-nextest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Run tests in release - run: make nextest-release - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats - debug-tests-ubuntu: - name: debug-tests-ubuntu - needs: [check-labels] - if: needs.check-labels.outputs.skip_ci != 'true' - # Use self-hosted runners only on the sigp repo. - runs-on: ubuntu-latest #${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: moonrepo/setup-rust@v1 - with: - channel: stable - bins: cargo-nextest - - name: Run tests in debug - run: make nextest-debug - - name: Show cache stats - if: env.SELF_HOSTED_RUNNERS == 'true' - run: sccache --show-stats - check-code: - name: check-code - runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 1 - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - components: rustfmt,clippy - bins: cargo-audit - - name: Check formatting with cargo fmt - run: make cargo-fmt - - name: Lint code for quality and style with Clippy - run: make lint - - name: Certify Cargo.lock freshness - run: git diff --exit-code Cargo.lock - - name: Run cargo audit - run: make audit-CI - # Dependency conflicts are causing pain here. Once SLOG is removed from lighthouse this should - # be re-enabled - # - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose - #run: CARGO_HOME=$(readlink -f $HOME) make vendor - - name: Markdown-linter - run: make mdlint - check-msrv: - name: check-msrv - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Rust at Minimum Supported Rust Version (MSRV) - run: | - metadata=$(cargo metadata --no-deps --format-version 1) - msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "anchor")) | .[0].rust_version') - rustup override set $msrv - - name: Run cargo check - run: cargo check --workspace - cargo-udeps: - name: cargo-udeps - needs: [check-labels] - if: needs.check-labels.outputs.skip_ci != 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get latest version of nightly Rust - uses: moonrepo/setup-rust@v1 - with: - channel: nightly - bins: cargo-udeps - cache: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Create Cargo config dir - run: mkdir -p .cargo - - name: Install custom Cargo config - run: cp -f .github/custom/config.toml .cargo/config.toml - - name: Run cargo udeps to identify unused crates in the dependency graph - run: make udeps - env: - # Allow warnings on Nightly - RUSTFLAGS: "" - spell-check: - name: spellcheck - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - # Uses the canonical version so that the version is up to date: - # https://github.com/rojopolis/spellcheck-github-actions?tab=readme-ov-file#using-a-canonical-version - - uses: rojopolis/spellcheck-github-actions@v0 -# cli-check: -# name: cli-check -# needs: [check-labels] -# if: needs.check-labels.outputs.skip_ci != 'true' -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - name: Get latest version of stable Rust -# uses: moonrepo/setup-rust@v1 -# with: -# channel: stable -# cache-target: release -# - name: Run Makefile to trigger the bash script -# run: make cli + \ No newline at end of file From bea39a1c20b0667f9b23f9c1d37db22c2afff605 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 25 Nov 2024 16:14:17 +1300 Subject: [PATCH 04/46] branch check --- .github/workflows/test-suite.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index cf44911..8f63235 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -58,5 +58,9 @@ jobs: if: github.event_name == 'pull_request' || github.event_name == 'merge_group' steps: - name: Check that the pull request is not targeting the stable branch - run: test ${{ github.base_ref }} != "stable" + run: | + if [[ "${{ github.base_ref }}" == "stable" && "${{ github.head_ref }}" != "unstable" ]]; then + echo "Pull requests to the stable branch can only come from the unstable branch." + exit 1 + fi \ No newline at end of file From 72409c0a22c4044ed13de05e08ddbb4541f0ac97 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 25 Nov 2024 16:23:36 +1300 Subject: [PATCH 05/46] check when merging to stable --- .github/workflows/test-suite.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 56ba5a4..ca6e98a 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -58,7 +58,11 @@ jobs: if: github.event_name == 'pull_request' || github.event_name == 'merge_group' steps: - name: Check that the pull request is not targeting the stable branch - run: test ${{ github.base_ref }} != "stable" + run: | + if [[ "${{ github.base_ref }}" == "stable" && "${{ github.head_ref }}" != "unstable" ]]; then + echo "Pull requests to the stable branch can only come from the unstable branch." + exit 1 + fi release-tests-ubuntu: name: release-tests-ubuntu needs: [check-labels] From 51b7443463beca295d2466a57d1900575a5948ba Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 13:57:25 +1300 Subject: [PATCH 06/46] attempt to use hcv --- .github/workflows/docker.yml | 341 ++++++++++++++++++----------------- 1 file changed, 178 insertions(+), 163 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c29ff33..3d0b6db 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,163 +1,178 @@ -# TODO: Setup docker -# name: docker -# -# on: -# push: -# branches: -# - stable -# tags: -# - v* -# -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true -# -# env: -# DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} -# DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} -# # Enable self-hosted runners for the sigp repo only. -# SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} -# -# jobs: -# # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX -# # which is either empty or `-unstable`. -# # -# # It would be nice if the arch didn't get spliced into the version between `latest` and -# # `unstable`, but for now we keep the two parts of the version separate for backwards -# # compatibility. -# extract-version: -# runs-on: ubuntu-22.04 -# steps: -# - name: Extract version (if stable) -# if: github.event.ref == 'refs/heads/stable' -# run: | -# echo "VERSION=latest" >> $GITHUB_ENV -# echo "VERSION_SUFFIX=" >> $GITHUB_ENV -# - name: Extract version (if unstable) -# if: github.event.ref == 'refs/heads/unstable' -# run: | -# echo "VERSION=latest" >> $GITHUB_ENV -# echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV -# - name: Extract version (if tagged release) -# if: startsWith(github.event.ref, 'refs/tags') -# run: | -# echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV -# echo "VERSION_SUFFIX=" >> $GITHUB_ENV -# outputs: -# VERSION: ${{ env.VERSION }} -# VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} -# build-docker-single-arch: -# name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }} -# # Use self-hosted runners only on the sigp repo. -# runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} -# strategy: -# matrix: -# binary: [anchor] -# cpu_arch: [aarch64, x86_64] -# include: -# - profile: maxperf -# -# needs: [extract-version] -# env: -# VERSION: ${{ needs.extract-version.outputs.VERSION }} -# VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} -# steps: -# - uses: actions/checkout@v4 -# - name: Update Rust -# if: env.SELF_HOSTED_RUNNERS == 'false' -# run: rustup update stable -# - name: Dockerhub login -# run: | -# echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin -# -# - name: Sets env vars for Anchor -# if: startsWith(matrix.binary, 'anchor') -# run: | -# echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV -# -# - name: Set `make` command for anchor -# if: startsWith(matrix.binary, 'anchor') -# run: | -# echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV -# -# - name: Set `make` command for lcli -# if: startsWith(matrix.binary, 'lcli') -# run: | -# echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV -# -# - name: Cross build binaries -# run: | -# cargo install cross -# env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }} -# -# - name: Make bin dir -# run: mkdir ./bin -# -# - name: Move cross-built binary into Docker scope -# run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin -# -# - name: Map aarch64 to arm64 short arch -# if: startsWith(matrix.cpu_arch, 'aarch64') -# run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV -# -# - name: Map x86_64 to amd64 short arch -# if: startsWith(matrix.cpu_arch, 'x86_64') -# run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV; -# -# - name: Install QEMU -# if: env.SELF_HOSTED_RUNNERS == 'false' -# run: sudo apt-get update && sudo apt-get install -y qemu-user-static -# -# - name: Set up Docker Buildx -# if: env.SELF_HOSTED_RUNNERS == 'false' -# uses: docker/setup-buildx-action@v3 -# -# - name: Build and push (Anchor) -# if: startsWith(matrix.binary, 'anchor') -# uses: docker/build-push-action@v5 -# with: -# file: ./Dockerfile.cross -# context: . -# platforms: linux/${{ env.SHORT_ARCH }} -# push: true -# tags: | -# ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} -# -# - name: Build and push (lcli) -# if: startsWith(matrix.binary, 'lcli') -# uses: docker/build-push-action@v5 -# with: -# file: ./lcli/Dockerfile.cross -# context: . -# platforms: linux/${{ env.SHORT_ARCH }} -# push: true -# -# tags: | -# ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} -# -# -# build-docker-multiarch: -# name: build-docker-${{ matrix.binary }}-multiarch -# runs-on: ubuntu-22.04 -# strategy: -# matrix: -# binary: [anchor, -# lcli] -# needs: [build-docker-single-arch, extract-version] -# env: -# VERSION: ${{ needs.extract-version.outputs.VERSION }} -# VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} -# steps: -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v3 -# -# - name: Dockerhub login -# run: | -# echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin -# -# - name: Create and push multiarch manifests -# run: | -# docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \ -# ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \ -# ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX}; -# +name: docker + +on: + push: + branches: + - stable + tags: + - v* + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Enable self-hosted runners for the sigp repo only. + SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} + +jobs: + # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX + # which is either empty or `-unstable`. + # + # It would be nice if the arch didn't get spliced into the version between `latest` and + # `unstable`, but for now we keep the two parts of the version separate for backwards + # compatibility. + extract-version: + runs-on: ubuntu-22.04 + steps: + - name: Extract version (if stable) + if: github.event.ref == 'refs/heads/stable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + - name: Extract version (if unstable) + if: github.event.ref == 'refs/heads/unstable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV + - name: Extract version (if tagged release) + if: startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + outputs: + VERSION: ${{ env.VERSION }} + VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + build-docker-single-arch: + name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }} + # Use self-hosted runners only on the sigp repo. + runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} + strategy: + matrix: + binary: [anchor] + cpu_arch: [aarch64, x86_64] + include: + - profile: maxperf + + needs: [extract-version] + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - uses: actions/checkout@v4 + - name: Update Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + run: rustup update stable + + - name: Retrieve Docker credentials from Vault + uses: hashicorp/vault-action@v2 + with: + url: https://vault.sigp.io + method: github + githubToken: ${{ secrets.VAULT_TOKEN }} + secrets: | + spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; + spesi_kv/data/gh_wf_testing DOCKER_PASSWORD + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + - name: Sets env vars for Anchor + if: startsWith(matrix.binary, 'anchor') + run: | + echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV + + - name: Set `make` command for anchor + if: startsWith(matrix.binary, 'anchor') + run: | + echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV + + - name: Set `make` command for lcli + if: startsWith(matrix.binary, 'lcli') + run: | + echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV + + - name: Cross build binaries + run: | + cargo install cross + env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }} + + - name: Make bin dir + run: mkdir ./bin + + - name: Move cross-built binary into Docker scope + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin + + - name: Map aarch64 to arm64 short arch + if: startsWith(matrix.cpu_arch, 'aarch64') + run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + + - name: Map x86_64 to amd64 short arch + if: startsWith(matrix.cpu_arch, 'x86_64') + run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV; + + - name: Install QEMU + if: env.SELF_HOSTED_RUNNERS == 'false' + run: sudo apt-get update && sudo apt-get install -y qemu-user-static + + - name: Set up Docker Buildx + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: docker/setup-buildx-action@v3 + + - name: Build and push (Anchor) + if: startsWith(matrix.binary, 'anchor') + uses: docker/build-push-action@v5 + with: + file: ./Dockerfile.cross + context: . + platforms: linux/${{ env.SHORT_ARCH }} + push: true + tags: | + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} + + - name: Build and push (lcli) + if: startsWith(matrix.binary, 'lcli') + uses: docker/build-push-action@v5 + with: + file: ./lcli/Dockerfile.cross + context: . + platforms: linux/${{ env.SHORT_ARCH }} + push: true + tags: | + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} + + build-docker-multiarch: + name: build-docker-${{ matrix.binary }}-multiarch + runs-on: ubuntu-22.04 + strategy: + matrix: + binary: [anchor, + lcli] + needs: [build-docker-single-arch, extract-version] + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Retrieve Docker credentials from Vault + uses: hashicorp/vault-action@v2 + with: + url: https://vault.sigp.io + method: github + githubToken: ${{ secrets.GITHUB_TOKEN }} + secrets: | + spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; + spesi_kv/data/gh_wf_testing DOCKER_PASSWORD + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + - name: Create and push multiarch manifests + run: | + docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX}; From 36e937b799689ab1653060bbb84685bccef35f6f Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 14:47:19 +1300 Subject: [PATCH 07/46] tmp comment out --- .github/workflows/test-suite.yml | 130 +++++++++++++++---------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index ca6e98a..31f1ae2 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -106,71 +106,71 @@ jobs: - name: Show cache stats if: env.SELF_HOSTED_RUNNERS == 'true' run: sccache --show-stats - check-code: - name: check-code - runs-on: ubuntu-latest - env: - CARGO_INCREMENTAL: 1 - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - components: rustfmt,clippy - bins: cargo-audit - - name: Check formatting with cargo fmt - run: make cargo-fmt - - name: Lint code for quality and style with Clippy - run: make lint - - name: Certify Cargo.lock freshness - run: git diff --exit-code Cargo.lock - - name: Run cargo audit - run: make audit-CI - # Dependency conflicts are causing pain here. Once SLOG is removed from lighthouse this should - # be re-enabled - # - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose - #run: CARGO_HOME=$(readlink -f $HOME) make vendor - - name: Markdown-linter - run: make mdlint - check-msrv: - name: check-msrv - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Rust at Minimum Supported Rust Version (MSRV) - run: | - metadata=$(cargo metadata --no-deps --format-version 1) - msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "anchor")) | .[0].rust_version') - rustup override set $msrv - - name: Run cargo check - run: cargo check --workspace - cargo-udeps: - name: cargo-udeps - needs: [check-labels] - if: needs.check-labels.outputs.skip_ci != 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get latest version of nightly Rust - uses: moonrepo/setup-rust@v1 - with: - channel: nightly - bins: cargo-udeps - cache: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Create Cargo config dir - run: mkdir -p .cargo - - name: Install custom Cargo config - run: cp -f .github/custom/config.toml .cargo/config.toml - - name: Run cargo udeps to identify unused crates in the dependency graph - run: make udeps - env: - # Allow warnings on Nightly - RUSTFLAGS: "" - spell-check: + # check-code: + # name: check-code + # runs-on: ubuntu-latest + # env: + # CARGO_INCREMENTAL: 1 + # steps: + # - uses: actions/checkout@v4 + # - name: Get latest version of stable Rust + # uses: moonrepo/setup-rust@v1 + # with: + # channel: stable + # cache-target: release + # components: rustfmt,clippy + # bins: cargo-audit + # - name: Check formatting with cargo fmt + # run: make cargo-fmt + # - name: Lint code for quality and style with Clippy + # run: make lint + # - name: Certify Cargo.lock freshness + # run: git diff --exit-code Cargo.lock + # - name: Run cargo audit + # run: make audit-CI + # # Dependency conflicts are causing pain here. Once SLOG is removed from lighthouse this should + # # be re-enabled + # # - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose + # #run: CARGO_HOME=$(readlink -f $HOME) make vendor + # - name: Markdown-linter + # run: make mdlint + # check-msrv: + # name: check-msrv + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Install Rust at Minimum Supported Rust Version (MSRV) + # run: | + # metadata=$(cargo metadata --no-deps --format-version 1) + # msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "anchor")) | .[0].rust_version') + # rustup override set $msrv + # - name: Run cargo check + # run: cargo check --workspace + # cargo-udeps: + # name: cargo-udeps + # needs: [check-labels] + # if: needs.check-labels.outputs.skip_ci != 'true' + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Get latest version of nightly Rust + # uses: moonrepo/setup-rust@v1 + # with: + # channel: nightly + # bins: cargo-udeps + # cache: false + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # - name: Create Cargo config dir + # run: mkdir -p .cargo + # - name: Install custom Cargo config + # run: cp -f .github/custom/config.toml .cargo/config.toml + # - name: Run cargo udeps to identify unused crates in the dependency graph + # run: make udeps + # env: + # # Allow warnings on Nightly + # RUSTFLAGS: "" + # spell-check: name: spellcheck runs-on: ubuntu-latest steps: From 1838ec76c8ce12cc496be3e2f989ba5fec88dea5 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 15:03:04 +1300 Subject: [PATCH 08/46] tmp rm CROSS_FEATURES --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3d0b6db..73e9c7e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,7 +96,7 @@ jobs: - name: Cross build binaries run: | cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }} + env CROSS_PROFILE=${{ matrix.profile }} make ${{ env.MAKE_CMD }} - name: Make bin dir run: mkdir ./bin From 229740b692ccf4eac093e13f657a39c8e4a8b976 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 16:01:48 +1300 Subject: [PATCH 09/46] upgrade --- Cargo.toml | 2 +- anchor/Dockerfile | 18 ++++++++++++++++++ anchor/Dockerfile.cross | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 anchor/Dockerfile create mode 100644 anchor/Dockerfile.cross diff --git a/Cargo.toml b/Cargo.toml index bbdc34b..1d469f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ clap = { version = "4.5.15", features = ["derive", "wrap_help"]} discv5 = "0.8.0" dirs = "5.0.1" either = "1.13.0" -futures = "0.3.30" +futures = "0.3.31" tower-http = {version = "0.6", features = ["cors"] } hyper = "1.4" parking_lot = "0.12" diff --git a/anchor/Dockerfile b/anchor/Dockerfile new file mode 100644 index 0000000..ff7f14d --- /dev/null +++ b/anchor/Dockerfile @@ -0,0 +1,18 @@ +FROM rust:1.78.0-bullseye AS builder +RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev +COPY . lighthouse +ARG FEATURES +ARG PROFILE=release +ARG CARGO_USE_GIT_CLI=true +ENV FEATURES=$FEATURES +ENV PROFILE=$PROFILE +ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_USE_GIT_CLI +RUN cd lighthouse && make + +FROM ubuntu:22.04 +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ + libssl-dev \ + ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /usr/local/cargo/bin/lighthouse /usr/local/bin/lighthouse diff --git a/anchor/Dockerfile.cross b/anchor/Dockerfile.cross new file mode 100644 index 0000000..3691490 --- /dev/null +++ b/anchor/Dockerfile.cross @@ -0,0 +1,10 @@ +# This image is meant to enable cross-architecture builds. +# It assumes the lighthouse binary has already been +# compiled for `$TARGETPLATFORM` and moved to `./bin`. +FROM --platform=$TARGETPLATFORM ubuntu:22.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl-dev \ + ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +COPY ./bin/anchor /usr/local/bin/anchor From cde21a84a8b4e67357bf566f9589c5dc6b88ec5c Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 16:02:53 +1300 Subject: [PATCH 10/46] simplify --- .github/workflows/docker.yml | 104 ++++------------------------------- 1 file changed, 12 insertions(+), 92 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 73e9c7e..4bce4a8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,10 +18,6 @@ env: jobs: # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX # which is either empty or `-unstable`. - # - # It would be nice if the arch didn't get spliced into the version between `latest` and - # `unstable`, but for now we keep the two parts of the version separate for backwards - # compatibility. extract-version: runs-on: ubuntu-22.04 steps: @@ -43,14 +39,14 @@ jobs: outputs: VERSION: ${{ env.VERSION }} VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} - build-docker-single-arch: - name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }} + + build-docker: + name: build-docker-${{ matrix.binary }} # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} strategy: matrix: - binary: [anchor] - cpu_arch: [aarch64, x86_64] + binary: [anchor] include: - profile: maxperf @@ -69,7 +65,7 @@ jobs: with: url: https://vault.sigp.io method: github - githubToken: ${{ secrets.VAULT_TOKEN }} + githubToken: ${{ secrets.GITHUB_TOKEN }} secrets: | spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; spesi_kv/data/gh_wf_testing DOCKER_PASSWORD @@ -78,101 +74,25 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Sets env vars for Anchor - if: startsWith(matrix.binary, 'anchor') - run: | - echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV - - - name: Set `make` command for anchor - if: startsWith(matrix.binary, 'anchor') - run: | - echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV - - - name: Set `make` command for lcli - if: startsWith(matrix.binary, 'lcli') + - name: Build binary run: | - echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV - - - name: Cross build binaries - run: | - cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} make ${{ env.MAKE_CMD }} + make - name: Make bin dir run: mkdir ./bin - - name: Move cross-built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin - - - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.cpu_arch, 'aarch64') - run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - - - name: Map x86_64 to amd64 short arch - if: startsWith(matrix.cpu_arch, 'x86_64') - run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV; - - - name: Install QEMU - if: env.SELF_HOSTED_RUNNERS == 'false' - run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Move built binary into Docker scope + run: mv ./target/release/${{ matrix.binary }} ./bin - name: Set up Docker Buildx if: env.SELF_HOSTED_RUNNERS == 'false' uses: docker/setup-buildx-action@v3 - - name: Build and push (Anchor) - if: startsWith(matrix.binary, 'anchor') + - name: Build and push uses: docker/build-push-action@v5 with: - file: ./Dockerfile.cross + file: ./Dockerfile context: . - platforms: linux/${{ env.SHORT_ARCH }} push: true tags: | - ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} - - - name: Build and push (lcli) - if: startsWith(matrix.binary, 'lcli') - uses: docker/build-push-action@v5 - with: - file: ./lcli/Dockerfile.cross - context: . - platforms: linux/${{ env.SHORT_ARCH }} - push: true - tags: | - ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} - - build-docker-multiarch: - name: build-docker-${{ matrix.binary }}-multiarch - runs-on: ubuntu-22.04 - strategy: - matrix: - binary: [anchor, - lcli] - needs: [build-docker-single-arch, extract-version] - env: - VERSION: ${{ needs.extract-version.outputs.VERSION }} - VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Retrieve Docker credentials from Vault - uses: hashicorp/vault-action@v2 - with: - url: https://vault.sigp.io - method: github - githubToken: ${{ secrets.GITHUB_TOKEN }} - secrets: | - spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; - spesi_kv/data/gh_wf_testing DOCKER_PASSWORD - - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - - name: Create and push multiarch manifests - run: | - docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX}; + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} From 03c689b9bb2be0eb503e58398318bd856231f6e4 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 16:10:00 +1300 Subject: [PATCH 11/46] use GH_TOKEN --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4bce4a8..89807eb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,7 +65,7 @@ jobs: with: url: https://vault.sigp.io method: github - githubToken: ${{ secrets.GITHUB_TOKEN }} + githubToken: ${{ secrets.GH_TOKEN }} secrets: | spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; spesi_kv/data/gh_wf_testing DOCKER_PASSWORD From 2b9899883acf1ac4f976d704f82641622bcbe6fa Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 16:26:05 +1300 Subject: [PATCH 12/46] hardcode docker repo, set dockerfile location --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 89807eb..8502cd2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -91,8 +91,8 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./Dockerfile + file: ./anchor/Dockerfile context: . push: true tags: | - ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} + hughestech/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} From cb96e776d4abc29ca49166a8f2df629918a08ec7 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 16:47:21 +1300 Subject: [PATCH 13/46] use correct dir --- anchor/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anchor/Dockerfile b/anchor/Dockerfile index ff7f14d..a13f866 100644 --- a/anchor/Dockerfile +++ b/anchor/Dockerfile @@ -1,13 +1,13 @@ FROM rust:1.78.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev -COPY . lighthouse +COPY . anchor ARG FEATURES ARG PROFILE=release ARG CARGO_USE_GIT_CLI=true ENV FEATURES=$FEATURES ENV PROFILE=$PROFILE ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_USE_GIT_CLI -RUN cd lighthouse && make +RUN cd anchor && make FROM ubuntu:22.04 RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends \ From eb2d461d71559cb0cebc01cdc01ffee54ee3dcfc Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 26 Nov 2024 17:21:50 +1300 Subject: [PATCH 14/46] update rust and path --- anchor/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anchor/Dockerfile b/anchor/Dockerfile index a13f866..2e34f16 100644 --- a/anchor/Dockerfile +++ b/anchor/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.78.0-bullseye AS builder +FROM rust:1.80.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . anchor ARG FEATURES @@ -15,4 +15,4 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-reco ca-certificates \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/cargo/bin/lighthouse /usr/local/bin/lighthouse +COPY --from=builder /usr/local/cargo/bin/anchor /usr/local/bin/anchor From 33f3a57cffab9e5378109b5a7899b71d9de6f2b8 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 09:49:05 +1300 Subject: [PATCH 15/46] use Cargo.toml as source of truth --- .github/scripts/toml_reader.sh | 55 ++++++++++++++++++++++++++++++++++ .github/workflows/docker.yml | 8 +++++ anchor/Dockerfile | 3 +- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/toml_reader.sh diff --git a/.github/scripts/toml_reader.sh b/.github/scripts/toml_reader.sh new file mode 100755 index 0000000..ce2ec83 --- /dev/null +++ b/.github/scripts/toml_reader.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# TOML Reader - A script to read values from simple TOML files +# Usage: ./toml_reader.sh
+ +get_section() { + # Function to get the section from a TOML file + # Parameters: + # $1 - TOML file path + # $2 - section name + local file="$1" + local section="$2" + + sed -n "/^\[$section\]/,/^\[/p" "$file" | sed '$d' +} + +get_toml_value() { + # Function to get a value from a TOML file + # Parameters: + # $1 - TOML file path + # $2 - section + # $3 - key + local file="$1" + local section="$2" + local key="$3" + + get_section "$file" "$section" | grep "^$key " | cut -d "=" -f2- | tr -d ' "' +} + +# Show usage if script is called directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + if [ "$#" -ne 3 ]; then + echo "Error: Incorrect number of arguments" + echo "Usage: $0
" + echo "Example: $0 ./config.toml server_b domain" + exit 1 + fi + + # Check if file exists + if [ ! -f "$1" ]; then + echo "Error: File '$1' does not exist" + exit 1 + fi + + # Get the value + result=$(get_toml_value "$1" "$2" "$3") + + # Check if value was found + if [ -z "$result" ]; then + echo "Error: No value found for section '[$2]' and key '$3'" + exit 1 + fi + + echo "$result" +fi diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8502cd2..a21d478 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,13 +14,19 @@ concurrency: env: # Enable self-hosted runners for the sigp repo only. SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} + RUST_VERSION: 'abc' jobs: + # Extract the rust-version from the Cargo.toml file. # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX # which is either empty or `-unstable`. extract-version: runs-on: ubuntu-22.04 steps: + - name: Get rust-version + id: get-rust-version + run: RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) + - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' run: | @@ -96,3 +102,5 @@ jobs: push: true tags: | hughestech/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} diff --git a/anchor/Dockerfile b/anchor/Dockerfile index 2e34f16..3d31897 100644 --- a/anchor/Dockerfile +++ b/anchor/Dockerfile @@ -1,4 +1,5 @@ -FROM rust:1.80.0-bullseye AS builder +ARG RUST_VERSION=1.80.0 +FROM rust:${RUST_VERSION}-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . anchor ARG FEATURES From 3867484d8b1d45eb412cd1d8bc5046721e0f6dec Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 09:54:27 +1300 Subject: [PATCH 16/46] fix path for toml_reader.sh --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a21d478..00daf47 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Get rust-version id: get-rust-version - run: RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) + run: RUST_VERSION=$(./scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' From e508fa1d8ed47a35faf4ba82b03431d0f4dd60f8 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:04:24 +1300 Subject: [PATCH 17/46] debugging --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 00daf47..5531270 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,10 @@ jobs: steps: - name: Get rust-version id: get-rust-version - run: RUST_VERSION=$(./scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) + run: | + echo "Current working directory: $(pwd)" + ls -la $(pwd) + RUST_VERSION=$(./scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' From 0890343183ac820caecae76e953fed9d19afb4d6 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:09:49 +1300 Subject: [PATCH 18/46] corrected path --- .github/workflows/docker.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5531270..67fc9a3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,9 +26,7 @@ jobs: - name: Get rust-version id: get-rust-version run: | - echo "Current working directory: $(pwd)" - ls -la $(pwd) - RUST_VERSION=$(./scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) + RUST_VERSION=$(./../.github/scripts/toml_reader.sh Cargo.toml package rust-version) - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' From ef2e799d9f63c4c13ac02ec386a542ae63b05981 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:14:09 +1300 Subject: [PATCH 19/46] debugging --- .github/workflows/docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 67fc9a3..c770fcf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,6 +26,9 @@ jobs: - name: Get rust-version id: get-rust-version run: | + echo "Current working directory: $(pwd)" + ls -la $(pwd) + ls -la ./../.github/scripts/ RUST_VERSION=$(./../.github/scripts/toml_reader.sh Cargo.toml package rust-version) - name: Extract version (if stable) From 35b5db93c08aa22e4de3e08bcbeca35e6abe4a80 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:27:07 +1300 Subject: [PATCH 20/46] use after actions/checkout@v4 --- .github/workflows/docker.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c770fcf..281c1ef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,13 +23,7 @@ jobs: extract-version: runs-on: ubuntu-22.04 steps: - - name: Get rust-version - id: get-rust-version - run: | - echo "Current working directory: $(pwd)" - ls -la $(pwd) - ls -la ./../.github/scripts/ - RUST_VERSION=$(./../.github/scripts/toml_reader.sh Cargo.toml package rust-version) + - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' @@ -70,6 +64,14 @@ jobs: if: env.SELF_HOSTED_RUNNERS == 'false' run: rustup update stable + - name: Get rust-version + id: get-rust-version + run: | + echo "Current working directory: $(pwd)" + ls -la $(pwd) + ls -la ./../.github/scripts/ + RUST_VERSION=$(./../.github/scripts/toml_reader.sh Cargo.toml package rust-version) + - name: Retrieve Docker credentials from Vault uses: hashicorp/vault-action@v2 with: From 71351c28c2ad91ba39b27cd55ae56b97796f9fb7 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:31:31 +1300 Subject: [PATCH 21/46] re-correcting dir --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 281c1ef..f98f6cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -69,8 +69,8 @@ jobs: run: | echo "Current working directory: $(pwd)" ls -la $(pwd) - ls -la ./../.github/scripts/ - RUST_VERSION=$(./../.github/scripts/toml_reader.sh Cargo.toml package rust-version) + ls -la ./.github/scripts/ + RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) - name: Retrieve Docker credentials from Vault uses: hashicorp/vault-action@v2 From 547147373ad6939371f1f11554b17a217500c44d Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 10:55:50 +1300 Subject: [PATCH 22/46] assign rust version to GH_ENV --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f98f6cc..5dab993 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -67,10 +67,10 @@ jobs: - name: Get rust-version id: get-rust-version run: | - echo "Current working directory: $(pwd)" - ls -la $(pwd) - ls -la ./.github/scripts/ - RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version) + echo "RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version)" >> $GITHUB_ENV + + - name: Get latest version of stable Rust + run: echo "rust version is ${{ env.RUST_VERSION }}" - name: Retrieve Docker credentials from Vault uses: hashicorp/vault-action@v2 From e8a62a62c74ddd6fa8b9d38d8120e510666c2383 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 11:40:16 +1300 Subject: [PATCH 23/46] remove test docker org --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5dab993..4f6427c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -100,13 +100,13 @@ jobs: if: env.SELF_HOSTED_RUNNERS == 'false' uses: docker/setup-buildx-action@v3 - - name: Build and push + - name: Build and push (Anchor) uses: docker/build-push-action@v5 with: file: ./anchor/Dockerfile context: . push: true tags: | - hughestech/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} build-args: | RUST_VERSION=${{ env.RUST_VERSION }} From 195d947560702560574215a7a22a9c0fab6389fb Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 12:59:32 +1300 Subject: [PATCH 24/46] re-enable check-code --- .github/workflows/test-suite.yml | 132 +++++++++++++++---------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 31f1ae2..ac0c250 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -106,71 +106,71 @@ jobs: - name: Show cache stats if: env.SELF_HOSTED_RUNNERS == 'true' run: sccache --show-stats - # check-code: - # name: check-code - # runs-on: ubuntu-latest - # env: - # CARGO_INCREMENTAL: 1 - # steps: - # - uses: actions/checkout@v4 - # - name: Get latest version of stable Rust - # uses: moonrepo/setup-rust@v1 - # with: - # channel: stable - # cache-target: release - # components: rustfmt,clippy - # bins: cargo-audit - # - name: Check formatting with cargo fmt - # run: make cargo-fmt - # - name: Lint code for quality and style with Clippy - # run: make lint - # - name: Certify Cargo.lock freshness - # run: git diff --exit-code Cargo.lock - # - name: Run cargo audit - # run: make audit-CI - # # Dependency conflicts are causing pain here. Once SLOG is removed from lighthouse this should - # # be re-enabled - # # - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose - # #run: CARGO_HOME=$(readlink -f $HOME) make vendor - # - name: Markdown-linter - # run: make mdlint - # check-msrv: - # name: check-msrv - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - name: Install Rust at Minimum Supported Rust Version (MSRV) - # run: | - # metadata=$(cargo metadata --no-deps --format-version 1) - # msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "anchor")) | .[0].rust_version') - # rustup override set $msrv - # - name: Run cargo check - # run: cargo check --workspace - # cargo-udeps: - # name: cargo-udeps - # needs: [check-labels] - # if: needs.check-labels.outputs.skip_ci != 'true' - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - name: Get latest version of nightly Rust - # uses: moonrepo/setup-rust@v1 - # with: - # channel: nightly - # bins: cargo-udeps - # cache: false - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # - name: Create Cargo config dir - # run: mkdir -p .cargo - # - name: Install custom Cargo config - # run: cp -f .github/custom/config.toml .cargo/config.toml - # - name: Run cargo udeps to identify unused crates in the dependency graph - # run: make udeps - # env: - # # Allow warnings on Nightly - # RUSTFLAGS: "" - # spell-check: + check-code: + name: check-code + runs-on: ubuntu-latest + env: + CARGO_INCREMENTAL: 1 + steps: + - uses: actions/checkout@v4 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + components: rustfmt,clippy + bins: cargo-audit + - name: Check formatting with cargo fmt + run: make cargo-fmt + - name: Lint code for quality and style with Clippy + run: make lint + - name: Certify Cargo.lock freshness + run: git diff --exit-code Cargo.lock + - name: Run cargo audit + run: make audit-CI + # Dependency conflicts are causing pain here. Once SLOG is removed from lighthouse this should + # be re-enabled + # - name: Run cargo vendor to make sure dependencies can be vendored for packaging, reproducibility and archival purpose + #run: CARGO_HOME=$(readlink -f $HOME) make vendor + - name: Markdown-linter + run: make mdlint + check-msrv: + name: check-msrv + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust at Minimum Supported Rust Version (MSRV) + run: | + metadata=$(cargo metadata --no-deps --format-version 1) + msrv=$(echo $metadata | jq -r '.packages | map(select(.name == "anchor")) | .[0].rust_version') + rustup override set $msrv + - name: Run cargo check + run: cargo check --workspace + cargo-udeps: + name: cargo-udeps + needs: [check-labels] + if: needs.check-labels.outputs.skip_ci != 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get latest version of nightly Rust + uses: moonrepo/setup-rust@v1 + with: + channel: nightly + bins: cargo-udeps + cache: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create Cargo config dir + run: mkdir -p .cargo + - name: Install custom Cargo config + run: cp -f .github/custom/config.toml .cargo/config.toml + - name: Run cargo udeps to identify unused crates in the dependency graph + run: make udeps + env: + # Allow warnings on Nightly + RUSTFLAGS: "" + spell-check: name: spellcheck runs-on: ubuntu-latest steps: @@ -191,4 +191,4 @@ jobs: # channel: stable # cache-target: release # - name: Run Makefile to trigger the bash script -# run: make cli +# run: make cli \ No newline at end of file From 3d83330bc46dbfef23d2baab9813dff60e842308 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 12:59:49 +1300 Subject: [PATCH 25/46] rm temp file --- temp.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 temp.txt diff --git a/temp.txt b/temp.txt deleted file mode 100644 index d606037..0000000 --- a/temp.txt +++ /dev/null @@ -1 +0,0 @@ -test2 \ No newline at end of file From f74882341e8340de625f0bf768de55fd89fedcb1 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 13:15:17 +1300 Subject: [PATCH 26/46] update futures --- Cargo.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b41410..9b05b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2832,9 +2832,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -2857,9 +2857,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -2867,15 +2867,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -2885,9 +2885,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -2901,9 +2901,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -2923,15 +2923,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-ticker" @@ -2952,9 +2952,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", From 732ed0167adeb3b37875ede4353cfee11052005e Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 13:50:31 +1300 Subject: [PATCH 27/46] tmp workaround for remaining audit error --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9db6002..eecefbc 100644 --- a/Makefile +++ b/Makefile @@ -135,8 +135,9 @@ audit: install-audit audit-CI install-audit: cargo install --force cargo-audit +# TODO - remove when updated in LH(?) audit-CI: - cargo audit + cargo audit --ignore RUSTSEC-2024-0399 # Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose. vendor: From 5e0d751471f5efaae7582f683fa1721b9689ef8f Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 15:07:22 +1300 Subject: [PATCH 28/46] update hcv path --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f6427c..de20c5c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -79,8 +79,8 @@ jobs: method: github githubToken: ${{ secrets.GH_TOKEN }} secrets: | - spesi_kv/data/gh_wf_testing DOCKER_USERNAME ; - spesi_kv/data/gh_wf_testing DOCKER_PASSWORD + spesi_kv/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/dev/docker/anchor DOCKER_PASSWORD - name: Dockerhub login run: | From 6a57a37201c8cc56875d73018118426847e76eb3 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 27 Nov 2024 15:11:50 +1300 Subject: [PATCH 29/46] try another hcv path --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index de20c5c..ea553af 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -79,8 +79,8 @@ jobs: method: github githubToken: ${{ secrets.GH_TOKEN }} secrets: | - spesi_kv/dev/docker/anchor DOCKER_USERNAME ; - spesi_kv/dev/docker/anchor DOCKER_PASSWORD + spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - name: Dockerhub login run: | From e6151203b6e20932842ce7342c57c866acdac04a Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Thu, 28 Nov 2024 10:55:19 +1300 Subject: [PATCH 30/46] Add extract-version workflow and refactor docker workflow to use it --- .github/workflows/docker.yml | 26 +- .github/workflows/extract-version.yml | 36 ++ .github/workflows/release.yml | 495 +++++++++++++------------- 3 files changed, 282 insertions(+), 275 deletions(-) create mode 100644 .github/workflows/extract-version.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ea553af..f5d3476 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,32 +17,8 @@ env: RUST_VERSION: 'abc' jobs: - # Extract the rust-version from the Cargo.toml file. - # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX - # which is either empty or `-unstable`. extract-version: - runs-on: ubuntu-22.04 - steps: - - - - name: Extract version (if stable) - if: github.event.ref == 'refs/heads/stable' - run: | - echo "VERSION=latest" >> $GITHUB_ENV - echo "VERSION_SUFFIX=" >> $GITHUB_ENV - - name: Extract version (if unstable) - if: github.event.ref == 'refs/heads/unstable' - run: | - echo "VERSION=latest" >> $GITHUB_ENV - echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV - - name: Extract version (if tagged release) - if: startsWith(github.event.ref, 'refs/tags') - run: | - echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV - echo "VERSION_SUFFIX=" >> $GITHUB_ENV - outputs: - VERSION: ${{ env.VERSION }} - VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + uses: ./.github/workflows/extract-version.yml build-docker: name: build-docker-${{ matrix.binary }} diff --git a/.github/workflows/extract-version.yml b/.github/workflows/extract-version.yml new file mode 100644 index 0000000..5ce2743 --- /dev/null +++ b/.github/workflows/extract-version.yml @@ -0,0 +1,36 @@ +name: Extract Version + +on: + workflow_call: + outputs: + VERSION: + description: "The extracted version (latest or vX.Y.Z)" + value: ${{ jobs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: + description: "The version suffix (empty or -unstable)" + value: ${{ jobs.extract-version.outputs.VERSION_SUFFIX }} + +jobs: + extract-version: + runs-on: ubuntu-22.04 + outputs: + VERSION: ${{ env.VERSION }} + VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} + steps: + - name: Extract version (if stable) + if: github.event.ref == 'refs/heads/stable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Extract version (if unstable) + if: github.event.ref == 'refs/heads/unstable' + run: | + echo "VERSION=latest" >> $GITHUB_ENV + echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV + + - name: Extract version (if tagged release) + if: startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f5c8b39..de9dcbc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,251 +1,246 @@ # TODO: Setup release pipeline -# name: Release Suite -# -# on: -# push: -# tags: -# - v* -# -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true -# -# env: -# DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} -# DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} -# REPO_NAME: ${{ github.repository_owner }}/anchor -# IMAGE_NAME: ${{ github.repository_owner }}/anchor -# # Enable self-hosted runners for the sigp repo only. -# SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} -# -# jobs: -# extract-version: -# runs-on: ubuntu-latest -# steps: -# - name: Extract version -# run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT -# id: extract_version -# outputs: -# VERSION: ${{ steps.extract_version.outputs.VERSION }} -# build: -# name: Build Release -# strategy: -# matrix: -# arch: [aarch64-unknown-linux-gnu, -# x86_64-unknown-linux-gnu, -# x86_64-apple-darwin, -# x86_64-windows] -# include: -# - arch: aarch64-unknown-linux-gnu -# runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} -# profile: maxperf -# - arch: x86_64-unknown-linux-gnu -# runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} -# profile: maxperf -# - arch: x86_64-apple-darwin -# runner: macos-13 -# profile: maxperf -# - arch: x86_64-windows -# runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }} -# profile: maxperf -# -# runs-on: ${{ matrix.runner }} -# needs: extract-version -# steps: -# - name: Checkout sources -# uses: actions/checkout@v4 -# - name: Get latest version of stable Rust -# if: env.SELF_HOSTED_RUNNERS == 'false' -# run: rustup update stable -# -# # ============================== -# # Windows dependencies -# # ============================== -# -# - uses: KyleMayes/install-llvm-action@v1 -# if: env.SELF_HOSTED_RUNNERS == 'false' && startsWith(matrix.arch, 'x86_64-windows') -# with: -# version: "17.0" -# directory: ${{ runner.temp }}/llvm -# - name: Set LIBCLANG_PATH -# if: startsWith(matrix.arch, 'x86_64-windows') -# run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV -# -# # ============================== -# # Builds -# # ============================== -# -# - name: Build Anchor for aarch64-unknown-linux-gnu -# if: matrix.arch == 'aarch64-unknown-linux-gnu' -# run: | -# cargo install cross -# env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64 -# -# - name: Build Anchor for x86_64-unknown-linux-gnu -# if: matrix.arch == 'x86_64-unknown-linux-gnu' -# run: | -# cargo install cross -# env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 -# -# - name: Move cross-compiled binary -# if: contains(matrix.arch, 'unknown-linux-gnu') -# run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/anchor ~/.cargo/bin/anchor -# -# - name: Build Anchor for x86_64-apple-darwin -# if: matrix.arch == 'x86_64-apple-darwin' -# run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} -# -# - name: Build Anchor for Windows -# if: matrix.arch == 'x86_64-windows' -# run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} -# -# - name: Configure GPG and create artifacts -# if: startsWith(matrix.arch, 'x86_64-windows') != true -# env: -# GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} -# GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} -# run: | -# export GPG_TTY=$(tty) -# echo "$GPG_SIGNING_KEY" | gpg --batch --import -# mkdir artifacts -# mv ~/.cargo/bin/anchor ./artifacts -# cd artifacts -# tar -czf anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz anchor -# echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz -# for ext in "tar.gz" "tar.gz.asc";\ -# do for f in *.$ext;\ -# do cp $f "../${f%.$ext}.$ext";\ -# done;\ -# done -# mv *tar.gz* .. -# -# - name: Configure GPG and create artifacts Windows -# if: startsWith(matrix.arch, 'x86_64-windows') -# env: -# GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} -# GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} -# run: | -# echo $env:GPG_SIGNING_KEY | gpg --batch --import -# mkdir artifacts -# move $env:USERPROFILE/.cargo/bin/anchor.exe ./artifacts -# cd artifacts -# tar -czf anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz anchor.exe -# gpg --passphrase "$env:GPG_PASSPHRASE" --batch --pinentry-mode loopback -ab anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz -# move *tar.gz* .. -# -# # ======================================================================= -# # Upload artifacts -# # This is required to share artifacts between different jobs -# # ======================================================================= -# -# - name: Upload artifact -# uses: actions/upload-artifact@v4 -# with: -# name: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz -# path: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz -# compression-level: 0 -# -# - name: Upload signature -# uses: actions/upload-artifact@v4 -# with: -# name: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc -# path: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc -# compression-level: 0 -# -# draft-release: -# name: Draft Release -# needs: [build, extract-version] -# runs-on: ubuntu-latest -# env: -# VERSION: ${{ needs.extract-version.outputs.VERSION }} -# steps: -# # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts. -# - name: Checkout sources -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# # ============================== -# # Download artifacts -# # ============================== -# -# - name: Download artifacts -# uses: actions/download-artifact@v4 -# -# # ============================== -# # Create release draft -# # ============================== -# -# - name: Generate Full Changelog -# id: changelog -# run: | -# echo "CHANGELOG<> $GITHUB_OUTPUT -# echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT -# echo "EOF" >> $GITHUB_OUTPUT -# -# - name: Create Release Draft -# env: -# GITHUB_USER: ${{ github.repository_owner }} -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# -# # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml -# run: | -# body=$(cat <<- "ENDBODY" -# -# -# ## Testing Checklist (DELETE ME) -# -# - [ ] Run on synced Holesky Sigma Prime nodes. -# - [ ] Run on synced Canary (mainnet) Sigma Prime nodes. -# - [ ] Resync a Holesky node. -# - [ ] Resync a mainnet node. -# -# ## Release Checklist (DELETE ME) -# -# - [ ] Merge `unstable` -> `stable`. -# - [ ] Ensure docker images are published (check `latest` and the version tag). -# - [ ] Prepare Discord post. -# - [ ] Prepare Twitter post. -# - [ ] Prepare mailing list email. -# -# ## Summary -# -# Add a summary. -# -# ## Update Priority -# -# This table provides priorities for which classes of users should update particular components. -# -# |User Class |Beacon Node | Validator Client| -# --- | --- | --- -# |Staking Users| | | -# |Non-Staking Users| |---| -# -# *See [Update -# Priorities](https://anchor-book.sigmaprime.io/installation-priorities.html) -# more information about this table.* -# -# ## All Changes -# -# ${{ steps.changelog.outputs.CHANGELOG }} -# -# ## Binaries -# -# [See pre-built binaries documentation.](https://anchor-book.sigmaprime.io/installation-binaries.html) -# -# The binaries are signed with Sigma Prime's PGP key: `15E66D941F697E28F49381F426416DC3F30674B0` -# -# | System | Architecture | Binary | PGP Signature | -# |:---:|:---:|:---:|:---| -# | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | -# | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | -# | | aarch64 | [anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | -# | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-windows.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-windows.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-windows.tar.gz.asc) | -# | | | | | -# | **System** | **Option** | - | **Resource** | -# | | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | -# ENDBODY -# ) -# assets=(./anchor-*.tar.gz*/anchor-*.tar.gz*) -# tag_name="${{ env.VERSION }}" -# echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}" +name: Release Suite + +on: + push: + tags: + - v* + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + REPO_NAME: ${{ github.repository_owner }}/anchor + IMAGE_NAME: ${{ github.repository_owner }}/anchor + # Enable self-hosted runners for the sigp repo only. + SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} + +jobs: + extract-version: + uses: ./.github/workflows/extract-version.yml + + build: + name: Build Release + strategy: + matrix: + arch: [aarch64-unknown-linux-gnu, + x86_64-unknown-linux-gnu, + x86_64-apple-darwin, + x86_64-windows] + include: + - arch: aarch64-unknown-linux-gnu + runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + - arch: x86_64-unknown-linux-gnu + runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release", "large"]') || 'ubuntu-latest' }} + profile: maxperf + - arch: x86_64-apple-darwin + runner: macos-13 + profile: maxperf + - arch: x86_64-windows + runner: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "windows", "release"]') || 'windows-2019' }} + profile: maxperf + + runs-on: ${{ matrix.runner }} + needs: extract-version + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Get latest version of stable Rust + if: env.SELF_HOSTED_RUNNERS == 'false' + run: rustup update stable + + # ============================== + # Windows dependencies + # ============================== + + - uses: KyleMayes/install-llvm-action@v1 + if: env.SELF_HOSTED_RUNNERS == 'false' && startsWith(matrix.arch, 'x86_64-windows') + with: + version: "17.0" + directory: ${{ runner.temp }}/llvm + - name: Set LIBCLANG_PATH + if: startsWith(matrix.arch, 'x86_64-windows') + run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV + + # ============================== + # Builds + # ============================== + + - name: Build Anchor for aarch64-unknown-linux-gnu + if: matrix.arch == 'aarch64-unknown-linux-gnu' + run: | + cargo install cross + env CROSS_PROFILE=${{ matrix.profile }} make build-aarch64 + + - name: Build Anchor for x86_64-unknown-linux-gnu + if: matrix.arch == 'x86_64-unknown-linux-gnu' + run: | + cargo install cross + env CROSS_PROFILE=${{ matrix.profile }} make build-x86_64 + + - name: Move cross-compiled binary + if: contains(matrix.arch, 'unknown-linux-gnu') + run: mv target/${{ matrix.arch }}/${{ matrix.profile }}/anchor ~/.cargo/bin/anchor + + - name: Build Anchor for x86_64-apple-darwin + if: matrix.arch == 'x86_64-apple-darwin' + run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + + - name: Build Anchor for Windows + if: matrix.arch == 'x86_64-windows' + run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + + - name: Configure GPG and create artifacts + if: startsWith(matrix.arch, 'x86_64-windows') != true + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + export GPG_TTY=$(tty) + echo "$GPG_SIGNING_KEY" | gpg --batch --import + mkdir artifacts + mv ~/.cargo/bin/anchor ./artifacts + cd artifacts + tar -czf anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz anchor + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + for ext in "tar.gz" "tar.gz.asc";\ + do for f in *.$ext;\ + do cp $f "../${f%.$ext}.$ext";\ + done;\ + done + mv *tar.gz* .. + + - name: Configure GPG and create artifacts Windows + if: startsWith(matrix.arch, 'x86_64-windows') + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + echo $env:GPG_SIGNING_KEY | gpg --batch --import + mkdir artifacts + move $env:USERPROFILE/.cargo/bin/anchor.exe ./artifacts + cd artifacts + tar -czf anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz anchor.exe + gpg --passphrase "$env:GPG_PASSPHRASE" --batch --pinentry-mode loopback -ab anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + move *tar.gz* .. + + # ======================================================================= + # Upload artifacts + # This is required to share artifacts between different jobs + # ======================================================================= + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + path: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz + compression-level: 0 + + - name: Upload signature + uses: actions/upload-artifact@v4 + with: + name: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc + path: anchor-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.arch }}.tar.gz.asc + compression-level: 0 + + draft-release: + name: Draft Release + needs: [build, extract-version] + runs-on: ubuntu-latest + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + steps: + # This is necessary for generating the changelog. It has to come before "Download Artifacts" or else it deletes the artifacts. + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # ============================== + # Download artifacts + # ============================== + + - name: Download artifacts + uses: actions/download-artifact@v4 + + # ============================== + # Create release draft + # ============================== + + - name: Generate Full Changelog + id: changelog + run: | + echo "CHANGELOG<> $GITHUB_OUTPUT + echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Create Release Draft + env: + GITHUB_USER: ${{ github.repository_owner }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # The formatting here is borrowed from OpenEthereum: https://github.com/openethereum/openethereum/blob/main/.github/workflows/build.yml + run: | + body=$(cat <<- "ENDBODY" + + + ## Testing Checklist (DELETE ME) + + - [ ] Run on synced Holesky Sigma Prime nodes. + - [ ] Run on synced Canary (mainnet) Sigma Prime nodes. + - [ ] Resync a Holesky node. + - [ ] Resync a mainnet node. + + ## Release Checklist (DELETE ME) + + - [ ] Merge `unstable` -> `stable`. + - [ ] Ensure docker images are published (check `latest` and the version tag). + - [ ] Prepare Discord post. + - [ ] Prepare Twitter post. + - [ ] Prepare mailing list email. + + ## Summary + + Add a summary. + + ## Update Priority + + This table provides priorities for which classes of users should update particular components. + + |User Class |Beacon Node | Validator Client| + --- | --- | --- + |Staking Users| | | + |Non-Staking Users| |---| + + *See [Update + Priorities](https://anchor-book.sigmaprime.io/installation-priorities.html) + more information about this table.* + + ## All Changes + + ${{ steps.changelog.outputs.CHANGELOG }} + + ## Binaries + + [See pre-built binaries documentation.](https://anchor-book.sigmaprime.io/installation-binaries.html) + + The binaries are signed with Sigma Prime's PGP key: `15E66D941F697E28F49381F426416DC3F30674B0` + + | System | Architecture | Binary | PGP Signature | + |:---:|:---:|:---:|:---| + | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | + | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | + | | aarch64 | [anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | + | | x86_64 | [anchor-${{ env.VERSION }}-x86_64-windows.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-windows.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/anchor-${{ env.VERSION }}-x86_64-windows.tar.gz.asc) | + | | | | | + | **System** | **Option** | - | **Resource** | + | | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | + ENDBODY + ) + assets=(./anchor-*.tar.gz*/anchor-*.tar.gz*) + tag_name="${{ env.VERSION }}" + echo "$body" | gh release create --draft -F "-" "$tag_name" "${assets[@]}" From 2d98a2dbd9db5a96483f37ea4cbe82e29b5bc458 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Thu, 28 Nov 2024 12:02:34 +1300 Subject: [PATCH 31/46] Refactor Docker workflow to support single and multi-architecture builds --- .github/workflows/docker.yml | 59 +++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f5d3476..fccd291 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,13 +20,13 @@ jobs: extract-version: uses: ./.github/workflows/extract-version.yml - build-docker: - name: build-docker-${{ matrix.binary }} + build-docker-single-arch: + name: build-docker-anchor-${{ matrix.cpu_arch }} # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} strategy: matrix: - binary: [anchor] + cpu_arch: [aarch64, x86_64] include: - profile: maxperf @@ -64,25 +64,70 @@ jobs: - name: Build binary run: | - make + cargo install cross + env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} - name: Make bin dir run: mkdir ./bin - name: Move built binary into Docker scope - run: mv ./target/release/${{ matrix.binary }} ./bin + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin + + - name: Map aarch64 to arm64 short arch + if: startsWith(matrix.cpu_arch, 'aarch64') + run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + + - name: Map x86_64 to amd64 short arch + if: startsWith(matrix.cpu_arch, 'x86_64') + run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV + + - name: Install QEMU + if: env.SELF_HOSTED_RUNNERS == 'false' + run: sudo apt-get update && sudo apt-get install -y qemu-user-static - name: Set up Docker Buildx if: env.SELF_HOSTED_RUNNERS == 'false' uses: docker/setup-buildx-action@v3 - - name: Build and push (Anchor) + - name: Build and push uses: docker/build-push-action@v5 with: file: ./anchor/Dockerfile context: . + platforms: linux/${{ env.SHORT_ARCH }} push: true tags: | - ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} + ${{ github.repository_owner}}/anchor:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} build-args: | RUST_VERSION=${{ env.RUST_VERSION }} + + build-docker-multiarch: + name: build-docker-anchor-multiarch + runs-on: ubuntu-22.04 + needs: [build-docker-single-arch, extract-version] + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Retrieve Docker credentials from Vault + uses: hashicorp/vault-action@v2 + with: + url: https://vault.sigp.io + method: github + githubToken: ${{ secrets.GH_TOKEN }} + secrets: | + spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + - name: Create and push multiarch manifests + run: | + docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} From 9533df25cf48cd5ef1978cbc6e47c6e9df700b98 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Mon, 2 Dec 2024 11:09:43 +1300 Subject: [PATCH 32/46] Enhance extract-version workflow to default to 'latest' and add debug step --- .github/workflows/extract-version.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extract-version.yml b/.github/workflows/extract-version.yml index 5ce2743..34de80e 100644 --- a/.github/workflows/extract-version.yml +++ b/.github/workflows/extract-version.yml @@ -5,7 +5,7 @@ on: outputs: VERSION: description: "The extracted version (latest or vX.Y.Z)" - value: ${{ jobs.extract-version.outputs.VERSION }} + value: ${{ jobs.extract-version.outputs.VERSION || 'latest' }} VERSION_SUFFIX: description: "The version suffix (empty or -unstable)" value: ${{ jobs.extract-version.outputs.VERSION_SUFFIX }} @@ -17,6 +17,8 @@ jobs: VERSION: ${{ env.VERSION }} VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} steps: + + - name: Extract version (if stable) if: github.event.ref == 'refs/heads/stable' run: | @@ -28,9 +30,21 @@ jobs: run: | echo "VERSION=latest" >> $GITHUB_ENV echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV + + - name: Extract version (if other branch) + if: github.event.ref != 'refs/heads/stable' && github.event.ref != 'refs/heads/unstable' && !startsWith(github.event.ref, 'refs/tags') + run: | + echo "VERSION=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo "VERSION_SUFFIX=" >> $GITHUB_ENV - name: Extract version (if tagged release) if: startsWith(github.event.ref, 'refs/tags') run: | echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV echo "VERSION_SUFFIX=" >> $GITHUB_ENV + + - name: Debug - Print environment variables + run: | + echo "[debug] VERSION=${{ env.VERSION }}" + echo "[debug] VERSION_SUFFIX=${{ env.VERSION_SUFFIX }}" + From ae912311ef02b833b3bc7191c4df4b8d50af9aa7 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 14:17:49 +1300 Subject: [PATCH 33/46] Refactor Docker workflow to improve architecture support and add debugging steps --- .github/workflows/docker.yml | 208 +++++++++++++++++++++-------------- 1 file changed, 125 insertions(+), 83 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fccd291..bfaeea9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: docker +name: Docker build and push on: push: @@ -15,6 +15,7 @@ env: # Enable self-hosted runners for the sigp repo only. SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} RUST_VERSION: 'abc' + SHORT_ARCH: 'amd64' jobs: extract-version: @@ -27,6 +28,8 @@ jobs: strategy: matrix: cpu_arch: [aarch64, x86_64] + # cpu_arch: [x86_64] + # cpu_arch: [aarch64] include: - profile: maxperf @@ -35,10 +38,40 @@ jobs: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} steps: - - uses: actions/checkout@v4 - - name: Update Rust - if: env.SELF_HOSTED_RUNNERS == 'false' - run: rustup update stable + - name: Debug extract-version output + run: | + echo "[docker.yml] VERSION: ${{ needs.extract-version.outputs.VERSION }}" + echo "[docker.yml] VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}" + + - name: Map aarch64 to arm64 short arch + if: startsWith(matrix.cpu_arch, 'aarch64') + run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + + - name: Map x86_64 to amd64 short arch + if: startsWith(matrix.cpu_arch, 'x86_64') + run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV + + - name: Debug short arch + run: echo "SHORT_ARCH=${{ env.SHORT_ARCH }}" + + - name: Install Rust and Cargo + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV + echo "rustup toolchain list is: $(rustup toolchain list)" + cargo --version + + - name: cargo install cross + run: | + cargo install cross + + # - uses: actions/checkout@v4 + # - name: Update Rust + # if: env.SELF_HOSTED_RUNNERS == 'false' + # run: rustup update stable + - name: Checkout sources + uses: actions/checkout@v4 - name: Get rust-version id: get-rust-version @@ -48,86 +81,95 @@ jobs: - name: Get latest version of stable Rust run: echo "rust version is ${{ env.RUST_VERSION }}" - - name: Retrieve Docker credentials from Vault - uses: hashicorp/vault-action@v2 - with: - url: https://vault.sigp.io - method: github - githubToken: ${{ secrets.GH_TOKEN }} - secrets: | - spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; - spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + # - name: Retrieve Docker credentials from Vault + # uses: hashicorp/vault-action@v2 + # with: + # url: https://vault.sigp.io + # method: github + # githubToken: ${{ secrets.GH_TOKEN }} + # secrets: | + # spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + # spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - name: Build binary - run: | - cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} + # - name: Dockerhub login + # run: | + # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Make bin dir - run: mkdir ./bin - - - name: Move built binary into Docker scope - run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin - - - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.cpu_arch, 'aarch64') - run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV - - - name: Map x86_64 to amd64 short arch - if: startsWith(matrix.cpu_arch, 'x86_64') - run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - - - name: Install QEMU - if: env.SELF_HOSTED_RUNNERS == 'false' - run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - - name: Set up Docker Buildx - if: env.SELF_HOSTED_RUNNERS == 'false' - uses: docker/setup-buildx-action@v3 - - - name: Build and push - uses: docker/build-push-action@v5 - with: - file: ./anchor/Dockerfile - context: . - platforms: linux/${{ env.SHORT_ARCH }} - push: true - tags: | - ${{ github.repository_owner}}/anchor:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} - build-args: | - RUST_VERSION=${{ env.RUST_VERSION }} - - build-docker-multiarch: - name: build-docker-anchor-multiarch - runs-on: ubuntu-22.04 - needs: [build-docker-single-arch, extract-version] - env: - VERSION: ${{ needs.extract-version.outputs.VERSION }} - VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Retrieve Docker credentials from Vault - uses: hashicorp/vault-action@v2 - with: - url: https://vault.sigp.io - method: github - githubToken: ${{ secrets.GH_TOKEN }} - secrets: | - spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; - spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - - name: Dockerhub login + - name: Build binary run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + # cargo install cross + # env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} + echo "### matrix.cpu_arch is ${{ matrix.cpu_arch }}" + echo "### matrix.profile is ${{ matrix.profile }}" - - name: Create and push multiarch manifests + - name: Set `make` command for anchor run: | - docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} + echo "### cpu_arch is ${{ matrix.cpu_arch }}" + echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV + + # - name: Make bin dir + # run: mkdir ./bin + + # - name: Move built binary into Docker scope + # run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin + + + + # - name: Install QEMU + # if: env.SELF_HOSTED_RUNNERS == 'false' + # run: sudo apt-get update && sudo apt-get install -y qemu-user-static + + + # - name: Debug current directory + # run: echo "### The current directory is $(pwd)" + + + # - name: Set up Docker Buildx + # if: env.SELF_HOSTED_RUNNERS == 'false' + # uses: docker/setup-buildx-action@v3 + + # - name: Build and push + # uses: docker/build-push-action@v5 + # with: + # file: ./anchor/Dockerfile.cross + # context: . + # platforms: linux/${{ env.SHORT_ARCH }} + # push: true + # tags: | + # ${{ github.repository_owner}}/anchor:${{ env.VERSION }} + # build-args: | + # RUST_VERSION=${{ env.RUST_VERSION }} + + # build-docker-multiarch: + # name: build-docker-anchor-multiarch + # runs-on: ubuntu-22.04 + # strategy: + # matrix: + # binary: [anchor] + # needs: [build-docker-single-arch, extract-version] + # env: + # VERSION: ${{ needs.extract-version.outputs.VERSION }} + # VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + # steps: + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Retrieve Docker credentials from Vault + # uses: hashicorp/vault-action@v2 + # with: + # url: https://vault.sigp.io + # method: github + # githubToken: ${{ secrets.GH_TOKEN }} + # secrets: | + # spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + # spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD + + # - name: Dockerhub login + # run: | + # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + # - name: Create and push multiarch manifests + # run: | + # docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ + # ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ + # ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} From fee33f87c7a22f2427e34bdd604986d4ca19d233 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 14:36:55 +1300 Subject: [PATCH 34/46] Refactor Docker workflow to enable multi-architecture builds and streamline credential retrieval --- .github/workflows/docker.yml | 116 +++++++++++++++++------------------ 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bfaeea9..bb2b714 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -81,24 +81,24 @@ jobs: - name: Get latest version of stable Rust run: echo "rust version is ${{ env.RUST_VERSION }}" - # - name: Retrieve Docker credentials from Vault - # uses: hashicorp/vault-action@v2 - # with: - # url: https://vault.sigp.io - # method: github - # githubToken: ${{ secrets.GH_TOKEN }} - # secrets: | - # spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; - # spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - # - name: Dockerhub login - # run: | - # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Retrieve Docker credentials from Vault + uses: hashicorp/vault-action@v2 + with: + url: https://vault.sigp.io + method: github + githubToken: ${{ secrets.GH_TOKEN }} + secrets: | + spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - name: Build binary run: | - # cargo install cross - # env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} + cargo install cross + env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} echo "### matrix.cpu_arch is ${{ matrix.cpu_arch }}" echo "### matrix.profile is ${{ matrix.profile }}" @@ -107,11 +107,11 @@ jobs: echo "### cpu_arch is ${{ matrix.cpu_arch }}" echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV - # - name: Make bin dir - # run: mkdir ./bin + - name: Make bin dir + run: mkdir ./bin - # - name: Move built binary into Docker scope - # run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin + - name: Move built binary into Docker scope + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin @@ -128,45 +128,45 @@ jobs: # if: env.SELF_HOSTED_RUNNERS == 'false' # uses: docker/setup-buildx-action@v3 - # - name: Build and push - # uses: docker/build-push-action@v5 - # with: - # file: ./anchor/Dockerfile.cross - # context: . - # platforms: linux/${{ env.SHORT_ARCH }} - # push: true - # tags: | - # ${{ github.repository_owner}}/anchor:${{ env.VERSION }} - # build-args: | - # RUST_VERSION=${{ env.RUST_VERSION }} - - # build-docker-multiarch: - # name: build-docker-anchor-multiarch - # runs-on: ubuntu-22.04 - # strategy: - # matrix: - # binary: [anchor] - # needs: [build-docker-single-arch, extract-version] - # env: - # VERSION: ${{ needs.extract-version.outputs.VERSION }} - # VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - # steps: - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Retrieve Docker credentials from Vault - # uses: hashicorp/vault-action@v2 - # with: - # url: https://vault.sigp.io - # method: github - # githubToken: ${{ secrets.GH_TOKEN }} - # secrets: | - # spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; - # spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - # - name: Dockerhub login - # run: | - # echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Build and push + uses: docker/build-push-action@v5 + with: + file: ./anchor/Dockerfile.cross + context: . + platforms: linux/${{ env.SHORT_ARCH }} + push: true + tags: | + ${{ github.repository_owner}}/anchor:${{ env.VERSION }} + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} + + build-docker-multiarch: + name: build-docker-anchor-multiarch + runs-on: ubuntu-22.04 + strategy: + matrix: + binary: [anchor] + needs: [build-docker-single-arch, extract-version] + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Retrieve Docker credentials from Vault + uses: hashicorp/vault-action@v2 + with: + url: https://vault.sigp.io + method: github + githubToken: ${{ secrets.GH_TOKEN }} + secrets: | + spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD + + - name: Dockerhub login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin # - name: Create and push multiarch manifests # run: | From bd2240dad50b3b39c03edf2bd8b474005e307937 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 14:44:38 +1300 Subject: [PATCH 35/46] rm jemalloc --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eecefbc..ee3f76d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ BUILD_PATH_AARCH64 = "target/$(AARCH64_TAG)/release" PINNED_NIGHTLY ?= nightly # List of features to use when cross-compiling. Can be overridden via the environment. -CROSS_FEATURES ?= jemalloc +# CROSS_FEATURES ?= jemalloc +CROSS_FEATURES ?= # Cargo profile for Cross builds. Default is for local builds, CI uses an override. CROSS_PROFILE ?= release From 4c4c454800db210e6261888eb76e0864d71f7337 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 15:06:01 +1300 Subject: [PATCH 36/46] try with TARGETPLATFORM arg --- .github/workflows/docker.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bb2b714..c98e6d8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -115,18 +115,18 @@ jobs: - # - name: Install QEMU - # if: env.SELF_HOSTED_RUNNERS == 'false' - # run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Install QEMU + if: env.SELF_HOSTED_RUNNERS == 'false' + run: sudo apt-get update && sudo apt-get install -y qemu-user-static # - name: Debug current directory # run: echo "### The current directory is $(pwd)" - # - name: Set up Docker Buildx - # if: env.SELF_HOSTED_RUNNERS == 'false' - # uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + if: env.SELF_HOSTED_RUNNERS == 'false' + uses: docker/setup-buildx-action@v3 - name: Build and push uses: docker/build-push-action@v5 @@ -139,6 +139,7 @@ jobs: ${{ github.repository_owner}}/anchor:${{ env.VERSION }} build-args: | RUST_VERSION=${{ env.RUST_VERSION }} + TARGETPLATFORM=linux/${{ env.SHORT_ARCH }} build-docker-multiarch: name: build-docker-anchor-multiarch From 80fe1cd57e608e2aa3d3dc5f98e3fc71cd810192 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 15:21:48 +1300 Subject: [PATCH 37/46] Update Docker workflow to include architecture-specific tags and enable multiarch manifest creation --- .github/workflows/docker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c98e6d8..5ac6aa4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -136,7 +136,7 @@ jobs: platforms: linux/${{ env.SHORT_ARCH }} push: true tags: | - ${{ github.repository_owner}}/anchor:${{ env.VERSION }} + ${{ github.repository_owner}}/anchor:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} build-args: | RUST_VERSION=${{ env.RUST_VERSION }} TARGETPLATFORM=linux/${{ env.SHORT_ARCH }} @@ -169,8 +169,8 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - # - name: Create and push multiarch manifests - # run: | - # docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ - # ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ - # ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} + - name: Create and push multiarch manifests + run: | + docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} From ed1b6f619eaf8797a190ed9ac18598bb3b63b269 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Tue, 3 Dec 2024 15:56:16 +1300 Subject: [PATCH 38/46] disable buildx imagetools, add labels --- .github/workflows/docker.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5ac6aa4..7580372 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -134,12 +134,18 @@ jobs: file: ./anchor/Dockerfile.cross context: . platforms: linux/${{ env.SHORT_ARCH }} + labels: | + git.revision=${{ github.sha }} + git.branch=${{ github.ref }} + git.tag=${{ github.ref }} + git.repository=${{ github.repository }} push: true tags: | ${{ github.repository_owner}}/anchor:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} build-args: | RUST_VERSION=${{ env.RUST_VERSION }} TARGETPLATFORM=linux/${{ env.SHORT_ARCH }} + GIT_COMMIT_HASH=${{ github.sha }} build-docker-multiarch: name: build-docker-anchor-multiarch @@ -169,8 +175,8 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Create and push multiarch manifests - run: | - docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ - ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} + # - name: Create and push multiarch manifests + # run: | + # docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ + # ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ + # ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} From d9fec9e107841af73254861db576dea2c9d07fb8 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 08:43:59 +1300 Subject: [PATCH 39/46] enable md book, only run on sipg org --- .github/workflows/book.yml | 57 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 86b0020..db295c2 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -1,34 +1,35 @@ -# name: mdbook +name: Markdown book build and upload to S3 -# on: -# push: -# branches: -# - unstable +on: + push: + branches: + - unstable -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true -# jobs: -# build-and-upload-to-s3: -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 +jobs: + build-and-upload-to-s3: + if: github.repository_owner == 'sigp' + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 -# - name: Setup mdBook -# uses: peaceiris/actions-mdbook@v1 -# with: -# mdbook-version: 'latest' + - name: Setup mdBook + uses: peaceiris/actions-mdbook@v1 + with: + mdbook-version: 'latest' -# - run: mdbook build -# working-directory: book + - run: mdbook build + working-directory: book -# - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83 -# with: -# args: --follow-symlinks --delete -# env: -# AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# AWS_REGION: 'ap-southeast-2' -# SOURCE_DIR: 'book/book' + - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83 + with: + args: --follow-symlinks --delete + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'ap-southeast-2' + SOURCE_DIR: 'book/book' From eb7d03201dc04cacb8d1b3510c98928d6483e7c9 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 10:19:47 +1300 Subject: [PATCH 40/46] fix yml formatting --- .github/workflows/book.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index db295c2..ba65645 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -26,10 +26,10 @@ jobs: - uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83 with: - args: --follow-symlinks --delete - env: - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'ap-southeast-2' - SOURCE_DIR: 'book/book' + args: --follow-symlinks --delete + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BOOK_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'ap-southeast-2' + SOURCE_DIR: 'book/book' From d3fe7dfa9e3927dda0ff211b194af9623045d6c5 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 10:44:23 +1300 Subject: [PATCH 41/46] re-Implement linkcheck workflow for mdbook with conditional execution for sigp organization --- .github/workflows/linkcheck.yml | 74 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 5ceca2e..32b0402 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,37 +1,37 @@ -# TODO: Setup mdbook -# name: linkcheck -# -# on: -# push: -# branches: -# - unstable -# pull_request: -# paths: -# - 'book/**' -# merge_group: -# -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true -# -# jobs: -# linkcheck: -# name: Check broken links -# runs-on: ubuntu-latest -# -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# -# - name: Run mdbook server -# run: | -# docker run -v ${{ github.workspace }}/book:/book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0 -# sleep 5 -# -# - name: Print logs -# run: docker logs book -# -# - name: Run linkcheck -# run: | -# curl -sL https://github.com/filiph/linkcheck/releases/download/3.0.0/linkcheck-3.0.0-linux-x64.tar.gz | tar xvzf - linkcheck/linkcheck --strip 1 -# ./linkcheck localhost:3000 -d +name: linkcheck + +on: + push: + branches: + - unstable + pull_request: + paths: + - 'book/**' + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + linkcheck: + name: Check broken links + if: github.repository_owner == 'sigp' + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run mdbook server + run: | + docker run -v ${{ github.workspace }}/book:/book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0 + sleep 5 + + - name: Print logs + run: docker logs book + + - name: Run linkcheck + run: | + curl -sL https://github.com/filiph/linkcheck/releases/download/3.0.0/linkcheck-3.0.0-linux-x64.tar.gz | tar xvzf - linkcheck/linkcheck --strip 1 + ./linkcheck localhost:3000 -d From 11ba2fdb7c82cf91d579d6420182c05751df4663 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 11:01:24 +1300 Subject: [PATCH 42/46] Add release, debug, and code quality test jobs to Docker workflow --- .github/workflows/docker.yml | 66 ++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7580372..2ef7aa9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,24 +16,69 @@ env: SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }} RUST_VERSION: 'abc' SHORT_ARCH: 'amd64' + # Deny warnings in CI + RUSTFLAGS: "-D warnings -C debuginfo=0" + # Prevent Github API rate limiting + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: extract-version: uses: ./.github/workflows/extract-version.yml + # Critical test jobs from test-suite.yml + release-tests: + name: release-tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-nextest + - name: Run tests in release + run: make nextest-release + + debug-tests: + name: debug-tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + bins: cargo-nextest + - name: Run tests in debug + run: make nextest-debug + + code-quality: + name: code-quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get latest version of stable Rust + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + components: rustfmt,clippy + - name: Check formatting with cargo fmt + run: make cargo-fmt + - name: Lint code for quality and style with Clippy + run: make lint + build-docker-single-arch: name: build-docker-anchor-${{ matrix.cpu_arch }} - # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} strategy: matrix: cpu_arch: [aarch64, x86_64] - # cpu_arch: [x86_64] - # cpu_arch: [aarch64] include: - profile: maxperf - needs: [extract-version] + needs: [extract-version, release-tests, debug-tests, code-quality] env: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} @@ -113,17 +158,10 @@ jobs: - name: Move built binary into Docker scope run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin - - - name: Install QEMU if: env.SELF_HOSTED_RUNNERS == 'false' run: sudo apt-get update && sudo apt-get install -y qemu-user-static - - # - name: Debug current directory - # run: echo "### The current directory is $(pwd)" - - - name: Set up Docker Buildx if: env.SELF_HOSTED_RUNNERS == 'false' uses: docker/setup-buildx-action@v3 @@ -174,9 +212,3 @@ jobs: - name: Dockerhub login run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - # - name: Create and push multiarch manifests - # run: | - # docker buildx imagetools create -t ${{ github.repository_owner}}/anchor:${VERSION}${VERSION_SUFFIX} \ - # ${{ github.repository_owner}}/anchor:${VERSION}-arm64${VERSION_SUFFIX} \ - # ${{ github.repository_owner}}/anchor:${VERSION}-amd64${VERSION_SUFFIX} From cb2bbcf5319dc66b39a4306699346a430a2a6fce Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 11:48:25 +1300 Subject: [PATCH 43/46] remove debugging --- .github/workflows/docker.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2ef7aa9..adfe078 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,16 +96,13 @@ jobs: if: startsWith(matrix.cpu_arch, 'x86_64') run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV - - name: Debug short arch - run: echo "SHORT_ARCH=${{ env.SHORT_ARCH }}" - name: Install Rust and Cargo run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source $HOME/.cargo/env echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV - echo "rustup toolchain list is: $(rustup toolchain list)" - cargo --version + - name: cargo install cross run: | @@ -133,7 +130,7 @@ jobs: method: github githubToken: ${{ secrets.GH_TOKEN }} secrets: | - spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; + spesi_kv/data/dev/docker/anchor DOCKER_USERNAME spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - name: Dockerhub login @@ -144,12 +141,10 @@ jobs: run: | cargo install cross env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }} - echo "### matrix.cpu_arch is ${{ matrix.cpu_arch }}" - echo "### matrix.profile is ${{ matrix.profile }}" + - name: Set `make` command for anchor run: | - echo "### cpu_arch is ${{ matrix.cpu_arch }}" echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV - name: Make bin dir From ebc6d86d495bc9de7546b36d34dae97ab799389c Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 11:49:13 +1300 Subject: [PATCH 44/46] remove debugging --- .github/workflows/docker.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index adfe078..2f32937 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -83,10 +83,6 @@ jobs: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} steps: - - name: Debug extract-version output - run: | - echo "[docker.yml] VERSION: ${{ needs.extract-version.outputs.VERSION }}" - echo "[docker.yml] VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}" - name: Map aarch64 to arm64 short arch if: startsWith(matrix.cpu_arch, 'aarch64') From 5036e06b0d525905e0d13d80ce0d5dcc3d3a0344 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 11:54:22 +1300 Subject: [PATCH 45/46] remove unneeded job --- .github/workflows/docker.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2f32937..8bee8a6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -175,31 +175,3 @@ jobs: RUST_VERSION=${{ env.RUST_VERSION }} TARGETPLATFORM=linux/${{ env.SHORT_ARCH }} GIT_COMMIT_HASH=${{ github.sha }} - - build-docker-multiarch: - name: build-docker-anchor-multiarch - runs-on: ubuntu-22.04 - strategy: - matrix: - binary: [anchor] - needs: [build-docker-single-arch, extract-version] - env: - VERSION: ${{ needs.extract-version.outputs.VERSION }} - VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Retrieve Docker credentials from Vault - uses: hashicorp/vault-action@v2 - with: - url: https://vault.sigp.io - method: github - githubToken: ${{ secrets.GH_TOKEN }} - secrets: | - spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ; - spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD - - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin From 584f9f211802ed1732f5995864b763992f32d722 Mon Sep 17 00:00:00 2001 From: Anton Hughes Date: Wed, 4 Dec 2024 13:13:34 +1300 Subject: [PATCH 46/46] remove features --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de9dcbc..04ad80c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,11 +88,11 @@ jobs: - name: Build Anchor for x86_64-apple-darwin if: matrix.arch == 'x86_64-apple-darwin' - run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + run: cargo install --path anchor --force --locked --profile ${{ matrix.profile }} - name: Build Anchor for Windows if: matrix.arch == 'x86_64-windows' - run: cargo install --path anchor --force --locked --features portable,gnosis --profile ${{ matrix.profile }} + run: cargo install --path anchor --force --locked --profile ${{ matrix.profile }} - name: Configure GPG and create artifacts if: startsWith(matrix.arch, 'x86_64-windows') != true