Merge pull request #3097 from autonomys/fix-small-gpu-support-backport #405
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This action enables building container images and executables for farmer and node, can be triggered manually or by | |
| # release creation. | |
| # | |
| # Container images are only pushed to GitHub Container Registry for releases. | |
| # Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets. | |
| name: Snapshot build | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "snapshot-*" | |
| - "gemini-*" | |
| # Incremental compilation here isn't helpful | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| jobs: | |
| container-linux: | |
| runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }} | |
| permissions: | |
| contents: write | |
| packages: write | |
| strategy: | |
| matrix: | |
| image: | |
| - farmer | |
| - node | |
| - bootstrap-node | |
| platform: | |
| - arch: linux/amd64 | |
| dockerfile-suffix: "" | |
| suffix: ubuntu-x86_64-${{ github.ref_name }} | |
| image-suffix: "" | |
| rustflags: "-C target-cpu=skylake" | |
| # We build AArch64 | |
| - arch: linux/amd64 | |
| dockerfile-suffix: ".aarch64" | |
| suffix: ubuntu-aarch64-${{ github.ref_name }} | |
| image-suffix: "-aarch64" | |
| fail-fast: false | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0 | |
| - name: Log into registry | |
| uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: | | |
| ghcr.io/${{ github.repository_owner }}/${{ matrix.image }} | |
| tags: | | |
| type=ref,event=tag | |
| type=ref,event=branch | |
| type=sha,format=long | |
| flavor: | | |
| latest=false | |
| suffix=${{ matrix.platform.image-suffix }} | |
| - name: Build and push ${{ matrix.image }} image | |
| uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 | |
| with: | |
| file: Dockerfile-${{ matrix.image }}${{ matrix.platform.dockerfile-suffix }} | |
| platforms: ${{ matrix.platform.arch }} | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }} | |
| RUSTFLAGS=${{ matrix.platform.rustflags }} | |
| - name: Trigger trivy-security-scan Workflow | |
| uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| event-type: trivy-scan-dispatch | |
| client-payload: '{"image": "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.sha }}"}' | |
| executables: | |
| strategy: | |
| matrix: | |
| build: | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: x86_64-unknown-linux-gnu | |
| suffix: ubuntu-x86_64-skylake-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=skylake" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: x86_64-unknown-linux-gnu | |
| suffix: ubuntu-x86_64-v2-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=x86-64-v2" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: aarch64-unknown-linux-gnu | |
| suffix: ubuntu-aarch64-${{ github.ref_name }} | |
| rustflags: "-C linker=aarch64-linux-gnu-gcc" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' || '"macos-14"') }} | |
| target: aarch64-apple-darwin | |
| suffix: macos-aarch64-${{ github.ref_name }} | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-skylake-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=skylake" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-v2-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=x86-64-v2" | |
| fail-fast: false | |
| runs-on: ${{ matrix.build.os }} | |
| env: | |
| PRODUCTION_TARGET: target/${{ matrix.build.target }}/production | |
| RUSTFLAGS: ${{ matrix.build.rustflags }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| # On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support | |
| - name: Install LLVM and Clang for macOS | |
| uses: KyleMayes/install-llvm-action@dec985c8d7b46a2f363ea1a78f660c946a3349ea # v2.0.1 | |
| with: | |
| env: true | |
| version: 17 | |
| if: runner.os == 'macOS' | |
| # Because macOS, see https://andreasfertig.blog/2021/02/clang-and-gcc-on-macos-catalina-finding-the-include-paths/ | |
| - name: Configure C compiler macOS | |
| run: | | |
| echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV | |
| if: runner.os == 'macOS' | |
| - name: Install glibtoolize (macOS) | |
| run: brew install libtool | |
| if: runner.os == 'macOS' | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Needed for hwloc | |
| - name: Install automake (macOS) | |
| run: brew install automake | |
| if: runner.os == 'macOS' | |
| # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll | |
| - name: Remove msys64 | |
| run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse | |
| # Doesn't exist on self-hosted runners | |
| if: matrix.os == 'windows-2022' | |
| - name: AArch64 cross-compile packages | |
| run: | | |
| FLAVOR="$(lsb_release -sc)" | |
| sudo tee /etc/apt/sources.list.d/arm64.list <<LIST | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} main restricted | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates main restricted | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} universe | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates universe | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR} multiverse | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-updates multiverse | |
| deb [arch=arm64] http://ports.ubuntu.com/ ${FLAVOR}-backports main restricted universe multiverse | |
| LIST | |
| sudo sed -i 's/deb http/deb [arch=amd64] http/' /etc/apt/sources.list | |
| # GitHub runners use mirror file | |
| sudo sed -i 's/deb mirror/deb [arch=amd64] mirror/' /etc/apt/sources.list | |
| sudo dpkg --add-architecture arm64 | |
| sudo apt-get update | |
| # zlib1g-dev:arm64 is only necessary because amd64 version is present on the host and cross-compilation of | |
| # hwlocality-sys fails otherwise | |
| sudo apt-get install -y --no-install-recommends \ | |
| g++-aarch64-linux-gnu \ | |
| gcc-aarch64-linux-gnu \ | |
| libc6-dev-arm64-cross \ | |
| zlib1g-dev:arm64 | |
| echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV | |
| if: matrix.build.target == 'aarch64-unknown-linux-gnu' | |
| - name: CUDA toolchain | |
| uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16 | |
| with: | |
| cuda: '12.4.1' | |
| method: network | |
| sub-packages: '["nvcc", "cudart"]' | |
| if: runner.os == 'Linux' || runner.os == 'Windows' | |
| - name: Configure cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer | |
| if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64') | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda | |
| # TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now | |
| if: (runner.os == 'Linux' || runner.os == 'Windows') && startsWith(matrix.build.target, 'x86_64') | |
| - name: Build node | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node | |
| - name: Sign Application (macOS) | |
| run: | | |
| echo "Importing certificate" | |
| echo "${{ secrets.MACOS_CERTIFICATE }}" | base64 --decode > certificate.p12 | |
| security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security default-keychain -s build.keychain | |
| security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign | |
| security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| echo "Signing farmer" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer | |
| echo "Signing node" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node | |
| echo "Creating an archive" | |
| mkdir ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip | |
| echo "Notarizing" | |
| brew update | |
| brew install mitchellh/gon/gon | |
| cat << EOF > gon.hcl | |
| source = ["subspace-binaries.zip"] | |
| bundle_id = "${{ secrets.MACOS_BUNDLE_ID }}" | |
| sign { | |
| application_identity = "${{ secrets.MACOS_IDENTITY }}" | |
| } | |
| apple_id { | |
| username = "${{ secrets.MACOS_APPLE_ID }}" | |
| password = "${{ secrets.MACOS_APP_PW }}" | |
| } | |
| EOF | |
| gon -log-level=info -log-json gon.hcl | |
| # Notarize the ZIP using notarytool | |
| xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait | |
| # // todo stapling for macOS artifacts | |
| # Staple the zip package | |
| # xcrun stapler staple subspace-binaries.zip | |
| echo "Done!" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'macOS' | |
| - name: Sign Application (Windows) | |
| run: | | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe" | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'Windows' | |
| - name: Prepare executables for uploading (Ubuntu) | |
| run: | | |
| mkdir executables | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }} | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }} | |
| if: runner.os == 'Linux' | |
| - name: Prepare executables for uploading (macOS) | |
| run: | | |
| mkdir executables | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }} | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }} | |
| # Zip it so that signature is not lost | |
| ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip | |
| ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip | |
| rm executables/subspace-farmer-${{ matrix.build.suffix }} | |
| rm executables/subspace-node-${{ matrix.build.suffix }} | |
| if: runner.os == 'macOS' | |
| - name: Prepare executables for uploading (Windows) | |
| run: | | |
| mkdir executables | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe | |
| if: runner.os == 'Windows' | |
| - name: Upload node and farmer executables to artifacts | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3 | |
| with: | |
| name: executables-${{ matrix.build.suffix }} | |
| path: | | |
| executables/* | |
| if-no-files-found: error | |
| - name: Upload node and farmer executables to assets | |
| uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| asset_paths: '["executables/*"]' | |
| if: github.event_name == 'push' && github.ref_type == 'tag' |