Snapshot build #452

Workflow file for this run

.github/workflows/snapshot-build.yml at 71c9387

	# This action enables building container images and executables for farmer and node, can be triggered manually or by
	# release creation.
	#
	# Container images are only pushed to GitHub Container Registry for releases.
	# Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets.
	name: Snapshot build

	on:
	workflow_dispatch:
	push:
	tags:
	- "snapshot-*"
	- "taurus-*"
	- "mainnet-*"

	# Incremental compilation here isn't helpful
	env:
	CARGO_INCREMENTAL: 0

	jobs:
	# This will build container images and then extract executables out of them
	ubuntu:
	runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-22.04-x86-64"]' \|\| '"ubuntu-22.04"') }}
	permissions:
	contents: write
	packages: write
	strategy:
	matrix:
	build:
	- image: farmer
	base-artifact: subspace-farmer
	upload-executables: true
	- image: node
	base-artifact: subspace-node
	upload-executables: true
	- image: gateway
	base-artifact: subspace-gateway
	upload-executables: true
	- image: bootstrap-node
	base-artifact: subspace-bootstrap-node
	upload-executables: false
	fail-fast: false

	steps:
	- name: Set up QEMU
	uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
	if: github.repository_owner == 'autonomys'

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
	with:
	# Limit concurrency so it can complete with small official runners
	buildkitd-config-inline: \|
	[worker.oci]
	max-parallelism = 1
	if: github.repository_owner != 'autonomys'

	- name: Log into registry
	uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ github.token }}

	- name: Extract Docker metadata
	id: meta
	uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
	with:
	images: \|
	ghcr.io/${{ github.repository_owner }}/${{ matrix.build.image }}
	tags: \|
	type=ref,event=tag
	type=ref,event=branch
	type=sha,format=long
	flavor: \|
	latest=false
	suffix=

	- name: Build and push ${{ matrix.build.image }} image
	uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
	with:
	file: docker/${{ matrix.build.image }}.Dockerfile
	# TODO: Add `linux/riscv64` once https://github.com/paritytech/polkadot-sdk/issues/5996 is resolved and ring
	# 0.16.x is no longer in dependencies
	# TODO: Add linux/amd64/v4 when runner supports it
	platforms: linux/amd64,linux/amd64/v2,linux/amd64/v3,linux/arm64
	pull: true
	push: true
	tags: ${{ steps.meta.outputs.tags }}
	labels: ${{ steps.meta.outputs.labels }}
	build-args: \|
	SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }}

	- name: Trigger trivy-security-scan Workflow
	uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	event-type: trivy-scan-dispatch
	client-payload: '{"image": "${{ fromJSON(steps.meta.outputs.json).tags[0] }}"}'

	- name: Prepare executables for uploading
	run: \|
	mkdir executables
	cd executables
	IMAGE="${{ fromJSON(steps.meta.outputs.json).tags[0] }}"
	ARTIFACT="${{ matrix.build.base-artifact }}"
	docker run --rm --platform linux/amd64 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-x86_64-skylake-${{ github.ref_name }}
	# TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028
	docker pull --platform linux/amd64/v2 $IMAGE
	docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-x86_64-v2-${{ github.ref_name }}
	if [ "${{ matrix.build.image }}" == "farmer" ]; then
	docker run --rm --platform linux/amd64 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-skylake-${{ github.ref_name }}
	# TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028
	docker pull --platform linux/amd64/v2 $IMAGE
	docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-v2-${{ github.ref_name }}
	fi
	docker run --rm --platform linux/arm64 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-aarch64-${{ github.ref_name }}
	if: matrix.build.upload-executables

	- name: Upload node, farmer, and gateway executables to artifacts
	uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3
	with:
	name: executables-ubuntu-${{ matrix.build.image }}-${{ github.ref_name }}
	path: \|
	executables/*
	if-no-files-found: error
	if: matrix.build.upload-executables

	- name: Upload node, farmer, and gateway executables to assets
	uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1
	env:
	GITHUB_TOKEN: ${{ github.token }}
	with:
	asset_paths: '["executables/*"]'
	if: matrix.build.upload-executables && github.event_name == 'push' && github.ref_type == 'tag'

	executables:
	strategy:
	matrix:
	build:
	- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' \|\| '"macos-14"') }}
	target: aarch64-apple-darwin
	suffix: macos-aarch64-${{ github.ref_name }}
	- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' \|\| '"windows-2022"') }}
	target: x86_64-pc-windows-msvc
	suffix: windows-x86_64-skylake-${{ github.ref_name }}
	rustflags: "-C target-cpu=skylake"
	- os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' \|\| '"windows-2022"') }}
	target: x86_64-pc-windows-msvc
	suffix: windows-x86_64-v2-${{ github.ref_name }}
	rustflags: "-C target-cpu=x86-64-v2"
	fail-fast: false
	runs-on: ${{ matrix.build.os }}
	env:
	PRODUCTION_TARGET: target/${{ matrix.build.target }}/production
	RUSTFLAGS: ${{ matrix.build.rustflags }}

	steps:
	- name: Checkout
	uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

	# On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support
	- name: Install LLVM and Clang for macOS
	uses: KyleMayes/install-llvm-action@dec985c8d7b46a2f363ea1a78f660c946a3349ea # v2.0.1
	with:
	env: true
	version: 17
	if: runner.os == 'macOS'

	# Because macOS, see https://andreasfertig.blog/2021/02/clang-and-gcc-on-macos-catalina-finding-the-include-paths/
	- name: Configure C compiler macOS
	run: \|
	echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV
	if: runner.os == 'macOS'

	- name: Install glibtoolize (macOS)
	run: brew install libtool
	if: runner.os == 'macOS'

	- name: Install Protoc
	uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	# Needed for hwloc
	- name: Install automake (macOS)
	run: brew install automake
	if: runner.os == 'macOS'

	- name: CUDA toolchain
	uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16
	with:
	cuda: '12.4.1'
	method: network
	sub-packages: '["nvcc", "cudart"]'
	if: runner.os == 'Windows'

	- name: Configure ROCm cache (Windows)
	uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
	id: rocm-cache
	with:
	path: C:\Program Files\AMD\ROCm
	key: ${{ runner.os }}-rocm
	if: runner.os == 'Windows'

	- name: ROCm toolchain
	run: \|
	$ErrorActionPreference = "Stop"
	Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
	Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait
	Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe"
	if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true'

	- name: Configure cache
	uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}
	restore-keys: \|
	${{ runner.os }}-cargo-

	- name: Build farmer
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer
	if: runner.os == 'macOS' \|\| !startsWith(matrix.build.target, 'x86_64')

	# ROCm can't be enabled together with CUDA for now
	- name: Build farmer (ROCm, Windows)
	env:
	# Why `PROGRA~1` instead of `Program Files`? Because Windows!
	HIPCC: C:\PROGRA~1\AMD\ROCm\6.1\bin\hipcc.bin.exe
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm
	move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe
	if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64')

	- name: Build farmer
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda
	# TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now
	if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64')

	- name: Build gateway
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-gateway

	- name: Build node
	run: \|
	cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node

	- name: Sign Application (macOS)
	run: \|
	echo "Importing certificate"
	echo "${{ secrets.MACOS_CERTIFICATE }}" \| base64 --decode > certificate.p12
	security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
	security default-keychain -s build.keychain
	security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain
	security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign
	security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain

	echo "Signing farmer"
	codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer

	echo "Signing gateway"
	codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-gateway

	echo "Signing node"
	codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node

	echo "Creating a ZIP archive"
	mkdir -p ${{ env.PRODUCTION_TARGET }}/macos-binaries
	cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries
	ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip

	echo "Notarizing ZIP archive file"
	xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait

	# stapling does not work for .zip archives only .app bundles and .dmg files. Commenting this for now!
	# echo "Stapling notarization to ZIP file"
	# xcrun stapler staple subspace-binaries.zip

	echo "Done!"
	# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
	continue-on-error: ${{ github.repository_owner != 'autonomys' \|\| github.event_name != 'push' \|\| github.ref_type != 'tag' }}
	if: runner.os == 'macOS'


	- name: Sign Application (Windows)
	run: \|
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe"
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe"
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-gateway.exe"
	AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe"
	# Allow code signing to fail on non-release builds and in non-subspace repos (forks)
	continue-on-error: ${{ github.repository_owner != 'autonomys' \|\| github.event_name != 'push' \|\| github.ref_type != 'tag' }}
	if: runner.os == 'Windows'

	- name: Prepare executables for uploading (macOS)
	run: \|
	mkdir executables
	mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }}
	mv ${{ env.PRODUCTION_TARGET }}/subspace-gateway executables/subspace-gateway-${{ matrix.build.suffix }}
	mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }}
	# Zip it so that signature is not lost
	ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip
	ditto -c -k --rsrc executables/subspace-gateway-${{ matrix.build.suffix }} executables/subspace-gateway-${{ matrix.build.suffix }}.zip
	ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip
	rm executables/subspace-farmer-${{ matrix.build.suffix }}
	rm executables/subspace-gateway-${{ matrix.build.suffix }}
	rm executables/subspace-node-${{ matrix.build.suffix }}
	if: runner.os == 'macOS'

	- name: Prepare executables for uploading (Windows)
	run: \|
	mkdir executables
	move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe
	move ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe executables/subspace-farmer-rocm-${{ matrix.build.suffix }}.exe
	move ${{ env.PRODUCTION_TARGET }}/subspace-gateway.exe executables/subspace-gateway-${{ matrix.build.suffix }}.exe
	move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe
	if: runner.os == 'Windows'

	- name: Upload node, farmer, and gateway executables to artifacts
	uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3
	with:
	name: executables-${{ matrix.build.suffix }}
	path: \|
	executables/*
	if-no-files-found: error

	- name: Upload node, farmer, and gateway executables to assets
	uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1
	env:
	GITHUB_TOKEN: ${{ github.token }}
	with:
	asset_paths: '["executables/*"]'
	if: github.event_name == 'push' && github.ref_type == 'tag'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Snapshot build #452

Workflow file

Snapshot build #452

Jobs

Run details

Workflow file for this run