prebuilts: reorganize how we consume prebuilts

Prebuilts are now handled differently and require a proper release process, in order to allow forks to also release we need to consume GITHUB_REPOSITORY while rendering the repo files. To build our binary and provide a prebuilt for those consuming over bzlmod, then we need a better repository structure. Change the release process, which is now heaviliy inspired in https://github.com/aspect-build/bazel-lib which also provides a few prebuilt golang binaries
rmohr · Jul 16, 2024 · 4626975 · 4626975
1 parent 7077b61
commit 4626975
Show file tree

Hide file tree

Showing 22 changed files with 531 additions and 321 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,11 @@
+docs/*.md linguist-generated=true
+
+# Configuration for 'git archive'
+# see https://git-scm.com/docs/git-archive/2.40.0#ATTRIBUTES
+# Exclude a bunch of paths to save some disk space
+e2e export-ignore
+.aspect export-ignore
+.github export-ignore
+pkg/*/testdata export-ignore
+tools/release export-ignore
+
diff --git a/.github/workflows/build_assets.sh b/.github/workflows/build_assets.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+set -x
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+${SCRIPT_DIR}/generate_tools_versions.sh
+
+bazel run //tools/release
diff --git a/.github/workflows/generate_tools_versions.sh b/.github/workflows/generate_tools_versions.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+set -x
+
+# Set by GH actions, see
+# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+REPO_URL="${GITHUB_REPOSITORY:-rmohr/bazeldnf}"
+OUT=${PREFIX:-.}/tools/version.bzl
+cat > ${OUT} <<EOF
+"Generated during release generate_tools_prebuilts.sh"
+
+VERSION = "${GITHUB_REF_NAME}"
+
+REPO_URL = "${REPO_URL}"
+EOF
diff --git a/.github/workflows/integrity.jq b/.github/workflows/integrity.jq
@@ -0,0 +1,19 @@
+# JQ filter to transform sha256 files to a value we can read from starlark.
+# NB: the sha256 files are expected to be newline-terminated.
+#
+# Input looks like
+# 48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636 bazeldnf-v0.6.0-rc7-linux-amd64\n...
+#
+# Output should look like
+# {
+#  "linux-amd64": "48552e399a1f2ab97e62ca7fce5783b6214e284330c7555383f43acf82446636",
+#  ...
+# }
+
+.
+| sub($ARGS.named.PREFIX; ""; "g")
+| rtrimstr("\n")
+| split("\n")
+| map(split(" "))
+| map({"key": .[1], "value": .[0]})
+| from_entries
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,41 +1,62 @@
-
-name: release
-
-# This workflow can be manually triggered to create a GitHub release and upload release bins.
-#
-# Steps to use:
-# - Create a new Git tag and push
-# - Navigate to https://github.com/rmohr/bazeldnf/actions/workflows/release.yml
-# - Click on "Run workflow"
-# - Select the tag you want to release
-# - Click on green button
-#
-# Requires read-write workflow permissions which can be configured in Actions repo settings.
+# Cut a release whenever a new tag is pushed to the repo.
+# You should use an annotated tag, like `git tag -a v1.2.3`
+# and put the release notes into the commit message for the tag.
+name: Release
 
 on:
-  workflow_dispatch:
-    inputs:
-      release_version:
-        description: 'Bazeldnf release version (e.g. v1.2.3)'
-        required: true
+  push:
+    tags:
+      - "v*.*.*"
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: bazel-contrib/[email protected]
+        with:
+          # Avoid downloading Bazel every time.
+          bazelisk-cache: true
+          # Keep a disk-cache
+          disk-cache: true
+          # Share repository cache between workflows.
+          repository-cache: true
+          # enable some flags for CI
+          bazelrc: |
+            import %workspace%/.aspect/workflows/ci.bazelrc
+          # keep a cache for MODULE.bazel repos
+          external-cache: true
+
+      - name: Build Binaries
+        run: .github/workflows/build_assets.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifacts
+          path: tools/release/latest
+          retention-days: 1
+
   release:
+    needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: bazelbuild/setup-bazelisk@v1
-      - run: |
-          git config --global user.name "CI"
-          git config --global user.email "[email protected]"
-          hack/prepare-release.sh
-          git push origin ${VERSION}
-          git push
-        env:
-          VERSION: ${{ inputs.release_version }}
-      - uses: softprops/action-gh-release@v1
+      - uses: actions/checkout@v4
+
+      # Fetch the built artifacts from build jobs above and extract into
+      # ${GITHUB_WORKSPACE}/artifacts/bazeldnf_dawrin-amd64
+      - uses: actions/download-artifact@v4
+
+      - name: Prepare release
+        run: .github/workflows/release_prep.sh > release_notes.txt
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
         with:
-          body_path: dist/releasenote.txt
-          tag_name: ${{ inputs.release_version }}
+          prerelease: true
+          # Use GH feature to populate the changelog automatically
+          generate_release_notes: true
+          body_path: release_notes.txt
           files: |
-            dist/bazeldnf-*
+            artifacts/*
+            bazeldnf-*.tar.gz
+          fail_on_unmatched_files: true
diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+set -x
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Set by GH actions, see
+# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+
+# The prefix is chosen to match what GitHub generates for source archives
+PREFIX="bazeldnf-${GITHUB_REF_NAME}"
+ARCHIVE="bazeldnf-${GITHUB_REF_NAME}.tar.gz"
+ARCHIVE_TMP=$(mktemp)
+
+# NB: configuration for 'git archive' is in /.gitattributes
+git archive --format=tar --prefix=${PREFIX}/ --worktree-attributes  ${GITHUB_REF_NAME} > $ARCHIVE_TMP
+
+############
+# Patch up the archive to have integrity hashes for built binaries that we downloaded in the GHA workflow.
+# Now that we've run `git archive` we are free to pollute the working directory.
+
+# Delete the placeholder files
+tar --file $ARCHIVE_TMP --delete ${PREFIX}/tools/version.bzl
+tar --file $ARCHIVE_TMP --delete ${PREFIX}/tools/integrity.bzl
+
+mkdir -p ${PREFIX}/tools
+
+PREFIX=$PREFIX ${SCRIPT_DIR}/generate_tools_versions.sh
+
+PREBUILTS=$(jq \
+  --from-file .github/workflows/integrity.jq \
+  --arg PREFIX "bazeldnf-${GITHUB_REF_NAME}-" \
+  --slurp \
+  --raw-input artifacts/*.sha256 \
+)
+
+cat >${PREFIX}/tools/integrity.bzl <<EOF
+"Generated during release by release_prep.sh, using integrity.jq"
+
+PREBUILTS = ${PREBUILTS}
+
+EOF
+
+# Append that generated files back into the archive
+tar --file $ARCHIVE_TMP --append ${PREFIX}/tools/version.bzl
+tar --file $ARCHIVE_TMP --append ${PREFIX}/tools/integrity.bzl
+
+# END patch up the archive
+############
+
+gzip < $ARCHIVE_TMP > $ARCHIVE
+SHA=$(shasum -a 256 $ARCHIVE | awk '{print $1}')
+
+# Set by GH actions, see
+# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+REPO_URL="${GITHUB_REPOSITORY:-rmohr/bazeldnf}"
+
+cat << EOF
+## Using [Bzlmod] with Bazel 6:
+
+Add to your \`MODULE.bazel\` file:
+
+\`\`\`starlark
+bazel_dep(name = "bazeldnf", version = "${GITHUB_REF_NAME:1}")
+\`\`\`
+
+This will register a prebuilt bazeldnf
+
+[Bzlmod]: https://bazel.build/build/bzlmod
+
+## Using WORKSPACE
+
+\`\`\`starlark
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+    name = "bazeldnf",
+    sha256 = "${SHA}",
+    strip_prefix = "${PREFIX}",
+    url = "https://github.com/${REPO_URL}/releases/download/${GITHUB_REF_NAME}/${ARCHIVE}",
+)
+
+load(
+    "@bazeldnf//bazeldnf:repositories.bzl",
+    "bazeldnf_dependencies",
+)
+
+bazeldnf_dependencies()
+\`\`\`
+EOF
diff --git a/.gitignore b/.gitignore
@@ -7,5 +7,12 @@ dist
 cmd/cmd
 .bazelrc.user
 
+# release artifacts
+artifacts
+bazeldnf-v*
+
 # temporary files from MODULE.bazel
 MODULE.bazel.lock
+
+# this is where we store the release artifacts, so we can ignore it
+tools/release/latest
diff --git a/bazeldnf/defs.bzl b/bazeldnf/defs.bzl
@@ -3,20 +3,25 @@ Public API to use bazeldnf from other repositories
 """
 
 load(
-    "@bazeldnf//internal:bazeldnf.bzl",
+    "//internal:bazeldnf.bzl",
     _bazeldnf = "bazeldnf",
 )
 load(
-    "@bazeldnf//internal:rpmtree.bzl",
+    "//internal:rpm.bzl",
+    _rpm = "rpm",
+)
+load(
+    "//internal:rpmtree.bzl",
     _rpmtree = "rpmtree",
     _tar2files = "tar2files",
 )
 load(
-    "@bazeldnf//internal:xattrs.bzl",
+    "//internal:xattrs.bzl",
     _xattrs = "xattrs",
 )
 
 bazeldnf = _bazeldnf
+rpm = _rpm
 rpmtree = _rpmtree
 tar2files = _tar2files
 xattrs = _xattrs
diff --git a/bazeldnf/deps.bzl b/bazeldnf/deps.bzl
@@ -1,77 +1,12 @@
 """bazeldnf public dependency for WORKSPACE"""
 
 load(
-    "@bazel_tools//tools/build_defs/repo:http.bzl",
-    "http_archive",
-    "http_file",
+    "//bazeldnf:repositories.bzl",
+    _bazeldnf_dependencies = "bazeldnf_dependencies",
+    _bazeldnf_register_toolchains = "bazeldnf_register_toolchains",
 )
-load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
-load(
-    "@bazeldnf//internal:rpm.bzl",
-    _rpm = "rpm",
-)
-
-rpm = _rpm
 
-def bazeldnf_dependencies():
-    """bazeldnf dependencies when consuming the repo externally"""
-    maybe(
-        http_archive,
-        name = "bazel_skylib",
-        sha256 = "f24ab666394232f834f74d19e2ff142b0af17466ea0c69a3f4c276ee75f6efce",
-        urls = [
-            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.0/bazel-skylib-1.4.0.tar.gz",
-            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.0/bazel-skylib-1.4.0.tar.gz",
-        ],
-    )
-    maybe(
-        http_archive,
-        name = "platforms",
-        urls = [
-            "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
-            "https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
-        ],
-        sha256 = "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee",
-    )
-    http_file(
-        name = "bazeldnf-linux-amd64",
-        executable = True,
-        sha256 = "7e1035d8bd2f25b787b04843f4d6a05e7cdbd3995926497c2a587e52da6262a8",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-linux-amd64"],
-    )
-    http_file(
-        name = "bazeldnf-linux-arm64",
-        executable = True,
-        sha256 = "d954b785bfd79dbbd66a2f3df02b0d3a51f1fed4508a6d88fda13a9d24c878cc",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-linux-arm64"],
-    )
-    http_file(
-        name = "bazeldnf-darwin-amd64",
-        executable = True,
-        sha256 = "92afc7f6475981adf9ae32b563b051869d433d8d8c9666e28a1c1c6e840394cd",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-darwin-amd64"],
-    )
-    http_file(
-        name = "bazeldnf-darwin-arm64",
-        executable = True,
-        sha256 = "c5e99ed72448026ee63259a0a28440f8b43a125414fa312edbd569c2976515a3",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-darwin-arm64"],
-    )
-    http_file(
-        name = "bazeldnf-linux-ppc64",
-        executable = True,
-        sha256 = "9d5337c1afe4bab858742718ac4c230d9ca368299cb97c50078eef14ae180922",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-linux-ppc64"],
-    )
-    http_file(
-        name = "bazeldnf-linux-ppc64le",
-        executable = True,
-        sha256 = "7ea4db00947914bc1c51e8f042fe220a3167c65815c487eccd0c541ecfa78aa1",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-linux-ppc64le"],
-    )
-    http_file(
-        name = "bazeldnf-linux-s390x",
-        executable = True,
-        sha256 = "09aa4abcb1d85da11642889826b982ef90547eb32099fc8177456c92f66a4cfd",
-        urls = ["https://github.com/rmohr/bazeldnf/releases/download/v0.5.9/bazeldnf-v0.5.9-linux-s390x"],
-    )
+def bazeldnf_dependencies(name = "bazeldnf", **kwargs):
+    """bazeldnf dependencies when consuming bazeldnf through WORKSPACE"""
+    _bazeldnf_dependencies()
+    _bazeldnf_register_toolchains(name = name, **kwargs)