Prepare release pipeline for ghcr.io (#1220)

To reduce CI spending, we're in the process of moving all new release to ghcr.io - tektoncd/plumbing#2157 This prepares the release pipeline for that switch (which will require a PR in the plumbing repo). This PR also aligns the pipeline with some other small improvements already adopted by other projects: - use remote resolution for tasks - only run build and tests when requested - it will be disabled by default on nightly builds Signed-off-by: Andrea Frittoli <[email protected]>
tektoncd · Oct 2, 2024 · 6bcfe64 · 6bcfe64
1 parent cd9106b
commit 6bcfe64
Show file tree

Hide file tree

Showing 2 changed files with 157 additions and 37 deletions.
diff --git a/release/publish.yaml b/release/publish.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Modifed for chains from tekton triggers: https://github.com/tektoncd/triggers/blob/main/tekton/publish.yaml
+# Modified for chains from tekton triggers: https://github.com/tektoncd/triggers/blob/main/tekton/publish.yaml
 apiVersion: tekton.dev/v1beta1
 kind: Task
 metadata:
@@ -27,6 +27,9 @@ spec:
   - name: images
     description: List of cmd/* paths to be published as images in release manifest release.yaml
     default: "controller"
+  - name: koExtraArgs
+    description: Extra args to be passed to ko
+    default: "--preserve-import-paths"
   - name: versionTag
     description: The vX.Y.Z version that the artifacts should be tagged with (including `v`)
   - name: imageRegistry
@@ -37,6 +40,9 @@ spec:
   - name: imageRegistryRegions
     description: The target image registry regions
     default: "us eu asia"
+  - name: imageRegistryUser
+    description: Username to be used to login to the container registry
+    default: "_json_key"
   - name: releaseAsLatest
     description: Whether to tag and publish this release as chainss' latest
     default: "true"
@@ -65,37 +71,43 @@ spec:
     env:
     - name: "PROJECT_ROOT"
       value: "$(workspaces.source.path)"
-    - name: CONTAINER_REGISTY_CREDENTIALS
+    - name: CONTAINER_REGISTRY_CREDENTIALS
       value: "$(workspaces.release-secret.path)/$(params.serviceAccountPath)"
     - name: CONTAINER_REGISTRY
       value: "$(params.imageRegistry)/$(params.imageRegistryPath)"
+    - name: IMAGE_REGISTRY_PATH
+      value: "$(params.imageRegistryPath)"
+    - name: CONTAINER_REGISTRY_USER
+      value: "$(params.imageRegistryUser)"
     - name: REGIONS
       value: "$(params.imageRegistryRegions)"
     - name: OUTPUT_RELEASE_DIR
       value: "$(workspaces.output.path)/$(params.versionTag)"
+    - name: KO_EXTRA_ARGS
+      value: "$(params.koExtraArgs)"
   steps:
 
-  - name: container-registy-auth
-    image: gcr.io/go-containerregistry/crane:debug@sha256:ff0e08eeae8097d28b2381c7f7123bf542757abc68d11bff58fb882b72843785
+  - name: container-registry-auth
+    image: cgr.dev/chainguard/crane:latest-dev@sha256:cdb799cd2e649a2c9a52e1115453b9e2fce6546505f3836628816bf5c6ce8f6e
     script: |
-      #!/busybox/sh
+      #!/bin/sh
       set -ex
 
       # Login to the container registry
-      DOCKER_CONFIG=$(cat ${CONTAINER_REGISTY_CREDENTIALS} | \
-        crane auth login -u _json_key --password-stdin $(params.imageRegistry) 2>&1 | \
+      DOCKER_CONFIG=$(cat ${CONTAINER_REGISTRY_CREDENTIALS} | \
+        crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin $(params.imageRegistry) 2>&1 | \
         sed 's,^.*logged in via \(.*\)$,\1,g')
 
       # Auth with account credentials for all regions.
       for region in ${REGIONS}
       do
         HOSTNAME=${region}.$(params.imageRegistry)
-        cat ${CONTAINER_REGISTY_CREDENTIALS} | crane auth login -u _json_key --password-stdin ${HOSTNAME}
+        cat ${CONTAINER_REGISTRY_CREDENTIALS} | crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin ${HOSTNAME}
       done
       cp ${DOCKER_CONFIG} /workspace/docker-config.json
 
   - name: run-ko
-    image: gcr.io/tekton-releases/dogfooding/ko:v20240926-3daa55a03e@sha256:393155dbdd7c8d920925b202c88e4846f46a70c1e1dc218b0ea5e2d7e388b576
+    image: gcr.io/tekton-releases/dogfooding/ko@sha256:9f261a80343b3637c88c77d2e8b081079faa28911c5cad945ae52fd9cf25f396
     env:
     - name: KO_DOCKER_REPO
       value: $(params.imageRegistry)/$(params.imageRegistryPath)
@@ -118,7 +130,7 @@ spec:
       # For each cmd/* directory, include a full gzipped tar of all source in
       # vendor/. This is overkill. Some deps' licenses require the source to be
       # included in the container image when they're used as a dependency.
-      # Rather than trying to determine which deps have this requirement (an(params.imageRegistryd
+      # Rather than trying to determine which deps have this requirement (and
       # probably get it wrong), we'll just targz up the whole vendor tree and
       # include it. As of 9/20/2019, this amounts to about 11MB of additional
       # data in each image.
@@ -130,50 +142,74 @@ spec:
         fi
       done
 
-      # Rewrite "devel" to params.versionTag
-      sed -i -e 's/\(chains.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${PROJECT_ROOT}/config/*.yaml
-
       # Publish images and create release.yaml
       mkdir -p $OUTPUT_RELEASE_DIR
 
-      ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml
+      # Make a local git tag to make git status happy :)
+      # The real "tagging" will happen with the "create-release" pipeline.
+      git tag $(params.versionTag)
 
+      ko resolve \
+        --image-label=org.opencontainers.image.source=https://$(params.package) \
+        --platform=$(params.platforms) \
+        -t $(params.versionTag) \
+        ${KO_EXTRA_ARGS} -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml
       # Publish images and create release.notags.yaml
       # This is useful if your container runtime doesn't support the `image-reference:tag@digest` notation
       # This is currently the case for `cri-o` (and most likely others)
-      ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml
-
+      ko resolve \
+        --image-label=org.opencontainers.image.source=https://$(params.package) \
+        --platform=$(params.platforms) \
+        ${KO_EXTRA_ARGS} -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml
 
+      # Rewrite "devel" to params.versionTag
+      sed -i -e 's/\(pipeline.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.yaml
+      sed -i -e 's/\(pipeline.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.notags.yaml
   - name: koparse
-    image: gcr.io/tekton-releases/dogfooding/koparse:v20240910-ec3cf3c749@sha256:5e8a522fc1e587fc00b69a6d73e0bfdf7a29ca143537a5542eb224680d2dbf2f
+    image: gcr.io/tekton-releases/dogfooding/koparse@sha256:6a3c7deeff8701a7c919ecc539e3d9fce9412dcbb9175738dfe07e3ba7d8caeb
     script: |
       set -ex
 
-      IMAGES_PATH=${CONTAINER_REGISTRY}/$(params.package)
+      # Find "--preserve-import-paths" in a list of args
+      function find_preserve_import_path() {
+        for arg in $@; do
+          if [[ "$arg" == "--preserve-import-paths" ]]; then
+            return 0
+          fi
+        done
+        return 1
+      }
+
+      # If "--preserve-import-paths" is used, include "package" in the expected path
+      find_preserve_import_path \
+        $(echo $KO_EXTRA_ARGS) && \
+        PRESERVE_IMPORT_PATH="--preserve-path" || \
+        PRESERVE_IMPORT_PATH="--no-preserve-path"
 
       for cmd in $(params.images)
       do
-        IMAGES="${IMAGES} ${IMAGES_PATH}/cmd/${cmd}:$(params.versionTag)"
+        IMAGES="${IMAGES} $(params.package)/cmd/${cmd}:$(params.versionTag)"
       done
 
       # Parse the built images from the release.yaml generated by ko
       koparse \
         --path $OUTPUT_RELEASE_DIR/release.yaml \
-        --base ${IMAGES_PATH} --images ${IMAGES} > /workspace/built_images
+        --base $(params.package) \
+        --container-registry ${CONTAINER_REGISTRY} \
+        --images ${IMAGES} \
+        ${PRESERVE_IMPORT_PATH} > /workspace/built_images
 
   - name: tag-images
-    image: gcr.io/go-containerregistry/crane:debug@sha256:ff0e08eeae8097d28b2381c7f7123bf542757abc68d11bff58fb882b72843785
+    image: cgr.dev/chainguard/crane:latest-dev@sha256:cdb799cd2e649a2c9a52e1115453b9e2fce6546505f3836628816bf5c6ce8f6e
     script: |
-      #!/busybox/sh
+      #!/bin/sh
       set -ex
 
       # Setup docker-auth
       DOCKER_CONFIG=~/.docker
       mkdir -p ${DOCKER_CONFIG}
       cp /workspace/docker-config.json ${DOCKER_CONFIG}/config.json
 
-      REGIONS="us eu asia"
-
       # Tag the images and put them in all the regions
       for IMAGE in $(cat /workspace/built_images)
       do

diff --git a/release/release-pipeline.yaml b/release/release-pipeline.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Modifed for chains from tekton triggers: https://github.com/tektoncd/triggers/blob/main/tekton/release-pipeline.yaml
+# Modified for chains from tekton triggers: https://github.com/tektoncd/triggers/blob/main/tekton/release-pipeline.yaml
 ---
 apiVersion: tekton.dev/v1beta1
 kind: Pipeline
@@ -33,6 +33,12 @@ spec:
   - name: imageRegistryPath
     description: The path (project) in the image registry
     default: tekton-releases
+  - name: imageRegistryRegions
+    description: The target image registry regions
+    default: "us eu asia"
+  - name: imageRegistryUser
+    description: The user for the image registry credentials
+    default: _json_key
   - name: versionTag
     description: The X.Y.Z version that the artifacts should be tagged with
   - name: releaseBucket
@@ -44,13 +50,23 @@ spec:
   - name: platforms
     description: Platforms to publish for the images (e.g. linux/amd64,linux/arm64)
     default: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
+  - name: koExtraArgs
+    description: Extra args to be passed to ko
+    default: "--preserve-import-paths"
   - name: serviceAccountPath
     description: The path to the service account file within the release-secret workspace
+  - name: serviceAccountImagesPath
+    description: The path to the service account file or credentials within the release-images-secret workspace
+  - name: runTests
+    description: If set to something other than "true", skip the build and test tasks
+    default: "true"
   workspaces:
   - name: workarea
     description: The workspace where the repo will be cloned.
   - name: release-secret
-    description: The secret that contains a service account authorized to push to the imageRegistry and to the output bucket
+    description: The secret that contains a service account authorized to push to the output bucket
+  - name: release-images-secret
+    description: The secret that contains a service account authorized to push to the imageRegistry
   results:
   - name: commit-sha
     description: the sha of the commit that was released
@@ -64,7 +80,12 @@ spec:
   tasks:
   - name: git-clone
     taskRef:
-      name: git-clone
+      resolver: hub
+      params:
+        - name: name
+          value: git-clone
+        - name: version
+          value: "0.7"
     workspaces:
     - name: output
       workspace: workarea
@@ -77,7 +98,16 @@ spec:
   - name: precheck
     runAfter: [git-clone]
     taskRef:
-      name: prerelease-checks
+      resolver: git
+      params:
+        - name: repo
+          value: plumbing
+        - name: org
+          value: tektoncd
+        - name: revision
+          value: aeed19e5a36f335ebfdc4b96fa78d1ce5bb4f7b8
+        - name: pathInRepo
+          value: tekton/resources/release/base/prerelease_checks.yaml
     params:
     - name: package
       value: $(params.package)
@@ -91,8 +121,17 @@ spec:
       subpath: git
   - name: unit-tests
     runAfter: [precheck]
+    when:
+      - cel: "'$(params.runTests)' == 'true'"
     taskRef:
-      name: golang-test
+      resolver: bundles
+      params:
+        - name: bundle
+          value: gcr.io/tekton-releases/catalog/upstream/golang-test:0.2
+        - name: name
+          value: golang-test
+        - name: kind
+          value: task
     params:
     - name: package
       value: $(params.package)
@@ -104,8 +143,17 @@ spec:
       subpath: git
   - name: build
     runAfter: [precheck]
+    when:
+      - cel: "'$(params.runTests)' == 'true'"
     taskRef:
-      name: golang-build
+      resolver: bundles
+      params:
+        - name: bundle
+          value: gcr.io/tekton-releases/catalog/upstream/golang-build:0.3
+        - name: name
+          value: golang-build
+        - name: kind
+          value: task
     params:
     - name: package
       value: $(params.package)
@@ -118,7 +166,16 @@ spec:
   - name: publish-images
     runAfter: [build, unit-tests]
     taskRef:
-      name: publish-chains-release
+      resolver: git
+      params:
+        - name: repo
+          value: pipeline
+        - name: org
+          value: tektoncd
+        - name: revision
+          value: $(params.gitRevision)
+        - name: pathInRepo
+          value: tekton/publish.yaml
     params:
     - name: package
       value: $(params.package)
@@ -128,12 +185,18 @@ spec:
       value: $(params.imageRegistry)
     - name: imageRegistryPath
       value: $(params.imageRegistryPath)
+    - name: imageRegistryUser
+      value: $(params.imageRegistryUser)
+    - name: imageRegistryRegions
+      value: $(params.imageRegistryRegions)
     - name: releaseAsLatest
       value: $(params.releaseAsLatest)
+    - name: serviceAccountPath
+      value: $(params.serviceAccountImagesPath)
     - name: platforms
       value: $(params.platforms)
-    - name: serviceAccountPath
-      value: $(params.serviceAccountPath)
+        - name: koExtraArgs
+          value: $(params.koExtraArgs)
     - name: CHAINS-GIT_COMMIT
       value: $(tasks.git-clone.results.commit)
     - name: CHAINS-GIT_URL
@@ -146,11 +209,18 @@ spec:
       workspace: workarea
       subpath: bucket
     - name: release-secret
-      workspace: release-secret
+      workspace: release-images-secret
   - name: publish-to-bucket
     runAfter: [publish-images]
     taskRef:
-      name: gcs-upload
+      resolver: bundles
+      params:
+        - name: bundle
+          value: gcr.io/tekton-releases/catalog/upstream/gcs-upload:0.3
+        - name: name
+          value: gcs-upload
+        - name: kind
+          value: task
     workspaces:
     - name: credentials
       workspace: release-secret
@@ -171,7 +241,14 @@ spec:
       operator: in
       values: ["true"]
     taskRef:
-      name: gcs-upload
+      resolver: bundles
+      params:
+        - name: bundle
+          value: gcr.io/tekton-releases/catalog/upstream/gcs-upload:0.3
+        - name: name
+          value: gcs-upload
+        - name: kind
+          value: task
     workspaces:
     - name: credentials
       workspace: release-secret
@@ -185,6 +262,8 @@ spec:
       value: $(params.versionTag)
     - name: serviceAccountPath
       value: $(params.serviceAccountPath)
+    - name: deleteExtraFiles
+      value: "true" # Uses rsync to copy content into latest
   - name: report-bucket
     runAfter: [publish-to-bucket]
     params:
@@ -204,8 +283,13 @@ spec:
       steps:
       - name: create-results
         image: docker.io/library/alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d
+        env:
+          - name: RELEASE_BUCKET
+            value: $(params.releaseBucket)
+          - name: VERSION_TAG
+            value: $(params.versionTag)
         script: |
-          BASE_URL=$(echo "$(params.releaseBucket)/previous/$(params.versionTag)")
+          BASE_URL=$(echo "${RELEASE_BUCKET}/previous/${VERSION_TAG}")
           # If the bucket is in the gs:// return the corresponding public https URL
           BASE_URL=$(echo ${BASE_URL} | sed 's,gs://,https://storage.googleapis.com/,g')
           echo "${BASE_URL}/release.yaml" > $(results.release.path)