Scorecard custom image update and relocation

Makefile

@@ -29,18 +29,21 @@ GO_BUILD_ARGS = \
 
 ANSIBLE_BASE_IMAGE = quay.io/operator-framework/ansible-operator
 HELM_BASE_IMAGE = quay.io/operator-framework/helm-operator
+CUSTOM_SCORECARD_TESTS_BASE_IMAGE = quay.io/operator-framework/custom-scorecard-tests
 SCORECARD_TEST_BASE_IMAGE = quay.io/operator-framework/scorecard-test
 SCORECARD_TEST_KUTTL_BASE_IMAGE = quay.io/operator-framework/scorecard-test-kuttl
 
 ANSIBLE_IMAGE ?= $(ANSIBLE_BASE_IMAGE)
 HELM_IMAGE ?= $(HELM_BASE_IMAGE)
+CUSTOM_SCORECARD_TESTS_IMAGE ?= $(CUSTOM_SCORECARD_TESTS_BASE_IMAGE)
 SCORECARD_TEST_IMAGE ?= $(SCORECARD_TEST_BASE_IMAGE)
 SCORECARD_TEST_KUTTL_IMAGE ?= $(SCORECARD_TEST_KUTTL_BASE_IMAGE)
 
-ANSIBLE_ARCHES:="amd64" "ppc64le" "arm64"
-HELM_ARCHES:="amd64" "ppc64le" "arm64"
-SCORECARD_TEST_ARCHES:="amd64" "ppc64le" "arm64"
-SCORECARD_TEST_KUTTL_ARCHES:="amd64" "ppc64le" "arm64"
+ANSIBLE_ARCHES:="amd64" "ppc64le" "s390x" "arm64"
+HELM_ARCHES:="amd64" "ppc64le" "s390x" "arm64"
+CUSTOM_SCORECARD_TESTS_ARCHES:="amd64" "ppc64le" "s390x" "arm64"
+SCORECARD_TEST_ARCHES:="amd64" "ppc64le" "s390x" "arm64"
+SCORECARD_TEST_KUTTL_ARCHES:="amd64" "ppc64le" "s390x" "arm64"
 
 export CGO_ENABLED:=0
 .DEFAULT_GOAL:=help
@@ -180,7 +183,7 @@ build/%.asc: ## Create release signatures for operator-sdk release binaries
 
 image: image-build image-push ## Build and push all images
 
-image-build: image-build-ansible image-build-helm image-build-scorecard-test image-build-scorecard-test-kuttl## Build all images
+image-build: image-build-ansible image-build-helm image-build-scorecard-test image-build-scorecard-test-kuttl image-build-custom-scorecard-tests ## Build all images
 
 image-push: image-push-ansible image-push-helm image-push-scorecard-test ## Push all images
 
@@ -217,9 +220,15 @@ image-push-helm-multiarch:
 # Scorecard test image scaffold/build/push.
 .PHONY: image-build-scorecard-test image-push-scorecard-test image-push-scorecard-test-multiarch
 
+# Scorecard custom test image scaffold/build/push.
+.PHONY: image-build-custom-scorecard-tests image-push-custom-scorecard-tests image-push-custom-scorecard-tests-multiarch
+
 # Scorecard test kuttl image scaffold/build/push.
 .PHONY: image-build-scorecard-test-kuttl image-push-scorecard-test-kuttl image-push-scorecard-test-kuttl-multiarch
 
+image-build-custom-scorecard-tests:
+	./hack/image/build-custom-scorecard-tests-image.sh $(CUSTOM_SCORECARD_TESTS_BASE_IMAGE):dev
+
 image-build-scorecard-test:
 	./hack/image/build-scorecard-test-image.sh $(SCORECARD_TEST_BASE_IMAGE):dev
 

changelog/fragments/scorecard-custom-image-relocate.yaml

@@ -0,0 +1,14 @@
+entries:
+  - description: Updated custom scorecard test example
+    kind: change
+    breaking: false
+    migration:
+      header: Update custom scorecard test example
+      body: |
+        The custom-scorecard-tests image is built like the other
+        scorecard images with this change, the Makefile is updated
+        to produce the custom-scorecard-tests image.  This makes it
+        easier to build the image using the normal build process and
+        will make testing the custom image easier, which will be
+        addressed in another PR.
+

changelog/fragments/scorecard-kuttl-version-bump.yaml

@@ -0,0 +1,10 @@
+entries:
+  - description: Updated scorecard-test-kuttl image to use latest kuttl
+    kind: change
+    breaking: false
+    migration:
+      header: Update scorecard-test-kuttl image to use latest kuttl
+      body: |
+        The scorecard-test-kuttl image is updated to use kuttl:v0.5.2
+        as the base image.
+

hack/image/build-custom-scorecard-tests-image.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -eux
+
+source hack/lib/image_lib.sh
+
+# build scorecard test image
+WD="$(dirname "$(pwd)")"
+GOOS=linux CGO_ENABLED=0 \
+  go build \
+  -gcflags "all=-trimpath=${WD}" \
+  -asmflags "all=-trimpath=${WD}" \
+  -o images/custom-scorecard-tests/custom-scorecard-tests \
+  images/custom-scorecard-tests/cmd/test/main.go
+
+# Build base image
+pushd images/custom-scorecard-tests
+docker build -t "$1" .
+# If using a kind cluster, load the image into all nodes.
+load_image_if_kind "$1"
+popd

internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml → images/custom-scorecard-tests/bundle/tests/scorecard/config.yaml

@@ -4,14 +4,14 @@ metadata:
   name: config
 stages:
 - tests:
-  - image: quay.io/username/custom-scorecard-tests:dev
+  - image: quay.io/operator-framework/custom-scorecard-tests:dev
     entrypoint:
     - custom-scorecard-tests
     - customtest1
     labels:
       suite: custom
       test: customtest1
-  - image: quay.io/username/custom-scorecard-tests:dev
+  - image: quay.io/operator-framework/custom-scorecard-tests:dev
     entrypoint:
     - custom-scorecard-tests
     - customtest2

internal/scorecard/examples/custom-scorecard-tests/images/custom-scorecard-tests/cmd/test/main.go → images/custom-scorecard-tests/cmd/test/main.go

@@ -22,9 +22,6 @@ import (
 
 	scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
 	apimanifests "github.com/operator-framework/api/pkg/manifests"
-
-	"github.com/operator-framework/operator-sdk/internal/scorecard"
-	"github.com/operator-framework/operator-sdk/internal/scorecard/examples/custom-scorecard-tests/internal/tests"
 )
 
 // This is the custom scorecard test example binary
@@ -36,14 +33,16 @@ import (
 // this binary to run various tests all from within a single
 // test image.
 
+const PodBundleRoot = "/bundle"
+
 func main() {
 	entrypoint := os.Args[1:]
 	if len(entrypoint) == 0 {
 		log.Fatal("Test name argument is required")
 	}
 
 	// Read the pod's untar'd bundle from a well-known path.
-	cfg, err := apimanifests.GetBundleFromDir(scorecard.PodBundleRoot)
+	cfg, err := apimanifests.GetBundleFromDir(PodBundleRoot)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
@@ -53,10 +52,10 @@ func main() {
 	// Names of the custom tests which would be passed in the
 	// `operator-sdk` command.
 	switch entrypoint[0] {
-	case tests.CustomTest1Name:
-		result = tests.CustomTest1(cfg)
-	case tests.CustomTest2Name:
-		result = tests.CustomTest2(cfg)
+	case CustomTest1Name:
+		result = CustomTest1(cfg)
+	case CustomTest2Name:
+		result = CustomTest2(cfg)
 	default:
 		result = printValidTests()
 	}
@@ -78,10 +77,52 @@ func printValidTests() scapiv1alpha3.TestStatus {
 	result.Suggestions = make([]string, 0)
 
 	str := fmt.Sprintf("Valid tests for this image include: %s %s",
-		tests.CustomTest1Name,
-		tests.CustomTest2Name)
+		CustomTest1Name,
+		CustomTest2Name)
 	result.Errors = append(result.Errors, str)
 	return scapiv1alpha3.TestStatus{
 		Results: []scapiv1alpha3.TestResult{result},
 	}
 }
+
+const (
+	CustomTest1Name = "customtest1"
+	CustomTest2Name = "customtest2"
+)
+
+// Define any operator specific custom tests here.
+// CustomTest1 and CustomTest2 are example test functions. Relevant operator specific
+// test logic is to be implemented in similarly.
+
+func CustomTest1(bundle *apimanifests.Bundle) scapiv1alpha3.TestStatus {
+	r := scapiv1alpha3.TestResult{}
+	r.Name = CustomTest1Name
+	r.State = scapiv1alpha3.PassState
+	r.Errors = make([]string, 0)
+	r.Suggestions = make([]string, 0)
+	almExamples := bundle.CSV.GetAnnotations()["alm-examples"]
+	if almExamples == "" {
+		fmt.Println("no alm-examples in the bundle CSV")
+	}
+
+	return wrapResult(r)
+}
+
+func CustomTest2(bundle *apimanifests.Bundle) scapiv1alpha3.TestStatus {
+	r := scapiv1alpha3.TestResult{}
+	r.Name = CustomTest2Name
+	r.State = scapiv1alpha3.PassState
+	r.Errors = make([]string, 0)
+	r.Suggestions = make([]string, 0)
+	almExamples := bundle.CSV.GetAnnotations()["alm-examples"]
+	if almExamples == "" {
+		fmt.Println("no alm-examples in the bundle CSV")
+	}
+	return wrapResult(r)
+}
+
+func wrapResult(r scapiv1alpha3.TestResult) scapiv1alpha3.TestStatus {
+	return scapiv1alpha3.TestStatus{
+		Results: []scapiv1alpha3.TestResult{r},
+	}
+}

images/scorecard-test-kuttl/Dockerfile

@@ -1,5 +1,5 @@
 # Base image
-FROM docker.io/kudobuilder/kuttl@sha256:d23368441f313107954e80a3a5f2884f374bd5a4746193e304c7733f98d6915e
+FROM docker.io/kudobuilder/kuttl@sha256:c9c5edc27ba6e5e994ffd8cea03368c0d4e274f3c7a00f51c1e360feb573f24e
 #FROM kudobuilder/kuttl:latest
 
 ENV TESTKUTTL=/usr/local/bin/scorecard-test-kuttl \