Rework custom-bundle.Dockerfile to use staged data

.github/workflows/build-openstack-operator.yaml

@@ -17,7 +17,7 @@ jobs:
       operator_name: openstack
       go_version: 1.19.x
       operator_sdk_version: 1.31.0
-      bundle_dockerfile: ./custom-bundle.Dockerfile.pinned
+      bundle_dockerfile: ./custom-bundle.Dockerfile
       catalog_extra_bundles_script: ./hack/pin-bundle-images.sh
     secrets:
       IMAGENAMESPACE: ${{ secrets.IMAGENAMESPACE }}

.gitignore

@@ -10,6 +10,7 @@ testbin/*
 
 bundle/*
 bundle.Dockerfile
+bundle_extra_data
 custom-bundle.Dockerfile.pinned
 storage-bundle.Dockerfile.pinned
 

.prow_ci.env

@@ -1 +1,2 @@
 export USE_IMAGE_DIGESTS=true
+export BUNDLE_DOCKERFILE=custom-bundle.Dockerfile

Makefile

@@ -234,18 +234,16 @@ $(GINKGO): $(LOCALBIN)
 	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo
 
 .PHONY: bundle
-bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.
+bundle: build manifests kustomize bundle-cache-extra-data ## Generate bundle manifests and metadata, then validate generated files.
 	operator-sdk generate kustomize manifests -q
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
 	$(KUSTOMIZE) build config/manifests | operator-sdk generate bundle $(BUNDLE_GEN_FLAGS)
 	cp dependencies.yaml ./bundle/metadata
 	operator-sdk bundle validate ./bundle
-	DOCKERFILE=custom-bundle.Dockerfile /bin/bash hack/pin-bundle-images.sh
-
 
 .PHONY: bundle-build
 bundle-build: bundle ## Build the bundle image.
-	podman build -f custom-bundle.Dockerfile.pinned -t $(BUNDLE_IMG) .
+	podman build -f custom-bundle.Dockerfile -t $(BUNDLE_IMG) .
 
 .PHONY: bundle-push
 bundle-push: ## Push the bundle image.
@@ -364,3 +362,11 @@ run-with-webhook: export HEALTH_PORT?=8081
 run-with-webhook: manifests generate fmt vet ## Run a controller from your host.
 	/bin/bash hack/configure_local_webhook.sh
 	go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)"
+
+# refresh the bundle extra data based on go.mod entries
+# bundle extra data includes:
+#  - extracted ENV vars from all operators (required for webhooks)
+#  - dataplane-operator bundle (this gets merged into openstack-operator's bundle)
+.PHONY: bundle-cache-extra-data
+bundle-cache-extra-data: build
+	bash hack/bundle-cache-data.sh

README.md

@@ -69,7 +69,7 @@ bundle.
 make dep-bundle-build-push
 ```
 
-3. Run bundle-build. This will execute podman to build the custom-bundle.Dockerfile.pinned.
+3. Run bundle-build. This will execute podman to build the custom-bundle.Dockerfile.
 
 ```sh
 make bundle-build

custom-bundle.Dockerfile

@@ -1,27 +1,5 @@
 ARG GOLANG_CTX=golang:1.19
-ARG INFRA_BUNDLE=quay.io/openstack-k8s-operators/infra-operator-bundle:latest
-ARG KEYSTONE_BUNDLE=quay.io/openstack-k8s-operators/keystone-operator-bundle:latest
-ARG MARIADB_BUNDLE=quay.io/openstack-k8s-operators/mariadb-operator-bundle:latest
-ARG PLACEMENT_BUNDLE=quay.io/openstack-k8s-operators/placement-operator-bundle:latest
-ARG OVN_BUNDLE=quay.io/openstack-k8s-operators/ovn-operator-bundle:latest
-ARG NEUTRON_BUNDLE=quay.io/openstack-k8s-operators/neutron-operator-bundle:latest
-ARG ANSIBLEEE_BUNDLE=quay.io/openstack-k8s-operators/openstack-ansibleee-operator-bundle:latest
-ARG DATAPLANE_BUNDLE=quay.io/openstack-k8s-operators/dataplane-operator-bundle:latest
-ARG NOVA_BUNDLE=quay.io/openstack-k8s-operators/nova-operator-bundle:latest
-ARG HEAT_BUNDLE=quay.io/openstack-k8s-operators/heat-operator-bundle:latest
-ARG IRONIC_BUNDLE=quay.io/openstack-k8s-operators/ironic-operator-bundle:latest
-ARG BAREMETAL_BUNDLE=quay.io/openstack-k8s-operators/openstack-baremetal-operator-bundle:latest
-ARG TELEMETRY_BUNDLE=quay.io/openstack-k8s-operators/telemetry-operator-bundle:latest
-ARG HORIZON_BUNDLE=quay.io/openstack-k8s-operators/horizon-operator-bundle:latest
-ARG GLANCE_BUNDLE=quay.io/openstack-k8s-operators/glance-operator-bundle:latest
-ARG CINDER_BUNDLE=quay.io/openstack-k8s-operators/cinder-operator-bundle:latest
-ARG MANILA_BUNDLE=quay.io/openstack-k8s-operators/manila-operator-bundle:latest
-ARG SWIFT_BUNDLE=quay.io/openstack-k8s-operators/swift-operator-bundle:latest
-ARG OCTAVIA_BUNDLE=quay.io/openstack-k8s-operators/octavia-operator-bundle:latest
-ARG DESIGNATE_BUNDLE=quay.io/openstack-k8s-operators/designate-operator-bundle:latest
-ARG BARBICAN_BUNDLE=quay.io/openstack-k8s-operators/barbican-operator-bundle:latest
-
-# Build the manager binary
+
 FROM $GOLANG_CTX as builder
 
 WORKDIR /workspace
@@ -39,90 +17,24 @@ RUN go mod download
 COPY cmd/csv-merger/csv-merger.go csv-merger.go
 COPY pkg/ pkg/
 
-# Build
+# Build the csv-merger
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o csv-merger csv-merger.go
 
-FROM $INFRA_BUNDLE as infra-bundle
-FROM $KEYSTONE_BUNDLE as keystone-bundle
-FROM $MARIADB_BUNDLE as mariadb-bundle
-FROM $PLACEMENT_BUNDLE as placement-bundle
-FROM $OVN_BUNDLE as ovn-bundle
-FROM $NEUTRON_BUNDLE as neutron-bundle
-FROM $ANSIBLEEE_BUNDLE as openstack-ansibleee-bundle
-FROM $DATAPLANE_BUNDLE as dataplane-bundle
-FROM $NOVA_BUNDLE as nova-bundle
-FROM $HEAT_BUNDLE as heat-bundle
-FROM $IRONIC_BUNDLE as ironic-bundle
-FROM $BAREMETAL_BUNDLE as baremetal-bundle
-FROM $TELEMETRY_BUNDLE as telemetry-bundle
-FROM $HORIZON_BUNDLE as horizon-bundle
-FROM $GLANCE_BUNDLE as glance-bundle
-FROM $CINDER_BUNDLE as cinder-bundle
-FROM $MANILA_BUNDLE as manila-bundle
-FROM $SWIFT_BUNDLE as swift-bundle
-FROM $OCTAVIA_BUNDLE as octavia-bundle
-FROM $DESIGNATE_BUNDLE as designate-bundle
-FROM $BARBICAN_BUNDLE as barbican-bundle
-
 FROM $GOLANG_CTX as merger
 WORKDIR /workspace
 COPY --from=builder /workspace/csv-merger .
 
 # local operator manifests
 COPY bundle/manifests /manifests/
+COPY bundle_extra_data /bundle_extra_data
+RUN cp -a /bundle_extra_data/manifests/* /manifests/
 
-# Custom Manifests
-COPY --from=keystone-bundle /manifests/* /manifests/
-COPY --from=mariadb-bundle /manifests/* /manifests/
-COPY --from=infra-bundle /manifests/* /manifests/
-COPY --from=placement-bundle /manifests/* /manifests/
-COPY --from=ovn-bundle /manifests/* /manifests/
-COPY --from=neutron-bundle /manifests/* /manifests/
-COPY --from=openstack-ansibleee-bundle /manifests/* /manifests/
-COPY --from=dataplane-bundle /manifests/* /manifests/
-COPY --from=nova-bundle /manifests/* /manifests/
-COPY --from=heat-bundle /manifests/* /manifests/
-COPY --from=ironic-bundle /manifests/* /manifests/
-COPY --from=baremetal-bundle /manifests/* /manifests/
-COPY --from=telemetry-bundle /manifests/* /manifests/
-COPY --from=horizon-bundle /manifests/* /manifests/
-COPY --from=glance-bundle /manifests/* /manifests/
-COPY --from=cinder-bundle /manifests/* /manifests/
-COPY --from=manila-bundle /manifests/* /manifests/
-COPY --from=swift-bundle /manifests/* /manifests/
-COPY --from=octavia-bundle /manifests/* /manifests/
-COPY --from=designate-bundle /manifests/* /manifests/
-COPY --from=barbican-bundle /manifests/* /manifests/
-
-# extract all the env vars (NOTE/FIXME: base-csv is unused below to be refactored)
-RUN /workspace/csv-merger \
-  --export-env-file=/env-vars.yaml \
-  --mariadb-csv=/manifests/mariadb-operator.clusterserviceversion.yaml \
-  --infra-csv=/manifests/infra-operator.clusterserviceversion.yaml \
-  --keystone-csv=/manifests/keystone-operator.clusterserviceversion.yaml \
-  --placement-csv=/manifests/placement-operator.clusterserviceversion.yaml \
-  --ovn-csv=/manifests/ovn-operator.clusterserviceversion.yaml \
-  --neutron-csv=/manifests/neutron-operator.clusterserviceversion.yaml \
-  --ansibleee-csv=/manifests/openstack-ansibleee-operator.clusterserviceversion.yaml \
-  --nova-csv=/manifests/nova-operator.clusterserviceversion.yaml \
-  --heat-csv=/manifests/heat-operator.clusterserviceversion.yaml \
-  --ironic-csv=/manifests/ironic-operator.clusterserviceversion.yaml \
-  --baremetal-csv=/manifests/openstack-baremetal-operator.clusterserviceversion.yaml \
-  --horizon-csv=/manifests/horizon-operator.clusterserviceversion.yaml \
-  --telemetry-csv=/manifests/telemetry-operator.clusterserviceversion.yaml \
-  --glance-csv=/manifests/glance-operator.clusterserviceversion.yaml \
-  --cinder-csv=/manifests/cinder-operator.clusterserviceversion.yaml \
-  --manila-csv=/manifests/manila-operator.clusterserviceversion.yaml \
-  --swift-csv=/manifests/swift-operator.clusterserviceversion.yaml \
-  --octavia-csv=/manifests/octavia-operator.clusterserviceversion.yaml \
-  --designate-csv=/manifests/designate-operator.clusterserviceversion.yaml \
-  --barbican-csv=/manifests/barbican-operator.clusterserviceversion.yaml \
-  --base-csv=/manifests/openstack-operator.clusterserviceversion.yaml | tee /fixme-required-for-now-but-will-can-made-optional.yaml
-
-# apply all the ENV vars to the actual base-csv
+# Merge things into our openstack-operator CSV:
+#  -dataplane-operator CSV
+#  -ENV vars from all operators (for webhooks)
 RUN /workspace/csv-merger \
-  --import-env-files=/env-vars.yaml \
-  --dataplane-csv=/manifests/dataplane-operator.clusterserviceversion.yaml \
+  --import-env-files=/bundle_extra_data/env-vars.yaml \
+  --dataplane-csv=/bundle_extra_data/manifests/dataplane-operator.clusterserviceversion.yaml \
   --base-csv=/manifests/openstack-operator.clusterserviceversion.yaml | tee /openstack-operator.clusterserviceversion.yaml.new
 
 # remove all individual operator CSV's
@@ -151,7 +63,7 @@ COPY bundle/tests/scorecard /tests/scorecard/
 
 # copy in manifests from operators
 COPY bundle/manifests /manifests/
-COPY --from=merger /manifests/dataplane.openstack.* /manifests/
+COPY --from=merger /manifests/* /manifests/
 
 # overwrite with the final merged CSV
 COPY --from=merger /openstack-operator.clusterserviceversion.yaml.new /manifests/openstack-operator.clusterserviceversion.yaml

hack/bundle-cache-data.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# extract select data from bundles:
+#  -env vars from all service operators
+#  -dataplane-operator bundle is cached (in order to merge at build time)
+set -ex
+
+function extract_bundle {
+    local IN_DIR=$1
+    local OUT_DIR=$2
+    for X in $(file ${IN_DIR}/* | grep gzip | cut -f 1 -d ':'); do
+        tar xvf $X -C ${OUT_DIR}/;
+    done
+}
+
+function extract_csv {
+    local IN_DIR=$1
+    local OUT_DIR=$2
+    for X in $(file ${IN_DIR}/* | grep gzip | cut -f 1 -d ':'); do
+        tar xvf $X -C ${OUT_DIR} --wildcards --no-anchor '**/*clusterserviceversion.yaml'
+    done
+}
+
+OUT_BUNDLE=bundle_extra_data
+EXTRACT_DIR=tmp/bundle_extract
+
+mkdir -p "$EXTRACT_DIR"
+mkdir -p "$EXTRACT_DIR/csvs"
+mkdir -p "$OUT_BUNDLE"
+
+for BUNDLE in $(hack/pin-bundle-images.sh | tr "," " "); do
+    skopeo copy "docker://$BUNDLE" dir:${EXTRACT_DIR}/tmp;
+    if echo $BUNDLE | grep dataplane-operator &> /dev/null; then
+        extract_bundle "${EXTRACT_DIR}/tmp" "${OUT_BUNDLE}/"
+    else
+        extract_csv "${EXTRACT_DIR}/tmp" "${EXTRACT_DIR}/csvs"
+    fi
+done
+
+# Extract the ENV vars from all the CSVs
+CSV_DIR="${EXTRACT_DIR}/csvs/manifests"
+bin/csv-merger \
+    --export-env-file=$OUT_BUNDLE/env-vars.yaml \
+    --mariadb-csv=$CSV_DIR/mariadb-operator.clusterserviceversion.yaml \
+    --infra-csv=$CSV_DIR/infra-operator.clusterserviceversion.yaml \
+    --keystone-csv=$CSV_DIR/keystone-operator.clusterserviceversion.yaml \
+    --placement-csv=$CSV_DIR/placement-operator.clusterserviceversion.yaml \
+    --ovn-csv=$CSV_DIR/ovn-operator.clusterserviceversion.yaml \
+    --neutron-csv=$CSV_DIR/neutron-operator.clusterserviceversion.yaml \
+    --ansibleee-csv=$CSV_DIR/openstack-ansibleee-operator.clusterserviceversion.yaml \
+    --nova-csv=$CSV_DIR/nova-operator.clusterserviceversion.yaml \
+    --heat-csv=$CSV_DIR/heat-operator.clusterserviceversion.yaml \
+    --ironic-csv=$CSV_DIR/ironic-operator.clusterserviceversion.yaml \
+    --baremetal-csv=$CSV_DIR/openstack-baremetal-operator.clusterserviceversion.yaml \
+    --horizon-csv=$CSV_DIR/horizon-operator.clusterserviceversion.yaml \
+    --telemetry-csv=$CSV_DIR/telemetry-operator.clusterserviceversion.yaml \
+    --glance-csv=$CSV_DIR/glance-operator.clusterserviceversion.yaml \
+    --cinder-csv=$CSV_DIR/cinder-operator.clusterserviceversion.yaml \
+    --manila-csv=$CSV_DIR/manila-operator.clusterserviceversion.yaml \
+    --swift-csv=$CSV_DIR/swift-operator.clusterserviceversion.yaml \
+    --octavia-csv=$CSV_DIR/octavia-operator.clusterserviceversion.yaml \
+    --designate-csv=$CSV_DIR/designate-operator.clusterserviceversion.yaml \
+    --barbican-csv=$CSV_DIR/barbican-operator.clusterserviceversion.yaml \
+    --base-csv=config/manifests/bases/openstack-operator.clusterserviceversion.yaml | tee "$EXTRACT_DIR/out.yaml"
+
+# cleanup our temporary dir used for extraction
+rm -Rf "$EXTRACT_DIR"
+
+# we only keep manifests from extracted (merged) bundles
+rm -Rf "$OUT_BUNDLE/metadata"
+rm -Rf "$OUT_BUNDLE/tests"