Let payload provider render feature gate yaml

Currently we render the feature gate at the HCP level which results in a discrepancy when mco consume it along with a given NodePool payload-version that differs from the one used by the HCP to generate it.
Related #2643

CI Failure example
bootstrap.go:47] error running MCC[BOOTSTRAP]: error creating feature gate access: unable to determine features: missing desired version \"4.14.0-0.ci.test-2023-06-08-101141-ci-op-gvin17ww-initial\" in featuregates.config.openshift.io/cluster\n"}

https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_machine-config-operator/3724/pull-ci-openshift-machine-config-operator-master-e2e-hypershift/1666749702499995648

https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/origin-ci-test/pr-logs/pull/openshift_machine-config-operator/3724/pull-ci-openshift-machine-config-operator-master-e2e-hypershift/1666749702499995648/artifacts/e2e-hypershift/run-e2e/artifacts/TestNodePool_PreTeardownClusterDump/namespaces/e2e-clusters-wvxmd-example-fmxk8/core/pods/logs/ignition-server-768699fd7b-w49lq-ignition-server.log

control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go

@@ -437,7 +437,7 @@ func reconcileDeployment(deployment *appsv1.Deployment,
 						},
 						Args: []string{
 							"-c",
-							invokeFeatureGateRenderScript("/shared", releaseVersion, string(featureGateYAML)),
+							invokeFeatureGateRenderScript("/shared", string(featureGateYAML)),
 						},
 						Resources: corev1.ResourceRequirements{
 							Requests: corev1.ResourceList{
@@ -575,24 +575,19 @@ func reconcilePodMonitor(podMonitor *prometheusoperatorv1.PodMonitor,
 	return nil
 }
 
-func invokeFeatureGateRenderScript(workDir, payloadVersion, featureGateYAML string) string {
+// invokeFeatureGateRenderScript writes the passed yaml to disk in a given dir.
+func invokeFeatureGateRenderScript(workDir, featureGateYAML string) string {
 	var script = `#!/bin/bash
 set -e
 cd /tmp
 mkdir input output manifests
 
 touch /tmp/manifests/99_feature-gate.yaml
 cat <<EOF >/tmp/manifests/99_feature-gate.yaml
-%[3]s
+%[2]s
 EOF
 
-/usr/bin/cluster-config-operator render \
-   --config-output-file config \
-   --asset-input-dir /tmp/input \
-   --asset-output-dir /tmp/output \
-   --rendered-manifest-files=/tmp/manifests \
-   --payload-version=%[2]s
 cp /tmp/manifests/99_feature-gate.yaml %[1]s/99_feature-gate.yaml
 `
-	return fmt.Sprintf(script, workDir, payloadVersion, featureGateYAML)
+	return fmt.Sprintf(script, workDir, featureGateYAML)
 }

ignition-server/controllers/local_ignitionprovider.go

@@ -240,23 +240,6 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 		return nil, fmt.Errorf("failed to extract templates from image: %w", err)
 	}
 
-	// Write out the feature gate manifest to the MCC dir.
-	// Use the feature gate from the hosted control plane which should reflect the feature gate of the cluster.
-	if err := func() error {
-		featureGateBytes, err := os.ReadFile(p.FeatureGateManifest)
-		if err != nil {
-			return fmt.Errorf("failed to read feature gate: %w", err)
-		}
-
-		if err := os.WriteFile(filepath.Join(mccBaseDir, "99_feature-gate.yaml"), featureGateBytes, 0644); err != nil {
-			return fmt.Errorf("failed to write feature gate: %w", err)
-		}
-
-		return nil
-	}(); err != nil {
-		return nil, fmt.Errorf("failed to extract feature gate: %w", err)
-	}
-
 	// Extract binaries from the MCO image into the bin directory
 	err = func() error {
 		start := time.Now()
@@ -276,6 +259,28 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 				return fmt.Errorf("failed to close file: %w", err)
 			}
 		}
+
+		component := "cluster-config-operator"
+		clusterConfigOperatorImage, ok := imageProvider.ImageExist(component)
+		if !ok {
+			return fmt.Errorf("release image does not contain cluster-config-operator (images: %v)", imageProvider.ComponentImages())
+		}
+		log.Info("discovered cluster config operator image", "image", clusterConfigOperatorImage)
+
+		file, err := os.Create(filepath.Join(binDir, component))
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+		if err := file.Chmod(0777); err != nil {
+			return fmt.Errorf("failed to chmod file: %w", err)
+		}
+		if err := p.ImageFileCache.extractImageFile(ctx, clusterConfigOperatorImage, pullSecret, filepath.Join("usr/bin/", component), file); err != nil {
+			return fmt.Errorf("failed to extract image file: %w", err)
+		}
+		if err := file.Close(); err != nil {
+			return fmt.Errorf("failed to close file: %w", err)
+		}
+
 		log.Info("downloaded binaries", "time", time.Since(start).Round(time.Second).String())
 		return nil
 	}()
@@ -288,6 +293,31 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 		return nil, fmt.Errorf("failed to parse payload version: %w", err)
 	}
 
+	featureGateBytes, err := os.ReadFile(p.FeatureGateManifest)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read feature gate: %w", err)
+	}
+
+	err = func() error {
+		start := time.Now()
+
+		args := []string{
+			"-c",
+			invokeFeatureGateRenderScript(mccBaseDir, payloadVersion.String(), string(featureGateBytes)),
+		}
+
+		cmd := exec.CommandContext(ctx, "/bin/bash", args...)
+		out, err := cmd.CombinedOutput()
+		log.Info("cluster-config-operator process completed", "time", time.Since(start).Round(time.Second).String(), "output", string(out))
+		if err != nil {
+			return fmt.Errorf("cluster-config-operator process failed: %w", err)
+		}
+		return nil
+	}()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute cluster-config-operator: %w", err)
+	}
+
 	// First, run the MCO using templates and image refs as input. This generates
 	// output for the MCC.
 	err = func() error {
@@ -678,3 +708,25 @@ func findMatchingManifest(imageRef string, deserializedManifestList *manifestlis
 
 	return "", fmt.Errorf("imageRef is an unknown format to parse, imageRef: %s", imageRef)
 }
+
+func invokeFeatureGateRenderScript(workDir, payloadVersion, featureGateYAML string) string {
+	var script = `#!/bin/bash
+set -e
+cd /tmp
+mkdir input output manifests
+
+touch /tmp/manifests/99_feature-gate.yaml
+cat <<EOF >/tmp/manifests/99_feature-gate.yaml
+%[3]s
+EOF
+
+/usr/bin/cluster-config-operator render \
+   --config-output-file config \
+   --asset-input-dir /tmp/input \
+   --asset-output-dir /tmp/output \
+   --rendered-manifest-files=/tmp/manifests \
+   --payload-version=%[2]s
+cp /tmp/manifests/99_feature-gate.yaml %[1]s/99_feature-gate.yaml
+`
+	return fmt.Sprintf(script, workDir, payloadVersion, featureGateYAML)
+}