Let payload provider render feature gate yaml

Currently we render the feature gate at the HCP level which results in a discrepancy when mco consume it along with a given NodePool payload-version that differs from the one used by the HCP to generate it.
Related #2643

CI Failure example
bootstrap.go:47] error running MCC[BOOTSTRAP]: error creating feature gate access: unable to determine features: missing desired version \"4.14.0-0.ci.test-2023-06-08-101141-ci-op-gvin17ww-initial\" in featuregates.config.openshift.io/cluster\n"}

https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_machine-config-operator/3724/pull-ci-openshift-machine-config-operator-master-e2e-hypershift/1666749702499995648

https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/origin-ci-test/pr-logs/pull/openshift_machine-config-operator/3724/pull-ci-openshift-machine-config-operator-master-e2e-hypershift/1666749702499995648/artifacts/e2e-hypershift/run-e2e/artifacts/TestNodePool_PreTeardownClusterDump/namespaces/e2e-clusters-wvxmd-example-fmxk8/core/pods/logs/ignition-server-768699fd7b-w49lq-ignition-server.log

control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go

@@ -420,6 +420,19 @@ func reconcileDeployment(deployment *appsv1.Deployment,
 							EmptyDir: &corev1.EmptyDirVolumeSource{},
 						},
 					},
+					{
+						Name: "bootstrap-manifests",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "manifests",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+
 					{
 						Name: "shared",
 						VolumeSource: corev1.VolumeSource{
@@ -437,7 +450,7 @@ func reconcileDeployment(deployment *appsv1.Deployment,
 						},
 						Args: []string{
 							"-c",
-							invokeFeatureGateRenderScript("/shared", releaseVersion, string(featureGateYAML)),
+							invokeFeatureGateRenderScript("/shared", string(featureGateYAML)),
 						},
 						Resources: corev1.ResourceRequirements{
 							Requests: corev1.ResourceList{
@@ -522,6 +535,14 @@ func reconcileDeployment(deployment *appsv1.Deployment,
 								Name:      "payloads",
 								MountPath: "/payloads",
 							},
+							{
+								Name:      "bootstrap-manifests",
+								MountPath: "/usr/share/bootkube/manifests/bootstrap-manifests",
+							},
+							{
+								Name:      "manifests",
+								MountPath: "/usr/share/bootkube/manifests/manifests",
+							},
 							{
 								Name:      "shared",
 								MountPath: "/shared",
@@ -575,24 +596,19 @@ func reconcilePodMonitor(podMonitor *prometheusoperatorv1.PodMonitor,
 	return nil
 }
 
-func invokeFeatureGateRenderScript(workDir, payloadVersion, featureGateYAML string) string {
+// invokeFeatureGateRenderScript writes the passed yaml to disk in a given dir.
+func invokeFeatureGateRenderScript(workDir, featureGateYAML string) string {
 	var script = `#!/bin/bash
 set -e
 cd /tmp
 mkdir input output manifests
 
 touch /tmp/manifests/99_feature-gate.yaml
 cat <<EOF >/tmp/manifests/99_feature-gate.yaml
-%[3]s
+%[2]s
 EOF
 
-/usr/bin/cluster-config-operator render \
-   --config-output-file config \
-   --asset-input-dir /tmp/input \
-   --asset-output-dir /tmp/output \
-   --rendered-manifest-files=/tmp/manifests \
-   --payload-version=%[2]s
 cp /tmp/manifests/99_feature-gate.yaml %[1]s/99_feature-gate.yaml
 `
-	return fmt.Sprintf(script, workDir, payloadVersion, featureGateYAML)
+	return fmt.Sprintf(script, workDir, featureGateYAML)
 }

ignition-server/controllers/local_ignitionprovider.go

@@ -240,23 +240,6 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 		return nil, fmt.Errorf("failed to extract templates from image: %w", err)
 	}
 
-	// Write out the feature gate manifest to the MCC dir.
-	// Use the feature gate from the hosted control plane which should reflect the feature gate of the cluster.
-	if err := func() error {
-		featureGateBytes, err := os.ReadFile(p.FeatureGateManifest)
-		if err != nil {
-			return fmt.Errorf("failed to read feature gate: %w", err)
-		}
-
-		if err := os.WriteFile(filepath.Join(mccBaseDir, "99_feature-gate.yaml"), featureGateBytes, 0644); err != nil {
-			return fmt.Errorf("failed to write feature gate: %w", err)
-		}
-
-		return nil
-	}(); err != nil {
-		return nil, fmt.Errorf("failed to extract feature gate: %w", err)
-	}
-
 	// Extract binaries from the MCO image into the bin directory
 	err = func() error {
 		start := time.Now()
@@ -276,6 +259,28 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 				return fmt.Errorf("failed to close file: %w", err)
 			}
 		}
+
+		component := "cluster-config-operator"
+		clusterConfigOperatorImage, ok := imageProvider.ImageExist(component)
+		if !ok {
+			return fmt.Errorf("release image does not contain cluster-config-operator (images: %v)", imageProvider.ComponentImages())
+		}
+		log.Info("discovered cluster config operator image", "image", clusterConfigOperatorImage)
+
+		file, err := os.Create(filepath.Join(binDir, component))
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+		if err := file.Chmod(0777); err != nil {
+			return fmt.Errorf("failed to chmod file: %w", err)
+		}
+		if err := p.ImageFileCache.extractImageFile(ctx, clusterConfigOperatorImage, pullSecret, filepath.Join("usr/bin/", component), file); err != nil {
+			return fmt.Errorf("failed to extract image file: %w", err)
+		}
+		if err := file.Close(); err != nil {
+			return fmt.Errorf("failed to close file: %w", err)
+		}
+
 		log.Info("downloaded binaries", "time", time.Since(start).Round(time.Second).String())
 		return nil
 	}()
@@ -288,6 +293,31 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 		return nil, fmt.Errorf("failed to parse payload version: %w", err)
 	}
 
+	featureGateBytes, err := os.ReadFile(p.FeatureGateManifest)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read feature gate: %w", err)
+	}
+
+	err = func() error {
+		start := time.Now()
+
+		args := []string{
+			"-c",
+			invokeFeatureGateRenderScript(filepath.Join(binDir, "cluster-config-operator"), filepath.Join(workDir, "cco"), mccBaseDir, payloadVersion, string(featureGateBytes)),
+		}
+
+		cmd := exec.CommandContext(ctx, "/bin/bash", args...)
+		out, err := cmd.CombinedOutput()
+		log.Info("cluster-config-operator process completed", "time", time.Since(start).Round(time.Second).String(), "output", string(out))
+		if err != nil {
+			return fmt.Errorf("cluster-config-operator process failed: %s: %w", string(out), err)
+		}
+		return nil
+	}()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute cluster-config-operator: %w", err)
+	}
+
 	// First, run the MCO using templates and image refs as input. This generates
 	// output for the MCC.
 	err = func() error {
@@ -678,3 +708,50 @@ func findMatchingManifest(imageRef string, deserializedManifestList *manifestlis
 
 	return "", fmt.Errorf("imageRef is an unknown format to parse, imageRef: %s", imageRef)
 }
+
+func invokeFeatureGateRenderScript(binary, workDir, outputDir string, payloadVersion semver.Version, featureGateYAML string) string {
+	var script = `#!/bin/bash
+set -e
+mkdir -p %[2]s
+
+cd %[2]s
+mkdir -p input output manifests
+
+touch %[2]s/manifests/99_feature-gate.yaml
+cat <<EOF >%[2]s/manifests/99_feature-gate.yaml
+%[5]s
+EOF
+
+%[1]s render \
+   --config-output-file config \
+   --asset-input-dir %[2]s/input \
+   --asset-output-dir %[2]s/output \
+   --rendered-manifest-files=%[2]s/manifests \
+   --payload-version=%[4]s 
+cp %[2]s/manifests/99_feature-gate.yaml %[3]s/99_feature-gate.yaml
+`
+
+	// Depending on the version, we need different args.
+	if payloadVersion.Minor < 14 {
+		script = `#!/bin/bash
+set -e
+mkdir -p %[2]s
+
+cd %[2]s
+mkdir -p input output manifests
+
+touch %[2]s/manifests/99_feature-gate.yaml
+cat <<EOF >%[2]s/manifests/99_feature-gate.yaml
+%[5]s
+EOF
+
+%[1]s render \
+   --config-output-file config \
+   --asset-input-dir %[2]s/input \
+   --asset-output-dir %[2]s/output
+cp %[2]s/manifests/99_feature-gate.yaml %[3]s/99_feature-gate.yaml
+`
+	}
+
+	return fmt.Sprintf(script, binary, workDir, outputDir, payloadVersion, featureGateYAML)
+}

test/e2e/nodepool_test.go

@@ -89,11 +89,10 @@ func TestNodePool(t *testing.T) {
 				manifestBuilder: NewNTOMachineConfigInPlaceRolloutTestManifest(hostedCluster),
 			},
 		*/
-		// Until this is fixed https://github.com/openshift/hypershift/pull/2664.
-		//{
-		//	name: "TestNodePoolReplaceUpgrade",
-		//	test: NewNodePoolUpgradeTest(ctx, mgmtClient, hostedCluster, hostedClusterClient, clusterOpts, globalOpts.PreviousReleaseImage, globalOpts.LatestReleaseImage),
-		//},
+		{
+			name: "TestNodePoolReplaceUpgrade",
+			test: NewNodePoolUpgradeTest(ctx, mgmtClient, hostedCluster, hostedClusterClient, clusterOpts, globalOpts.PreviousReleaseImage, globalOpts.LatestReleaseImage),
+		},
 		// TODO: (jparrill) Re-enable when https://issues.redhat.com/browse/OCPBUGS-10218 is fixed
 		/*
 			{