Add controller to deploy machine-api-controllers for full functiona…

…lity (#7)

The original deployment is done by [machine-api-operator](https://github.com/openshift/machine-api-operator/blob/9c3e4a04009ae84958c25b4cbb380a24e7260761/pkg/operator/sync.go#L70-L164), but it there is no possibility of using this on with non-inlined providers.

The controller refuses to create a deployment if the upstream one exists and uses jsonnet for easier rendering. Upstream images are taken from the upstream image configmap.

The deployment was taken from a vSphere cluster with some hidden dependencies like [machine-controller-manager](https://github.com/openshift/machine-api-operator/blob/a6fe8378811630a030c8509b180fcec5f53ce3d5/Dockerfile#L12) removed.

Makefile

@@ -57,8 +57,9 @@ clean: ## Cleans up the generated resources
 	rm -rf .tmpvendor
 
 .PHONY: run
+RUN_TARGET ?= manager
 run: generate fmt vet ## Run a controller from your host.
-	go run ./main.go
+	go run ./main.go "-target=$(RUN_TARGET)"
 
 ###
 ### Assets

config/samples/machineset-cloudscale-known-working.yml

@@ -0,0 +1,43 @@
+apiVersion: machine.openshift.io/v1beta1
+kind: MachineSet
+metadata:
+  name: app
+  namespace: openshift-machine-api
+  labels:
+    machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
+    name: app
+spec:
+  deletePolicy: Oldest
+  replicas: 0
+  selector:
+    matchLabels:
+      machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
+      machine.openshift.io/cluster-api-machineset: app
+  template:
+    metadata:
+      labels:
+        machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
+        machine.openshift.io/cluster-api-machine-role: app
+        machine.openshift.io/cluster-api-machine-type: app
+        machine.openshift.io/cluster-api-machineset: app
+    spec:
+      lifecycleHooks: {}
+      metadata:
+        labels:
+          node-role.kubernetes.io/app: ""
+          node-role.kubernetes.io/worker: ""
+      providerSpec:
+        value:
+          zone: rma1
+          baseDomain: lab-cloudscale-rma-0.appuio.cloud
+          flavor: flex-16-4
+          image: custom:rhcos-4.15
+          rootVolumeSizeGB: 100
+          antiAffinityKey: app
+          interfaces:
+          - type: Private
+            networkUUID: fd2b132d-f5d0-4024-b99f-68e5321ab4d1
+          userDataSecret:
+            name: cloudscale-user-data
+          tokenSecret:
+            name: cloudscale-rw-token

controllers/machine_api_controllers_controller.go

@@ -0,0 +1,125 @@
+package controllers
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/go-jsonnet"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+const (
+	imagesConfigMapName            = "machine-api-operator-images"
+	originalUpstreamDeploymentName = "machine-api-controllers"
+	imageKey                       = "images.json"
+
+	caBundleConfigMapName = "appuio-machine-api-ca-bundle"
+)
+
+//go:embed machine_api_controllers_deployment.jsonnet
+var deploymentTemplate string
+
+// MachineAPIControllersReconciler creates a appuio-machine-api-controllers deployment based on the images.json ConfigMap
+// if the upstream machine-api-controllers does not exist.
+type MachineAPIControllersReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+
+	Namespace string
+}
+
+func (r *MachineAPIControllersReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	if req.Name != imagesConfigMapName {
+		return ctrl.Result{}, nil
+	}
+
+	l := log.FromContext(ctx).WithName("UpstreamDeploymentReconciler.Reconcile")
+	l.Info("Reconciling")
+
+	var imageCM corev1.ConfigMap
+	if err := r.Get(ctx, req.NamespacedName, &imageCM); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	ij, ok := imageCM.Data[imageKey]
+	if !ok {
+		return ctrl.Result{}, fmt.Errorf("%q key not found in ConfigMap %q", imageKey, imagesConfigMapName)
+	}
+	images := make(map[string]string)
+	if err := json.Unmarshal([]byte(ij), &images); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to unmarshal %q from %q: %w", imageKey, imagesConfigMapName, err)
+	}
+
+	// Check that the original upstream deployment does not exist
+	// If it does, we should not create the new deployment
+	var upstreamDeployment appsv1.Deployment
+	err := r.Get(ctx, client.ObjectKey{
+		Name:      originalUpstreamDeploymentName,
+		Namespace: r.Namespace,
+	}, &upstreamDeployment)
+	if err == nil {
+		return ctrl.Result{}, fmt.Errorf("original upstream deployment %s already exists", originalUpstreamDeploymentName)
+	} else if !apierrors.IsNotFound(err) {
+		return ctrl.Result{}, fmt.Errorf("failed to check for original upstream deployment %s: %w", originalUpstreamDeploymentName, err)
+	}
+
+	vm, err := jsonnetVMWithContext(images)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to create jsonnet VM: %w", err)
+	}
+
+	ud, err := vm.EvaluateAnonymousSnippet("controllers_deployment.jsonnet", deploymentTemplate)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to evaluate jsonnet: %w", err)
+	}
+
+	// TODO(bastjan) this could be way more generic and support any kind of object.
+	// We don't need any other object types right now, so we're keeping it simple.
+	var toDeploy appsv1.Deployment
+	if err := json.Unmarshal([]byte(ud), &toDeploy); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to unmarshal jsonnet output: %w", err)
+	}
+	if toDeploy.APIVersion != "apps/v1" || toDeploy.Kind != "Deployment" {
+		return ctrl.Result{}, fmt.Errorf("expected Deployment, got %s/%s", toDeploy.APIVersion, toDeploy.Kind)
+	}
+	toDeploy.Namespace = r.Namespace
+	if err := controllerutil.SetControllerReference(&imageCM, &toDeploy, r.Scheme); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to set controller reference: %w", err)
+	}
+	if err := r.Client.Patch(ctx, &toDeploy, client.Apply, client.FieldOwner("upstream-deployment-controller")); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to apply Deployment %q: %w", toDeploy.GetName(), err)
+	}
+
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *MachineAPIControllersReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.ConfigMap{}).
+		Owns(&appsv1.Deployment{}).
+		Complete(r)
+}
+
+func jsonnetVMWithContext(images map[string]string) (*jsonnet.VM, error) {
+	jcr, err := json.Marshal(map[string]any{
+		"images": images,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal jsonnet context: %w", err)
+	}
+	jvm := jsonnet.MakeVM()
+	jvm.ExtCode("context", string(jcr))
+	// Don't allow imports
+	jvm.Importer(&jsonnet.MemoryImporter{})
+	return jvm, nil
+}

controllers/machine_api_controllers_controller_test.go

@@ -0,0 +1,159 @@
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func Test_MachineAPIControllersReconciler_Reconcile(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	const namespace = "openshift-machine-api"
+
+	scheme := runtime.NewScheme()
+	require.NoError(t, clientgoscheme.AddToScheme(scheme))
+
+	images := map[string]string{
+		"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
+		"kubeRBACProxy":      "registry.io/kube-rbac-proxy:v1.0.0",
+	}
+	imagesJSON, err := json.Marshal(images)
+	require.NoError(t, err)
+
+	ucm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      imagesConfigMapName,
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			imageKey: string(imagesJSON),
+		},
+	}
+
+	c := &fakeSSA{
+		fake.NewClientBuilder().
+			WithScheme(scheme).
+			WithRuntimeObjects(ucm).
+			Build(),
+	}
+
+	r := &MachineAPIControllersReconciler{
+		Client: c,
+		Scheme: scheme,
+
+		Namespace: namespace,
+	}
+
+	_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
+	require.NoError(t, err)
+
+	var deployment appsv1.Deployment
+	require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "appuio-" + originalUpstreamDeploymentName}, &deployment))
+
+	assert.Equal(t, "system-node-critical", deployment.Spec.Template.Spec.PriorityClassName)
+	for _, c := range deployment.Spec.Template.Spec.Containers {
+		if c.Image == images["machineAPIOperator"] || c.Image == images["kubeRBACProxy"] {
+			continue
+		}
+		t.Errorf("expected image %q or %q, got %q", images["machineAPIOperator"], images["kubeRBACProxy"], c.Image)
+	}
+}
+
+func Test_MachineAPIControllersReconciler_OriginalDeploymentExists(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	const namespace = "openshift-machine-api"
+
+	scheme := runtime.NewScheme()
+	require.NoError(t, clientgoscheme.AddToScheme(scheme))
+
+	images := map[string]string{
+		"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
+		"kubeRBACProxy":      "registry.io/kube-rbac-proxy:v1.0.0",
+	}
+	imagesJSON, err := json.Marshal(images)
+	require.NoError(t, err)
+
+	ucm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      imagesConfigMapName,
+			Namespace: namespace,
+		},
+		Data: map[string]string{
+			imageKey: string(imagesJSON),
+		},
+	}
+
+	origDeploy := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      originalUpstreamDeploymentName,
+			Namespace: namespace,
+		},
+	}
+
+	c := &fakeSSA{
+		fake.NewClientBuilder().
+			WithScheme(scheme).
+			WithRuntimeObjects(ucm, origDeploy).
+			Build(),
+	}
+
+	r := &MachineAPIControllersReconciler{
+		Client: c,
+		Scheme: scheme,
+
+		Namespace: namespace,
+	}
+
+	_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
+	require.ErrorContains(t, err, "machine-api-controllers already exists")
+}
+
+// fakeSSA is a fake client that approximates SSA.
+// It creates objects that don't exist yet and _updates_ them if they exist.
+// This is completely kaputt since the object is overwritten with the new object.
+// See https://github.com/kubernetes-sigs/controller-runtime/issues/2341
+type fakeSSA struct {
+	client.WithWatch
+}
+
+// Patch approximates SSA by creating objects that don't exist yet.
+func (f *fakeSSA) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	// Apply patches are supposed to upsert, but fake client fails if the object doesn't exist,
+	// if an apply patch occurs for an object that doesn't yet exist, create it.
+	if patch.Type() != types.ApplyPatchType {
+		return f.WithWatch.Patch(ctx, obj, patch, opts...)
+	}
+	check, ok := obj.DeepCopyObject().(client.Object)
+	if !ok {
+		return errors.New("could not check for object in fake client")
+	}
+	if err := f.WithWatch.Get(ctx, client.ObjectKeyFromObject(obj), check); apierrors.IsNotFound(err) {
+		if err := f.WithWatch.Create(ctx, check); err != nil {
+			return fmt.Errorf("could not inject object creation for fake: %w", err)
+		}
+	} else if err != nil {
+		return fmt.Errorf("could not check for object in fake client: %w", err)
+	}
+	return f.WithWatch.Update(ctx, obj)
+}