Managed Infrastructure Maintenance Operator - Milestone 1

.pipelines/e2e.yml

@@ -70,6 +70,8 @@ jobs:
 
       - script: |
           export CI=true
+          # Tell the E2E binary to run the MIMO tests
+          export ARO_E2E_MIMO=true
           . secrets/env
           . ./hack/e2e/run-rp-and-e2e.sh
 
@@ -84,6 +86,9 @@ jobs:
           run_selenium
           validate_selenium_running
 
+          run_mimo_actuator
+          validate_mimo_actuator_running
+
           run_rp
           validate_rp_running
 
@@ -128,6 +133,7 @@ jobs:
 
           delete_e2e_cluster
           kill_rp
+          kill_mimo_actuator
           kill_selenium
           kill_podman
           kill_vpn

Makefile

@@ -2,7 +2,7 @@ SHELL = /bin/bash
 TAG ?= $(shell git describe --exact-match 2>/dev/null)
 COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro
-E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml
+E2E_FLAGS ?= -test.v --ginkgo.vv --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml
 E2E_LABEL ?= !smoke&&!regressiontest
 GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper
 OC ?= oc
@@ -68,7 +68,7 @@ aro: check-release generate
 
 .PHONY: runlocal-rp
 runlocal-rp:
-	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} rp
 
 .PHONY: az
 az: pyenv
@@ -197,7 +197,11 @@ proxy:
 
 .PHONY: runlocal-portal
 runlocal-portal:
-	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} portal
+
+.PHONY: runlocal-actuator
+runlocal-actuator:
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro ${ARO_CMD_ARGS} mimo-actuator
 
 .PHONY: build-portal
 build-portal:

cmd/aro/main.go

@@ -28,6 +28,7 @@ func usage() {
 	fmt.Fprintf(flag.CommandLine.Output(), "  %s operator {master,worker}\n", os.Args[0])
 	fmt.Fprintf(flag.CommandLine.Output(), "  %s update-versions\n", os.Args[0])
 	fmt.Fprintf(flag.CommandLine.Output(), "  %s update-role-sets\n", os.Args[0])
+	fmt.Fprintf(flag.CommandLine.Output(), "  %s mimo-actuator\n", os.Args[0])
 	flag.PrintDefaults()
 }
 
@@ -74,6 +75,9 @@ func main() {
 	case "update-role-sets":
 		checkArgs(1)
 		err = updatePlatformWorkloadIdentityRoleSets(ctx, log)
+	case "mimo-actuator":
+		checkArgs(1)
+		err = mimoActuator(ctx, log)
 	default:
 		usage()
 		os.Exit(2)

cmd/aro/mimoactuator.go

@@ -0,0 +1,103 @@
+package main
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/database"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/metrics/statsd"
+	"github.com/Azure/ARO-RP/pkg/metrics/statsd/golang"
+	"github.com/Azure/ARO-RP/pkg/mimo/actuator"
+	"github.com/Azure/ARO-RP/pkg/mimo/tasks"
+	"github.com/Azure/ARO-RP/pkg/proxy"
+	"github.com/Azure/ARO-RP/pkg/util/encryption"
+)
+
+func mimoActuator(ctx context.Context, log *logrus.Entry) error {
+	stop := make(chan struct{})
+
+	_env, err := env.NewEnv(ctx, log, env.COMPONENT_MIMO_ACTUATOR)
+	if err != nil {
+		return err
+	}
+
+	keys := []string{}
+	if !_env.IsLocalDevelopmentMode() {
+		keys = []string{
+			"MDM_ACCOUNT",
+			"MDM_NAMESPACE",
+		}
+	}
+
+	if err = env.ValidateVars(keys...); err != nil {
+		return err
+	}
+
+	m := statsd.New(ctx, log.WithField("component", "actuator"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET"))
+
+	g, err := golang.NewMetrics(_env.Logger(), m)
+	if err != nil {
+		return err
+	}
+	go g.Run()
+
+	aead, err := encryption.NewAEADWithCore(ctx, _env, env.EncryptionSecretV2Name, env.EncryptionSecretName)
+	if err != nil {
+		return err
+	}
+
+	dbc, err := database.NewDatabaseClientFromEnv(ctx, _env, log, m, aead)
+	if err != nil {
+		return err
+	}
+
+	dbName, err := env.DBName(_env)
+	if err != nil {
+		return err
+	}
+
+	clusters, err := database.NewOpenShiftClusters(ctx, dbc, dbName)
+	if err != nil {
+		return err
+	}
+
+	manifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName)
+	if err != nil {
+		return err
+	}
+
+	dbg := database.NewDBGroup().
+		WithOpenShiftClusters(clusters).
+		WithMaintenanceManifests(manifests)
+
+	go database.EmitMIMOMetrics(ctx, log, manifests, m)
+
+	dialer, err := proxy.NewDialer(_env.IsLocalDevelopmentMode())
+	if err != nil {
+		return err
+	}
+
+	a := actuator.NewService(_env, _env.Logger(), dialer, dbg, m)
+	a.SetMaintenanceTasks(tasks.DEFAULT_MAINTENANCE_TASKS)
+
+	sigterm := make(chan os.Signal, 1)
+	done := make(chan struct{})
+	signal.Notify(sigterm, syscall.SIGTERM)
+
+	go a.Run(ctx, stop, done)
+
+	<-sigterm
+	log.Print("received SIGTERM")
+	close(stop)
+	<-done
+
+	return nil
+}

cmd/aro/rp.go

@@ -154,7 +154,7 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error {
 		return err
 	}
 
-	go database.EmitMetrics(ctx, log, dbOpenShiftClusters, metrics)
+	go database.EmitOpenShiftClustersMetrics(ctx, log, dbOpenShiftClusters, metrics)
 
 	feAead, err := encryption.NewMulti(ctx, _env.ServiceKeyvault(), env.FrontendEncryptionSecretV2Name, env.FrontendEncryptionSecretName)
 	if err != nil {
@@ -172,6 +172,15 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error {
 		WithPlatformWorkloadIdentityRoleSets(dbPlatformWorkloadIdentityRoleSets).
 		WithSubscriptions(dbSubscriptions)
 
+	// MIMO only activated in development for now
+	if _env.IsLocalDevelopmentMode() {
+		dbMaintenanceManifests, err := database.NewMaintenanceManifests(ctx, dbc, dbName)
+		if err != nil {
+			return err
+		}
+		dbg.WithMaintenanceManifests(dbMaintenanceManifests)
+	}
+
 	f, err := frontend.NewFrontend(ctx, audit, log.WithField("component", "frontend"), _env, dbg, api.APIs, metrics, clusterm, feAead, hiveClusterManager, adminactions.NewKubeActions, adminactions.NewAzureActions, adminactions.NewAppLensActions, clusterdata.NewParallelEnricher(metrics, _env))
 	if err != nil {
 		return err

docs/mimo/README.md

@@ -0,0 +1,22 @@
+# MIMO Documentation
+
+The Managed Infrastructure Maintenance Operator, or MIMO, is a component of the Azure Red Hat OpenShift Resource Provider (ARO-RP) which is responsible for automated maintenance of clusters provisioned by the platform.
+MIMO specifically focuses on "managed infrastructure", the parts of ARO that are deployed and maintained by the RP and ARO Operator instead of by OCP (in-cluster) or Hive (out-of-cluster).
+
+MIMO consists of two main components, the [Actuator](./actuator.md) and the [Scheduler](./scheduler.md). It is primarily interfaced with via the [Admin API](./admin-api.md).
+
+## A Primer On MIMO
+
+The smallest thing that you can tell MIMO to run is a **Task** (see [`pkg/mimo/tasks/`](../../pkg/mimo/tasks/)).
+A Task is composed of reusable **Steps** (see [`pkg/mimo/steps/`](../../pkg/mimo/steps/)), reusing the framework utilised by AdminUpdate/Update/Install methods in `pkg/cluster/`.
+A Task only runs in the scope of a singular cluster.
+These steps are run in sequence and can return either **Terminal** errors (causing the ran Task to fail and not be retried) or **Transient** errors (which indicates that the Task can be retried later).
+
+Tasks are executed by the **Actuator** by way of creation of a **Maintenance Manifest**.
+This Manifest is created with the cluster ID (which is elided from the cluster-scoped Admin APIs), the Task ID (which is currently a UUID), and optional priority, "start after", and "start before" times which are filled in with defaults if not provided.
+The Actuator will treat these Maintenance Manifests as a work queue, taking ones which are past their "start after" time and executing them in order of earliest start-after and priority.
+After running each, a state will be written into the Manifest (with optional free-form status text) with the result of the ran Task.
+Manifests past their start-before times are marked as having a "timed out" state and not ran.
+
+Currently, Manifests are created by the Admin API.
+In the future, the Scheduler will create some these Manifests depending on cluster state/version and wall-clock time, providing the ability to perform tasks like rotations of secrets autonomously.

docs/mimo/actuator.md

@@ -0,0 +1,30 @@
+# Managed Infrastructure Maintenance Operator: Actuator
+
+The Actuator is the MIMO component that performs execution of tasks.
+The process of running tasks looks like this:
+
+```mermaid
+graph TD;
+    START((Start))-->QUERY;
+    QUERY[Fetch all State = Pending] -->SORT;
+    SORT[Sort tasks by RUNAFTER and PRIORITY]-->ITERATE[Iterate over tasks];
+    ITERATE-- Per Task -->ISEXPIRED;
+    subgraph PerTask[ ]
+    ISEXPIRED{{Is RUNBEFORE > now?}}-- Yes --> STATETIMEDOUT([State = TimedOut]) --> CONTINUE[Continue];
+    ISEXPIRED-- No --> DEQUEUECLUSTER;
+    DEQUEUECLUSTER[Claim lease on OpenShiftClusterDocument] --> DEQUEUE;
+    DEQUEUE[Actuator dequeues task]--> ISRETRYLIMIT;
+    ISRETRYLIMIT{{Have we retried the task too many times?}} -- Yes --> STATERETRYEXCEEDED([State = RetriesExceeded]) --> CONTINUE;
+    ISRETRYLIMIT -- No -->STATEINPROGRESS;
+    STATEINPROGRESS([State = InProgress]) -->RUN[[Task is run]];
+    RUN -- Success --> SUCCESS
+    RUN-- Terminal Error-->TERMINALERROR;
+    RUN-- Transient Error-->TRANSIENTERROR;
+    SUCCESS([State = Completed])-->DELEASECLUSTER
+    TERMINALERROR([State = Failed])-->DELEASECLUSTER;
+    TRANSIENTERROR([State = Pending])-->DELEASECLUSTER;
+    DELEASECLUSTER[Release Lease on OpenShiftClusterDocument] -->CONTINUE;
+    end
+    CONTINUE-->ITERATE;
+    ITERATE-- Finished -->END;
+```

docs/mimo/admin-api.md

@@ -0,0 +1,30 @@
+# Admin API
+
+All need `api-version=admin`.
+
+## GET /admin/RESOURCE_ID/maintenanceManifests
+
+Returns a list of MIMO maintenance manifests.
+
+## PUT /admin/RESOURCE_ID/maintenanceManifests
+
+Creates a new manifest. Returns the created manifest.
+
+### Example
+
+```sh
+curl -X PUT -k "https://localhost:8443/admin/subscriptions/fe16a035-e540-4ab7-80d9-373fa9a3d6ae/resourcegroups/v4-westeurope/providers/microsoft.redhatopenshift/openshiftclusters/abrownmimom1test/maintenanceManifests?api-version
+=admin" -d '{"maintenanceTaskID": "b41749fc-af26-4ab7-b5a1-e03f3ee4cba6"}' --header "Content-Type: application/json"
+```
+
+## GET /admin/RESOURCE_ID/maintenanceManifests/MANIFEST_ID
+
+Returns a manifest.
+
+## DELETE /admin/RESOURCE_ID/maintenanceManifests/MANIFEST_ID
+
+Deletes a manifest. This is only to be used as a last resort.
+
+## POST /admin/RESOURCE_ID/maintenanceManifests/MANIFEST_ID/cancel
+
+Cancels the manifest (the state becomes CANCELLED). It does not stop a task that is in the current process of execution.

docs/mimo/local-dev.md

@@ -0,0 +1,6 @@
+# Local Development
+
+1. Ensure that you have remade your databases (so that you have the MIMO ones), see [Prepare Your Dev Environment](../prepare-your-dev-environment.md).
+1. Run the local RP as usual.
+1. Run `make runlocal-actuator` to spawn the actuator.
+1. Perform queries against the Admin API to queue/monitor MIMO manifests.

docs/mimo/scheduler.md

@@ -0,0 +1,3 @@
+# MIMO Scheduler
+
+The MIMO Scheduler is a planned component, but is not yet implemented.

docs/mimo/writing-tasks.md

@@ -0,0 +1,48 @@
+# Writing MIMO Tasks
+
+Writing a MIMO task consists of three major steps:
+
+1. Writing the new functions in [`pkg/mimo/steps/`](../../pkg/mimo/steps/) which implement the specific behaviour (e.g. rotating a certificate), along with tests.
+2. Writing the new Task in [`pkg/mimo/tasks/`](../../pkg/mimo/tasks/) which combines the Step you have written with any pre-existing "check" steps (e.g. `EnsureAPIServerIsUp`).
+3. Adding the task with a new ID to [`pkg/mimo/const.go`](../../pkg/mimo/const.go) and `DEFAULT_MAINTENANCE_TASKS` in [`pkg/mimo/tasks/taskrunner.go`](../../pkg/mimo/tasks/taskrunner.go).
+
+## New Step Functions
+
+MIMO Step functions are similar to functions used in `pkg/cluster/install.go` but have additional information on the `Context` to prevent the explosion of struct members as seen in that package. Instead, the `GetTaskContext` function will return a `TaskContext` with various methods that can be used to retrieve information about the cluster, clients to perform actions in Azure, or Kubernetes clients to perform actions in the cluster.
+
+Steps with similar logical domains should live in the same file/package. Currently, `pkg/mimo/steps/cluster/` is the only package, but functionality specific to the cluster's Azure resources may be better in a package called `pkg/mimo/steps/azure/` to make navigation easier.
+
+Your base Action Step will look something like this:
+
+```go
+func DoSomething(ctx context.Context) error {
+	tc, err := mimo.GetTaskContext(ctx)
+	if err != nil {
+		return mimo.TerminalError(err)
+	}
+
+    return nil
+}
+```
+
+Like `pkg/cluster/`, you can also implement `Condition`s which allow you to wait for some state. However, MIMO's design is such that it should not sit around for long periods of time waiting for things which should already be the case -- for example, the API server not being up should instead be a usual Action which returns one of either `mimo.TerminalError` or `mimo.TransientError`.
+
+`TransientError`s will be retried, and do not indicate a permanent failure. This is a good fit for errors that are possibly because of timeouts, random momentary outages, or cosmic winds flipping all the bits on your NIC for a nanosecond. MIMO will retry a task (at least, a few times) whose steps return a `TransientError`.
+
+`TerminalError`s are used when there is no likelihood of automatic recovery. For example, if an API server is healthy and returning data, but it says that some essential OpenShift object that we require is missing, it is unlikely that object will return after one or many retries in a short period of time. These failures ought to require either manual intervention because they are unexpected or indicate that a cluster is unservicable. When a `TerminalError` is returned, it will cause the Task to hard fail and MIMO will not retry it.
+
+## Testing
+
+MIMO provides a fake `TaskContext`, created by `test/mimo/tasks.NewFakeTestContext`. This fake takes a number of mandatory items, such as an inner `Context` for cancellation, an `env.Interface`, a `*logrus.Entry`, and a stand-in clock for testing timing. Additional parts of the `TaskContext` used can be provided by `WithXXX` functions provided at the end of the instantiator, such as `WithClientHelper` to add a `ClientHelper` that is accessible on the `TaskContext`.
+
+Attempting to use additional parts of the `TaskContext` without providing them will cause a panic or an error to be returned, in both the fake and real `TaskContext`. This behaviour is intended to make it clearer when some dependency is required.
+
+## Assembling a Task
+
+Once you have your Steps, you can assemble them into a Task in [`pkg/mimo/steps/`](../../pkg/mimo/steps/). See existing Tasks for examples.
+
+## Assumptions MIMO Makes Of Your Code
+
+- Your Steps may be run more than once -- both if they are in a Task more than once, or because a Task has been retried. Your Step must be resilient to being reran from a partial run.
+- Steps should fail fast and not sit around unless they have caused something to happen. Right now, Tasks only have a 60 minute timeout total, so use it wisely.
+- Steps use the `TaskContext` interface to get clients, and should not build them itself. If a Task requires a new client, it should be implemented in `TaskContext` to ensure that it can be tested the same way as other used clients.

go.mod

@@ -79,6 +79,7 @@ require (
 	github.com/vincent-petithory/dataurl v1.0.0
 	go.uber.org/mock v0.4.0
 	golang.org/x/crypto v0.28.0
+	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/net v0.30.0
 	golang.org/x/oauth2 v0.21.0
 	golang.org/x/sync v0.8.0
@@ -259,7 +260,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd // indirect
-	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.25.0 // indirect