Prepare release 1.8.1

Makefile

@@ -31,7 +31,7 @@ endif
 # Dependencies to fetch through `go`
 CONTROLLER_GEN_PKG?=sigs.k8s.io/controller-tools/cmd/[email protected]
 CONTROLLER_GEN=$(GOBIN)/controller-gen
-KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/[email protected]
+KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/[email protected]
 KUSTOMIZE=$(GOBIN)/kustomize
 GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/[email protected]
 GOLANGCI_LINT=$(GOBIN)/golangci-lint
@@ -190,7 +190,7 @@ rebuild-operator: container-build container-push-if-remote deploy bounce
 bounce:
 	kubectl $(KUBECTL_ARGS) delete pod -l app=fdb-kubernetes-operator-controller-manager
 
-samples: ${SAMPLES}
+samples: kustomize ${SAMPLES}
 
 config/samples/deployment.yaml: config/deployment/*.yaml
 	kustomize build ./config/deployment > $@

api/v1beta2/foundationdb_errors.go

@@ -0,0 +1,34 @@
+/*
+ * foundationdb_errors.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package v1beta2
+
+import "fmt"
+
+// TimeoutError represents a timeout for either the fdb client library or fdbcli
+// +k8s:deepcopy-gen=false
+type TimeoutError struct {
+	Err error
+}
+
+// Error returns the error message of the internal timeout error.
+func (timeoutErr TimeoutError) Error() string {
+	return fmt.Sprintf("fdb timeout: %s", timeoutErr.Err.Error())
+}

api/v1beta2/foundationdb_status.go

@@ -89,6 +89,10 @@ type FoundationDBStatusClusterInfo struct {
 	// FaultTolerance provides information about the fault tolerance status
 	// of the cluster.
 	FaultTolerance FaultTolerance `json:"fault_tolerance,omitempty"`
+
+	// IncompatibleConnections provides information about processes that try to connect to the cluster with an
+	// incompatible version.
+	IncompatibleConnections []string `json:"incompatible_connections,omitempty"`
 }
 
 // FaultTolerance provides information about the fault tolerance status

api/v1beta2/foundationdb_status_test.go

@@ -70,6 +70,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					DatabaseStatus: FoundationDBStatusClientDBStatus{Available: true, Healthy: true},
 				},
 				Cluster: FoundationDBStatusClusterInfo{
+					IncompatibleConnections: []string{},
 					FaultTolerance: FaultTolerance{
 						MaxZoneFailuresWithoutLosingAvailability: 1,
 						MaxZoneFailuresWithoutLosingData:         1,
@@ -459,6 +460,7 @@ var _ = Describe("FoundationDBStatus", func() {
 
 	When("parsing the status json with a 7.1.0-rc1 cluster", func() {
 		status := FoundationDBStatusClusterInfo{
+			IncompatibleConnections: []string{},
 			DatabaseConfiguration: DatabaseConfiguration{
 				RedundancyMode:  "double",
 				StorageEngine:   StorageEngineSSD2,

api/v1beta2/foundationdbcluster_types.go

@@ -1159,7 +1159,12 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if len(processGroup.ProcessGroupConditions) > 0 && !processGroup.IsMarkedForRemoval() {
-			logger.Info("Has unhealthy process group", "processGroupID", processGroup.ProcessGroupID, "state", "HasUnhealthyProcess")
+			conditions := make([]ProcessGroupConditionType, 0, len(processGroup.ProcessGroupConditions))
+			for _, condition := range processGroup.ProcessGroupConditions {
+				conditions = append(conditions, condition.ProcessGroupConditionType)
+			}
+
+			logger.Info("Has unhealthy process group", "processGroupID", processGroup.ProcessGroupID, "state", "HasUnhealthyProcess", "conditions", conditions)
 			cluster.Status.Generations.HasUnhealthyProcess = cluster.ObjectMeta.Generation
 			reconciled = false
 		}

charts/fdb-operator/Chart.yaml

@@ -21,7 +21,7 @@ version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: v1.8.0
+appVersion: v1.8.1
 
 maintainers:
   - name: "foundationdb-ci"

charts/fdb-operator/values.yaml

@@ -1,7 +1,7 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.8.0
+  tag: v1.8.1
   pullPolicy: IfNotPresent
 
 initContainers:

config/deployment/manager.yaml

@@ -83,7 +83,7 @@ spec:
       containers:
       - command:
         - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.8.0
+        image: foundationdb/fdb-kubernetes-operator:v1.8.1
         name: manager
         env:
           - name: WATCH_NAMESPACE

config/samples/deployment.yaml

@@ -150,7 +150,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.8.0
+        image: foundationdb/fdb-kubernetes-operator:v1.8.1
         name: manager
         ports:
         - containerPort: 8080

controllers/admin_client_mock.go

@@ -463,6 +463,16 @@ func (client *mockAdminClient) KillProcesses(addresses []fdbv1beta2.ProcessAddre
 	}
 	adminClientMutex.Unlock()
 
+	if client.Cluster.Status.RunningVersion != client.Cluster.Spec.Version {
+		// We have to do this in the mock client, in the real world the tryConnectionOptions in update_status,
+		// will update the version.
+		client.Cluster.Status.RunningVersion = client.Cluster.Spec.Version
+		err := client.KubeClient.Status().Update(context.TODO(), client.Cluster)
+		if err != nil {
+			return err
+		}
+	}
+
 	client.UnfreezeStatus()
 	return nil
 }

controllers/bounce_processes.go

@@ -58,7 +58,7 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 	minimumUptime := math.Inf(1)
 	addressMap := make(map[string][]fdbv1beta2.ProcessAddress, len(status.Cluster.Processes))
 	for _, process := range status.Cluster.Processes {
-		addressMap[process.Locality["instance_id"]] = append(addressMap[process.Locality["instance_id"]], process.Address)
+		addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]] = append(addressMap[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]], process.Address)
 
 		if process.UptimeSeconds < minimumUptime {
 			minimumUptime = process.UptimeSeconds
@@ -68,6 +68,7 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 	processesToBounce := fdbv1beta2.FilterByConditions(cluster.Status.ProcessGroups, map[fdbv1beta2.ProcessGroupConditionType]bool{
 		fdbv1beta2.IncorrectCommandLine: true,
 		fdbv1beta2.IncorrectPodSpec:     false,
+		fdbv1beta2.SidecarUnreachable:   false, // ignore all Process groups that are not reachable and therefore will not get any config map updates.
 	}, true)
 
 	addresses := make([]fdbv1beta2.ProcessAddress, 0, len(processesToBounce))
@@ -172,7 +173,7 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 
 		if useLocks && upgrading {
 			var req *requeue
-			addresses, req = getAddressesForUpgrade(r, adminClient, lockClient, cluster, version)
+			addresses, req = getAddressesForUpgrade(r, status, lockClient, cluster, version)
 			if req != nil {
 				return req
 			}
@@ -187,13 +188,13 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		if err != nil {
 			return &requeue{curError: err}
 		}
-	}
 
-	if upgrading {
-		cluster.Status.RunningVersion = cluster.Spec.Version
-		err = r.Status().Update(ctx, cluster)
-		if err != nil {
-			return &requeue{curError: err}
+		// If the cluster was upgraded we will requeue and let the update_status command set the correct version.
+		// Updating the version in this method has the drawback that we upgrade the version independent of the success
+		// of the kill command. The kill command is not reliable, which means that some kill request might not be
+		// delivered and the return value will still not contain any error.
+		if upgrading {
+			return &requeue{message: "fetch latest status after upgrade"}
 		}
 	}
 
@@ -202,28 +203,22 @@ func (bounceProcesses) reconcile(ctx context.Context, r *FoundationDBClusterReco
 
 // getAddressesForUpgrade checks that all processes in a cluster are ready to be
 // upgraded and returns the full list of addresses.
-func getAddressesForUpgrade(r *FoundationDBClusterReconciler, adminClient fdbadminclient.AdminClient, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
+func getAddressesForUpgrade(r *FoundationDBClusterReconciler, databaseStatus *fdbv1beta2.FoundationDBStatus, lockClient fdbadminclient.LockClient, cluster *fdbv1beta2.FoundationDBCluster, version fdbv1beta2.Version) ([]fdbv1beta2.ProcessAddress, *requeue) {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "bounceProcesses")
 	pendingUpgrades, err := lockClient.GetPendingUpgrades(version)
 	if err != nil {
 		return nil, &requeue{curError: err}
 	}
 
-	databaseStatus, err := adminClient.GetStatus()
-	if err != nil {
-		return nil, &requeue{curError: err}
-	}
-
 	if !databaseStatus.Client.DatabaseStatus.Available {
-		logger.Info("Deferring upgrade until database is available")
 		r.Recorder.Event(cluster, corev1.EventTypeNormal, "UpgradeRequeued", "Database is unavailable")
 		return nil, &requeue{message: "Deferring upgrade until database is available"}
 	}
 
 	notReadyProcesses := make([]string, 0)
 	addresses := make([]fdbv1beta2.ProcessAddress, 0, len(databaseStatus.Cluster.Processes))
 	for _, process := range databaseStatus.Cluster.Processes {
-		processID := process.Locality["instance_id"]
+		processID := process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]
 		if process.Version == version.String() {
 			continue
 		}

controllers/bounce_processes_test.go

@@ -166,8 +166,8 @@ var _ = Describe("bounceProcesses", func() {
 			}
 		})
 
-		It("should not requeue", func() {
-			Expect(requeue).To(BeNil())
+		It("should requeue", func() {
+			Expect(requeue).NotTo(BeNil())
 		})
 
 		It("should kill all the processes", func() {
@@ -233,8 +233,8 @@ var _ = Describe("bounceProcesses", func() {
 					Expect(err).NotTo(HaveOccurred())
 				})
 
-				It("should not requeue", func() {
-					Expect(requeue).To(BeNil())
+				It("should requeue", func() {
+					Expect(requeue).NotTo(BeNil())
 				})
 
 				It("should kill all the processes", func() {
@@ -272,8 +272,8 @@ var _ = Describe("bounceProcesses", func() {
 					cluster.Spec.LockOptions.DisableLocks = &disabled
 				})
 
-				It("should not requeue", func() {
-					Expect(requeue).To(BeNil())
+				It("should requeue", func() {
+					Expect(requeue).NotTo(BeNil())
 				})
 
 				It("should kill all the processes", func() {

controllers/cluster_controller.go

@@ -30,10 +30,9 @@ import (
 	"sort"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient"
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
@@ -53,15 +52,16 @@ import (
 // FoundationDBClusterReconciler reconciles a FoundationDBCluster object
 type FoundationDBClusterReconciler struct {
 	client.Client
-	Recorder               record.EventRecorder
-	Log                    logr.Logger
-	InSimulation           bool
-	PodLifecycleManager    podmanager.PodLifecycleManager
-	PodClientProvider      func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
-	DatabaseClientProvider DatabaseClientProvider
-	DeprecationOptions     internal.DeprecationOptions
-	GetTimeout             time.Duration
-	PostTimeout            time.Duration
+	Recorder                           record.EventRecorder
+	Log                                logr.Logger
+	InSimulation                       bool
+	EnableRestartIncompatibleProcesses bool
+	PodLifecycleManager                podmanager.PodLifecycleManager
+	PodClientProvider                  func(*fdbv1beta2.FoundationDBCluster, *corev1.Pod) (podclient.FdbPodClient, error)
+	DatabaseClientProvider             DatabaseClientProvider
+	DeprecationOptions                 internal.DeprecationOptions
+	GetTimeout                         time.Duration
+	PostTimeout                        time.Duration
 }
 
 // NewFoundationDBClusterReconciler creates a new FoundationDBClusterReconciler with defaults.
@@ -139,6 +139,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		addPVCs{},
 		addPods{},
 		generateInitialClusterFile{},
+		removeIncompatibleProcesses{},
 		updateSidecarVersions{},
 		updatePodConfig{},
 		updateLabels{},

controllers/exclude_processes.go

@@ -42,6 +42,7 @@ type excludeProcesses struct{}
 
 // reconcile runs the reconciler's work.
 func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "excludeProcesses")
 	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r)
 	if err != nil {
 		return &requeue{curError: err}
@@ -62,7 +63,7 @@ func (e excludeProcesses) reconcile(_ context.Context, r *FoundationDBClusterRec
 		if err != nil {
 			return &requeue{curError: err}
 		}
-		log.Info("current exclusions", "ex", exclusions)
+		logger.Info("current exclusions", "ex", exclusions)
 		fdbProcessesToExclude, processClassesToExclude = getProcessesToExclude(exclusions, cluster, removalCount)
 	}