controller: add support for global persistent configuration

cmd/manager/main.go

@@ -17,28 +17,31 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 	"time"
 
+	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/csiaddons/v1alpha1"
+	replicationstoragev1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1"
+	controllers "github.com/csi-addons/kubernetes-csi-addons/controllers/csiaddons"
+	replicationController "github.com/csi-addons/kubernetes-csi-addons/controllers/replication.storage"
+	"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
+	"github.com/csi-addons/kubernetes-csi-addons/internal/util"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-
-	csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/csiaddons/v1alpha1"
-	replicationstoragev1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1"
-	controllers "github.com/csi-addons/kubernetes-csi-addons/controllers/csiaddons"
-	replicationController "github.com/csi-addons/kubernetes-csi-addons/controllers/replication.storage"
-	"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -60,19 +63,22 @@ func init() {
 }
 
 func main() {
-	var metricsAddr string
-	var enableLeaderElection bool
-	var probeAddr string
-	var reclaimSpaceTimeout time.Duration
-	var maxConcurrentReconciles int
-	var enableAdmissionWebhooks bool
+	var (
+		metricsAddr             string
+		probeAddr               string
+		enableLeaderElection    bool
+		enableAdmissionWebhooks bool
+		ctx                     = context.Background()
+		cfg                     = util.NewConfig()
+	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.DurationVar(&reclaimSpaceTimeout, "reclaim-space-timeout", defaultTimeout, "Timeout for reclaimspace operation")
-	flag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", 100, "Maximum number of concurrent reconciles")
+	flag.DurationVar(&cfg.ReclaimSpaceTimeout, "reclaim-space-timeout", cfg.ReclaimSpaceTimeout, "Timeout for reclaimspace operation")
+	flag.IntVar(&cfg.MaxConcurrentReconciles, "max-concurrent-reconciles", cfg.MaxConcurrentReconciles, "Maximum number of concurrent reconciles")
+	flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed")
 	flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", true, "Enable the admission webhooks")
 	opts := zap.Options{
 		Development: true,
@@ -83,7 +89,20 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	kubeConfig := ctrl.GetConfigOrDie()
+	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		setupLog.Error(err, "unable to create client")
+		os.Exit(1)
+	}
+
+	err = cfg.ReadConfigMap(ctx, kubeClient)
+	if err != nil {
+		setupLog.Error(err, "unable to read configmap")
+		os.Exit(1)
+	}
+
+	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
 		Port:                   9443,
@@ -99,7 +118,7 @@ func main() {
 	connPool := connection.NewConnectionPool()
 
 	ctrlOptions := controller.Options{
-		MaxConcurrentReconciles: maxConcurrentReconciles,
+		MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
 	}
 	if err = (&controllers.CSIAddonsNodeReconciler{
 		Client:   mgr.GetClient(),
@@ -114,7 +133,7 @@ func main() {
 		Client:   mgr.GetClient(),
 		Scheme:   mgr.GetScheme(),
 		ConnPool: connPool,
-		Timeout:  reclaimSpaceTimeout,
+		Timeout:  cfg.ReclaimSpaceTimeout,
 	}).SetupWithManager(mgr, ctrlOptions); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ReclaimSpaceJob")
 		os.Exit(1)

config/manager/manager.yaml

@@ -26,6 +26,11 @@ spec:
         - --leader-elect
         image: controller:latest
         name: manager
+        env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+               fieldPath: metadata.namespace
         securityContext:
           allowPrivilegeEscalation: false
         livenessProbe:

config/manager/manager_auth_proxy_patch.yaml

@@ -31,6 +31,7 @@ spec:
             memory: 64Mi
       - name: manager
         args:
+        - "--namespace=$(POD_NAMESPACE)"
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"

deploy/controller/csi-addons-config.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: csi-addons-config
+  # replace the namespace with the namespace where the operator is deployed.
+  namespace: csi-addons-system
+data:
+  "reclaim-space-timeout": "3m"
+  "max-concurrent-reconciles": "100"

deploy/controller/install-all-in-one.yaml

@@ -1342,12 +1342,18 @@ spec:
     spec:
       containers:
       - args:
+        - --namespace=$(POD_NAMESPACE)
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --enable-admission-webhooks=true
         command:
         - /manager
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         image: quay.io/csiaddons/k8s-controller:latest
         livenessProbe:
           httpGet:

deploy/controller/setup-controller.yaml

@@ -56,12 +56,18 @@ spec:
             cpu: 10m
             memory: 64Mi
       - args:
+        - --namespace=$(POD_NAMESPACE)
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --enable-admission-webhooks=false
         command:
         - /manager
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         image: quay.io/csiaddons/k8s-controller:latest
         livenessProbe:
           httpGet:

docs/csi-addons-config.md

@@ -0,0 +1,14 @@
+# CSI-Addons Operator Configuration
+
+CSI-Addons Operator can consume configuration from a ConfigMap named `csi-addons-config`
+in the same namespace as the operator. This enables configuration of the operator to persist across
+upgrades. The ConfigMap can support the following configuration options:
+
+| Option                        | Default value   | Description                                   |
+| ----------------------------- | --------------- | --------------------------------------------- |
+| `reclaim-space-timeout`       | `"3m"`          | Timeout for reclaimspace operation            |
+| `max-concurrent-reconciles`   | `"100"`         | Maximum number of concurrent reconciles       |
+
+[`csi-addons-config` ConfigMap](../deploy/controller/csi-addons-config.yaml) is provided as an example.
+
+> Note: The operator pod needs to be restarted for any change in configuration to take effect.

docs/deploy-controller.md

@@ -15,6 +15,8 @@ The CSI-Addons Controller can be deployed by different ways:
 | `--max-concurrent-reconciles` | 100             | Maximum number of concurrent reconciles       |
 | `--enable-admission-webhooks` | `true`          | Enable the admission webhooks                 |
 
+> Note: Some of the above configuration options can also be configured using [`"csi-addons-config"` configmap](./csi-addons-config.md).
+
 ## Installation for versioned deployments
 
 The CSI-Addons Controller can also be installed  using the yaml files in `deploy/controller`.

internal/util/config.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2023 The Kubernetes-CSI-Addons Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Config holds the configuration options that
+// can be overrrided via a config file.
+type Config struct {
+	Namespace               string
+	ReclaimSpaceTimeout     time.Duration
+	MaxConcurrentReconciles int
+}
+
+const (
+	csiAddonsConfigMapName         = "csi-addons-config"
+	ReclaimSpaceTimeoutKey         = "reclaim-space-timeout"
+	MaxConcurrentReconcilesKey     = "max-concurrent-reconciles"
+	defaultNamespace               = "csi-addons-system"
+	defaultMaxConcurrentReconciles = 100
+	defaultReclaimSpaceTimeout     = time.Minute * 3
+)
+
+// NewConfig returns a new Config object with default values.
+func NewConfig() Config {
+	return Config{
+		Namespace:               defaultNamespace,
+		ReclaimSpaceTimeout:     defaultReclaimSpaceTimeout,
+		MaxConcurrentReconciles: defaultMaxConcurrentReconciles,
+	}
+}
+
+// ReadConfigFile fetches the config file and updates the
+// config options with values read from the config map.
+func (cfg *Config) ReadConfigMap(ctx context.Context, kubeClient *kubernetes.Clientset) error {
+	cm, err := kubeClient.CoreV1().ConfigMaps(cfg.Namespace).
+		Get(ctx, csiAddonsConfigMapName, metav1.GetOptions{})
+	if kerrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to get configmap %q: %w", csiAddonsConfigMapName, err)
+	}
+
+	return cfg.readConfig(cm.Data)
+}
+
+// readConfigFile reads the config dataMap and updates the
+// config options.
+func (cfg *Config) readConfig(dataMap map[string]string) error {
+	for key, val := range dataMap {
+		switch key {
+		case ReclaimSpaceTimeoutKey:
+			timeout, err := time.ParseDuration(val)
+			if err != nil {
+				return fmt.Errorf("failed to parse key %q value %q as duration: %w",
+					ReclaimSpaceTimeoutKey, val, err)
+			}
+			cfg.ReclaimSpaceTimeout = timeout
+
+		case MaxConcurrentReconcilesKey:
+			maxConcurrentReconciles, err := strconv.Atoi(val)
+			if err != nil {
+				return fmt.Errorf("failed to parse key %q value %q as int: %w",
+					MaxConcurrentReconcilesKey, val, err)
+			}
+			cfg.MaxConcurrentReconciles = maxConcurrentReconciles
+
+		default:
+			return fmt.Errorf("unknown config key %q", key)
+		}
+	}
+
+	return nil
+}