Skip to content

Commit

Permalink
controller: add support for global persistent configuration
Browse files Browse the repository at this point in the history
This commits adds support for configuration options consumed
through a optional configmap.
This will enable users to create a configmap to supply options
that are persisted throughout upgrades.
Current support parameters are `"reclaim-space-timeout"`
and `"max-concurrent-reconciles"`.

Signed-off-by: Rakshith R <[email protected]>
  • Loading branch information
Rakshith-R committed Jun 12, 2023
1 parent 69a0a22 commit 1e9e762
Show file tree
Hide file tree
Showing 10 changed files with 314 additions and 17 deletions.
53 changes: 36 additions & 17 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,31 @@ limitations under the License.
package main

import (
"context"
"flag"
"os"
"time"

csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/csiaddons/v1alpha1"
replicationstoragev1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1"
controllers "github.com/csi-addons/kubernetes-csi-addons/controllers/csiaddons"
replicationController "github.com/csi-addons/kubernetes-csi-addons/controllers/replication.storage"
"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
"github.com/csi-addons/kubernetes-csi-addons/internal/util"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"

"go.uber.org/zap/zapcore"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/csiaddons/v1alpha1"
replicationstoragev1alpha1 "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1"
controllers "github.com/csi-addons/kubernetes-csi-addons/controllers/csiaddons"
replicationController "github.com/csi-addons/kubernetes-csi-addons/controllers/replication.storage"
"github.com/csi-addons/kubernetes-csi-addons/internal/connection"
//+kubebuilder:scaffold:imports
)

Expand All @@ -60,19 +63,22 @@ func init() {
}

func main() {
var metricsAddr string
var enableLeaderElection bool
var probeAddr string
var reclaimSpaceTimeout time.Duration
var maxConcurrentReconciles int
var enableAdmissionWebhooks bool
var (
metricsAddr string
probeAddr string
enableLeaderElection bool
enableAdmissionWebhooks bool
ctx = context.Background()
cfg = util.NewConfig()
)
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
flag.DurationVar(&reclaimSpaceTimeout, "reclaim-space-timeout", defaultTimeout, "Timeout for reclaimspace operation")
flag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", 100, "Maximum number of concurrent reconciles")
flag.DurationVar(&cfg.ReclaimSpaceTimeout, "reclaim-space-timeout", cfg.ReclaimSpaceTimeout, "Timeout for reclaimspace operation")
flag.IntVar(&cfg.MaxConcurrentReconciles, "max-concurrent-reconciles", cfg.MaxConcurrentReconciles, "Maximum number of concurrent reconciles")
flag.StringVar(&cfg.Namespace, "namespace", cfg.Namespace, "Namespace where the CSIAddons pod is deployed")
flag.BoolVar(&enableAdmissionWebhooks, "enable-admission-webhooks", true, "Enable the admission webhooks")
opts := zap.Options{
Development: true,
Expand All @@ -83,7 +89,20 @@ func main() {

ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
kubeConfig := ctrl.GetConfigOrDie()
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
if err != nil {
setupLog.Error(err, "unable to create client")
os.Exit(1)
}

err = cfg.ReadConfigMap(ctx, kubeClient)
if err != nil {
setupLog.Error(err, "unable to read configmap")
os.Exit(1)
}

mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
Scheme: scheme,
MetricsBindAddress: metricsAddr,
Port: 9443,
Expand All @@ -99,7 +118,7 @@ func main() {
connPool := connection.NewConnectionPool()

ctrlOptions := controller.Options{
MaxConcurrentReconciles: maxConcurrentReconciles,
MaxConcurrentReconciles: cfg.MaxConcurrentReconciles,
}
if err = (&controllers.CSIAddonsNodeReconciler{
Client: mgr.GetClient(),
Expand All @@ -114,7 +133,7 @@ func main() {
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ConnPool: connPool,
Timeout: reclaimSpaceTimeout,
Timeout: cfg.ReclaimSpaceTimeout,
}).SetupWithManager(mgr, ctrlOptions); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ReclaimSpaceJob")
os.Exit(1)
Expand Down
5 changes: 5 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ spec:
- --leader-elect
image: controller:latest
name: manager
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
securityContext:
allowPrivilegeEscalation: false
livenessProbe:
Expand Down
1 change: 1 addition & 0 deletions config/manager/manager_auth_proxy_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ spec:
memory: 64Mi
- name: manager
args:
- "--namespace=$(POD_NAMESPACE)"
- "--health-probe-bind-address=:8081"
- "--metrics-bind-address=127.0.0.1:8080"
- "--leader-elect"
Expand Down
9 changes: 9 additions & 0 deletions deploy/controller/csi-addons-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: csi-addons-config
# replace the namespace with the namespace where the operator is deployed.
namespace: csi-addons-system
data:
"reclaim-space-timeout": "3m"
"max-concurrent-reconciles": "100"
6 changes: 6 additions & 0 deletions deploy/controller/install-all-in-one.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1342,12 +1342,18 @@ spec:
spec:
containers:
- args:
- --namespace=$(POD_NAMESPACE)
- --health-probe-bind-address=:8081
- --metrics-bind-address=127.0.0.1:8080
- --leader-elect
- --enable-admission-webhooks=true
command:
- /manager
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: quay.io/csiaddons/k8s-controller:latest
livenessProbe:
httpGet:
Expand Down
6 changes: 6 additions & 0 deletions deploy/controller/setup-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,18 @@ spec:
cpu: 10m
memory: 64Mi
- args:
- --namespace=$(POD_NAMESPACE)
- --health-probe-bind-address=:8081
- --metrics-bind-address=127.0.0.1:8080
- --leader-elect
- --enable-admission-webhooks=false
command:
- /manager
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: quay.io/csiaddons/k8s-controller:latest
livenessProbe:
httpGet:
Expand Down
14 changes: 14 additions & 0 deletions docs/csi-addons-config.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# CSI-Addons Operator Configuration

CSI-Addons Operator can consume configuration from a ConfigMap named `csi-addons-config`
in the same namespace as the operator. This enables configuration of the operator to persist across
upgrades. The ConfigMap can support the following configuration options:

| Option | Default value | Description |
| ----------------------------- | --------------- | --------------------------------------------- |
| `reclaim-space-timeout` | `"3m"` | Timeout for reclaimspace operation |
| `max-concurrent-reconciles` | `"100"` | Maximum number of concurrent reconciles |

[`csi-addons-config` ConfigMap](../deploy/controller/csi-addons-config.yaml) is provided as an example.

> Note: The operator pod needs to be restarted for any change in configuration to take effect.
2 changes: 2 additions & 0 deletions docs/deploy-controller.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ The CSI-Addons Controller can be deployed by different ways:
| `--max-concurrent-reconciles` | 100 | Maximum number of concurrent reconciles |
| `--enable-admission-webhooks` | `true` | Enable the admission webhooks |

> Note: Some of the above configuration options can also be configured using [`"csi-addons-config"` configmap](./csi-addons-config.md).
## Installation for versioned deployments

The CSI-Addons Controller can also be installed using the yaml files in `deploy/controller`.
Expand Down
98 changes: 98 additions & 0 deletions internal/util/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
Copyright 2023 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
"context"
"fmt"
"strconv"
"time"

kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

// Config holds the configuration options that
// can be overrrided via a config file.
type Config struct {
Namespace string
ReclaimSpaceTimeout time.Duration
MaxConcurrentReconciles int
}

const (
csiAddonsConfigMapName = "csi-addons-config"
ReclaimSpaceTimeoutKey = "reclaim-space-timeout"
MaxConcurrentReconcilesKey = "max-concurrent-reconciles"
defaultNamespace = "csi-addons-system"
defaultMaxConcurrentReconciles = 100
defaultReclaimSpaceTimeout = time.Minute * 3
)

// NewConfig returns a new Config object with default values.
func NewConfig() Config {
return Config{
Namespace: defaultNamespace,
ReclaimSpaceTimeout: defaultReclaimSpaceTimeout,
MaxConcurrentReconciles: defaultMaxConcurrentReconciles,
}
}

// ReadConfigFile fetches the config file and updates the
// config options with values read from the config map.
func (cfg *Config) ReadConfigMap(ctx context.Context, kubeClient *kubernetes.Clientset) error {
cm, err := kubeClient.CoreV1().ConfigMaps(cfg.Namespace).
Get(ctx, csiAddonsConfigMapName, metav1.GetOptions{})
if kerrors.IsNotFound(err) {
return nil
}
if err != nil {
return fmt.Errorf("failed to get configmap %q: %w", csiAddonsConfigMapName, err)
}

return cfg.readConfig(cm.Data)
}

// readConfigFile reads the config dataMap and updates the
// config options.
func (cfg *Config) readConfig(dataMap map[string]string) error {
for key, val := range dataMap {
switch key {
case ReclaimSpaceTimeoutKey:
timeout, err := time.ParseDuration(val)
if err != nil {
return fmt.Errorf("failed to parse key %q value %q as duration: %w",
ReclaimSpaceTimeoutKey, val, err)
}
cfg.ReclaimSpaceTimeout = timeout

case MaxConcurrentReconcilesKey:
maxConcurrentReconciles, err := strconv.Atoi(val)
if err != nil {
return fmt.Errorf("failed to parse key %q value %q as int: %w",
MaxConcurrentReconcilesKey, val, err)
}
cfg.MaxConcurrentReconciles = maxConcurrentReconciles

default:
return fmt.Errorf("unknown config key %q", key)
}
}

return nil
}
Loading

0 comments on commit 1e9e762

Please sign in to comment.