Skip to content

Commit

Permalink
Merge pull request #370 from sthaha/feat-redfish-rbac
Browse files Browse the repository at this point in the history
feat(redfish): reduce rbac permission given to operator and kepler for handling secrets
  • Loading branch information
vimalk78 authored Mar 11, 2024
2 parents c46fb30 + e384c7a commit ef88a4d
Show file tree
Hide file tree
Showing 9 changed files with 140 additions and 88 deletions.
11 changes: 8 additions & 3 deletions bundle/manifests/kepler-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ metadata:
capabilities: Basic Install
categories: Monitoring
containerImage: quay.io/sustainable_computing_io/kepler-operator:0.10.0
createdAt: "2024-03-01T19:29:40Z"
createdAt: "2024-03-11T03:26:22Z"
description: 'Deploys and Manages Kepler on Kubernetes '
operators.operatorframework.io/builder: operator-sdk-v1.27.0
operators.operatorframework.io/internal-objects: |-
Expand Down Expand Up @@ -117,7 +117,6 @@ spec:
resources:
- daemonsets
- deployments
- secrets
verbs:
- create
- delete
Expand Down Expand Up @@ -156,11 +155,17 @@ spec:
- nodes/metrics
- nodes/proxy
- nodes/stats
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- watch
- apiGroups:
- kepler.system.sustainable.computing.io
resources:
Expand Down
9 changes: 7 additions & 2 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ rules:
resources:
- daemonsets
- deployments
- secrets
verbs:
- create
- delete
Expand Down Expand Up @@ -48,11 +47,17 @@ rules:
- nodes/metrics
- nodes/proxy
- nodes/stats
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- watch
- apiGroups:
- kepler.system.sustainable.computing.io
resources:
Expand Down
31 changes: 19 additions & 12 deletions pkg/components/exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
"k8s.io/utils/ptr"
)

const (
Expand All @@ -45,11 +45,14 @@ const (
overviewDashboardName = "power-monitoring-overview"
nsInfoDashboardName = "power-monitoring-by-ns"
DashboardNs = "openshift-config-managed"
REDFISH_ARGS = "-redfish-cred-file-path=/etc/redfish/redfish.csv"
REDFISH_CSV = "redfish.csv"
REDFISH_ANNOTATION = "kepler.system.sustainable.computing.io/redfish-secret-ref"

IdxKeplerContainer = 0
RedfishArgs = "-redfish-cred-file-path=/etc/redfish/redfish.csv"
RedfishCSV = "redfish.csv"
RedfishSecretAnnotation = "kepler.system.sustainable.computing.io/redfish-secret-ref"
)

const (
KeplerContainerIndex k8s.ContainerIndex = 0
)

var (
Expand Down Expand Up @@ -138,15 +141,19 @@ func NewDaemonSet(detail components.Detail, k *v1alpha1.KeplerInternal) *appsv1.
}

func MountRedfishSecretToDaemonSet(ds *appsv1.DaemonSet, secret *corev1.Secret) {
spec := ds.Spec.Template.Spec
spec.Containers[IdxKeplerContainer].Command = append(spec.Containers[IdxKeplerContainer].Command, REDFISH_ARGS)
spec.Containers[IdxKeplerContainer].VolumeMounts = append(spec.Containers[IdxKeplerContainer].VolumeMounts,
corev1.VolumeMount{Name: "redfish-cred", MountPath: "/etc/redfish", ReadOnly: true})
spec := &ds.Spec.Template.Spec
keplerContainer := &spec.Containers[KeplerContainerIndex]
keplerContainer.Command = append(keplerContainer.Command, RedfishArgs)
keplerContainer.VolumeMounts = append(keplerContainer.VolumeMounts,
corev1.VolumeMount{Name: "redfish-cred", MountPath: "/etc/redfish", ReadOnly: true},
)
spec.Volumes = append(spec.Volumes,
k8s.VolumeFromSecret("redfish-cred", secret.ObjectMeta.Name))
ds.Spec.Template.Spec = spec

// NOTE: annotating the Pods with the secret's resource version
// forces pods to be reployed if the secret chanage
ds.Spec.Template.Annotations = map[string]string{
REDFISH_ANNOTATION: secret.ResourceVersion,
RedfishSecretAnnotation: secret.ResourceVersion,
}
}

Expand Down Expand Up @@ -606,7 +613,7 @@ func newExporterContainer(kiName, dsName string, deployment v1alpha1.InternalExp
bindAddress := "0.0.0.0:" + strconv.Itoa(int(deployment.Port))
return corev1.Container{
Name: dsName,
SecurityContext: &corev1.SecurityContext{Privileged: pointer.Bool(true)},
SecurityContext: &corev1.SecurityContext{Privileged: ptr.To(true)},
Image: deployment.Image,
Command: []string{
"/usr/bin/kepler",
Expand Down
4 changes: 2 additions & 2 deletions pkg/components/exporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,10 @@ func TestDaemonSet(t *testing.T) {
actual_hostPID := k8s.HostPIDFromDS(ds)
assert.Equal(t, actual_hostPID, tc.hostPID)

actual_exporterCommand := k8s.CommandFromDS(ds, IdxKeplerContainer)
actual_exporterCommand := k8s.CommandFromDS(ds, KeplerContainerIndex)
assert.Equal(t, actual_exporterCommand, tc.exporterCommand)

actual_volumeMounts := k8s.VolumeMountsFromDS(ds, IdxKeplerContainer)
actual_volumeMounts := k8s.VolumeMountsFromDS(ds, KeplerContainerIndex)
assert.Equal(t, actual_volumeMounts, tc.volumeMounts)

actual_Volumes := k8s.VolumesFromDS(ds)
Expand Down
5 changes: 3 additions & 2 deletions pkg/controllers/kepler_internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,13 @@ type KeplerInternalReconciler struct {
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=*,verbs=*

// RBAC for running Kepler exporter
//+kubebuilder:rbac:groups=apps,resources=daemonsets;deployments;secrets,verbs=list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=daemonsets;deployments,verbs=list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=secrets,verbs=list;watch
//+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=list;watch;create;update;patch;delete;use
//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=list;watch;create;update;patch;delete

// RBAC required by Kepler exporter
//+kubebuilder:rbac:groups=core,resources=nodes/metrics;nodes/proxy;nodes/stats;secrets,verbs=get;list;watch
//+kubebuilder:rbac:groups=core,resources=nodes/metrics;nodes/proxy;nodes/stats,verbs=get;list;watch

// SetupWithManager sets up the controller with the Manager.
func (r *KeplerInternalReconciler) SetupWithManager(mgr ctrl.Manager) error {
Expand Down
85 changes: 85 additions & 0 deletions pkg/reconciler/kepler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconciler

import (
"context"
"fmt"
"strconv"

"github.com/sustainable.computing.io/kepler-operator/pkg/api/v1alpha1"
"github.com/sustainable.computing.io/kepler-operator/pkg/components/exporter"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type KeplerDaemonSetReconciler struct {
Ki v1alpha1.KeplerInternal
Ds *appsv1.DaemonSet
}

func (r KeplerDaemonSetReconciler) Reconcile(ctx context.Context, cli client.Client, s *runtime.Scheme) Result {

secretRef := r.Ki.Spec.Exporter.Redfish.SecretRef
secret, err := r.getRedfishSecret(ctx, cli, secretRef)

if err != nil {
return Result{Action: Stop, Error: fmt.Errorf("Error occurred while getting Redfish secret %w", err)}
}

if secret == nil {
return Result{
Action: Stop,
Error: fmt.Errorf("Redfish secret %q configured, but not found in %q namespace", secretRef, r.Ki.Namespace()),
}
}
if _, ok := secret.Data[exporter.RedfishCSV]; !ok {
return Result{Action: Stop, Error: fmt.Errorf("Redfish secret is missing %q key", exporter.RedfishCSV)}
}

exporter.MountRedfishSecretToDaemonSet(r.Ds, secret)

return Updater{Owner: &r.Ki, Resource: r.Ds}.Reconcile(ctx, cli, s)
}

func (r KeplerDaemonSetReconciler) getRedfishSecret(ctx context.Context, cli client.Client, secretName string) (*corev1.Secret, error) {
ns := r.Ki.Spec.Exporter.Deployment.Namespace
redfishSecret := corev1.Secret{}
if err := cli.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &redfishSecret); err != nil {
return nil, client.IgnoreNotFound(err)
}
return &redfishSecret, nil
}

type KeplerConfigMapReconciler struct {
Ki v1alpha1.KeplerInternal
Cfm *corev1.ConfigMap
}

func (r KeplerConfigMapReconciler) Reconcile(ctx context.Context, cli client.Client, s *runtime.Scheme) Result {
rf := r.Ki.Spec.Exporter.Redfish
zero := metav1.Duration{}
if rf.ProbeInterval != zero {
r.Cfm.Data["REDFISH_PROBE_INTERVAL_IN_SECONDS"] = fmt.Sprintf("%f", rf.ProbeInterval.Duration.Seconds())
}
r.Cfm.Data["REDFISH_SKIP_SSL_VERIFY"] = strconv.FormatBool(rf.SkipSSLVerify)
return Updater{Owner: &r.Ki, Resource: r.Cfm}.Reconcile(ctx, cli, s)
}
57 changes: 0 additions & 57 deletions pkg/reconciler/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,8 @@ package reconciler

import (
"context"
"fmt"
"strconv"

"github.com/sustainable.computing.io/kepler-operator/pkg/api/v1alpha1"
"github.com/sustainable.computing.io/kepler-operator/pkg/components/exporter"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

Expand All @@ -52,52 +44,3 @@ type Result struct {
type Reconciler interface {
Reconcile(context.Context, client.Client, *runtime.Scheme) Result
}

type KeplerDaemonSetReconciler struct {
Ki v1alpha1.KeplerInternal
Ds *appsv1.DaemonSet
}

func (r KeplerDaemonSetReconciler) Reconcile(ctx context.Context, cli client.Client, s *runtime.Scheme) Result {

secretRef := r.Ki.Spec.Exporter.Redfish.SecretRef
secret, err := r.getRedfishSecret(ctx, cli, secretRef)

if err != nil {
return Result{Action: Stop, Error: fmt.Errorf("Error occured while getting secret %w", err)}
}
if secret == nil {
return Result{Action: Stop, Error: fmt.Errorf("Redfish secret configured, but secret %q not found", secretRef)}
}
if _, ok := secret.Data[exporter.REDFISH_CSV]; !ok {
return Result{Action: Stop, Error: fmt.Errorf("Redfish secret does not contain \"redfish.csv\"")}
}

exporter.MountRedfishSecretToDaemonSet(r.Ds, secret)

return Updater{Owner: &r.Ki, Resource: r.Ds}.Reconcile(ctx, cli, s)
}

func (r KeplerDaemonSetReconciler) getRedfishSecret(ctx context.Context, cli client.Client, secretName string) (*corev1.Secret, error) {
ns := r.Ki.Spec.Exporter.Deployment.Namespace
redfishSecret := corev1.Secret{}
if err := cli.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &redfishSecret); err != nil {
return nil, client.IgnoreNotFound(err)
}
return &redfishSecret, nil
}

type KeplerConfigMapReconciler struct {
Ki v1alpha1.KeplerInternal
Cfm *corev1.ConfigMap
}

func (r KeplerConfigMapReconciler) Reconcile(ctx context.Context, cli client.Client, s *runtime.Scheme) Result {
rf := r.Ki.Spec.Exporter.Redfish
zero := metav1.Duration{}
if rf.ProbeInterval != zero {
r.Cfm.Data["REDFISH_PROBE_INTERVAL_IN_SECONDS"] = fmt.Sprintf("%f", rf.ProbeInterval.Duration.Seconds())
}
r.Cfm.Data["REDFISH_SKIP_SSL_VERIFY"] = strconv.FormatBool(rf.SkipSSLVerify)
return Updater{Owner: &r.Ki, Resource: r.Cfm}.Reconcile(ctx, cli, s)
}
7 changes: 5 additions & 2 deletions pkg/utils/k8s/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ const (
OpenShift
)

// ContainerIndex type represents the hard-coded index of Containers in a PodSpec
type ContainerIndex int

type StringMap map[string]string

type SCCAllows struct {
Expand Down Expand Up @@ -158,15 +161,15 @@ func HostPIDFromDS(ds *appsv1.DaemonSet) bool {
return ds.Spec.Template.Spec.HostPID
}

func CommandFromDS(ds *appsv1.DaemonSet, index int) []string {
func CommandFromDS(ds *appsv1.DaemonSet, index ContainerIndex) []string {
return ds.Spec.Template.Spec.Containers[index].Command
}

func AnnotationFromDS(ds *appsv1.DaemonSet) map[string]string {
return ds.Spec.Template.Annotations
}

func VolumeMountsFromDS(ds *appsv1.DaemonSet, index int) []corev1.VolumeMount {
func VolumeMountsFromDS(ds *appsv1.DaemonSet, index ContainerIndex) []corev1.VolumeMount {
return ds.Spec.Template.Spec.Containers[index].VolumeMounts
}

Expand Down
19 changes: 11 additions & 8 deletions tests/e2e/kepler_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,11 @@ func TestKeplerInternal_ReconciliationWithRedfish(t *testing.T) {
ds := appsv1.DaemonSet{}
f.AssertNoResourceExists(ki.Name, testNs, &ds)

cond := f.GetKeplerInternal(name).Status.Exporter.Conditions
assert.True(t, len(cond) > 1)
assert.Equal(t, fmt.Sprintf("Redfish secret configured, but secret %q not found", secretName), cond[0].Message)
// provide time for controller to reconcile
// NOTE: reconcile should be false since the secret is not created yet
ki = f.WaitUntilInternalCondition(name, v1alpha1.Reconciled, v1alpha1.ConditionFalse)
reconciled, _ := k8s.FindCondition(ki.Status.Exporter.Conditions, v1alpha1.Reconciled)
assert.Equal(t, fmt.Sprintf("Redfish secret %q configured, but not found in %q namespace", secretName, testNs), reconciled.Message)

// create redfish secret
redfishSecret := corev1.Secret{
Expand All @@ -116,18 +118,19 @@ func TestKeplerInternal_ReconciliationWithRedfish(t *testing.T) {

// wait for DaemonSet to be created
f.AssertResourceExists(ki.Name, testNs, &ds)
cond = f.GetKeplerInternal(name).Status.Exporter.Conditions
assert.True(t, len(cond) > 1)

// expect reconcile to be true after secret is created
ki = f.WaitUntilInternalCondition(name, v1alpha1.Reconciled, v1alpha1.ConditionTrue)

containers := ds.Spec.Template.Spec.Containers
assert.Equal(t, 1, len(containers))
exp := containers[exporter.IdxKeplerContainer]
assert.Contains(t, exp.Command, exporter.REDFISH_ARGS)
exp := containers[exporter.KeplerContainerIndex]
assert.Contains(t, exp.Command, exporter.RedfishArgs)
assert.Contains(t, exp.VolumeMounts,
corev1.VolumeMount{Name: "redfish-cred", MountPath: "/etc/redfish", ReadOnly: true})
assert.Contains(t, ds.Spec.Template.Spec.Volumes,
k8s.VolumeFromSecret("redfish-cred", redfishSecret.Name))
assert.Contains(t, ds.Spec.Template.Annotations, exporter.REDFISH_ANNOTATION)
assert.Contains(t, ds.Spec.Template.Annotations, exporter.RedfishSecretAnnotation)

og := ds.Status.ObservedGeneration
assert.Equal(t, og, int64(1))
Expand Down

0 comments on commit ef88a4d

Please sign in to comment.