From 2c2623a28be725bbee89f17efb778b8aa3c4405c Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 12:38:06 +0300 Subject: [PATCH 01/65] Update README.md --- charts/cbcontainers-operator/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/cbcontainers-operator/README.md b/charts/cbcontainers-operator/README.md index 18e7b0a2..45366446 100644 --- a/charts/cbcontainers-operator/README.md +++ b/charts/cbcontainers-operator/README.md @@ -20,7 +20,6 @@ See [Customization](#namespace). Now, install the actual helm chart from source: ```sh -git checkout v6.0.1 # Move to the latest version of the operator cd charts/cbcontainers-operator helm install cbcontainers-operator ./cbcontainers-operator-chart ``` From 6bb0ee3ada16c885dee75f30b3ce971e60d45b10 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 12:38:51 +0300 Subject: [PATCH 02/65] Update README.md --- charts/cbcontainers-agent/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 27029489..bdc9a802 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -24,10 +24,8 @@ There are 8 required fields that need to be provided by the user: After setting these required fields in a `values.yaml` file you can install the chart from source ```sh -git checkout v6.0.1 # Move to the latest version of the operator cd charts/cbcontainers-agent -git checkout v6.0.0 # install the latest version of the operator -helm install cbcontainers-agent ./cbcontainers-agent-chart -f values.yaml --namespace cbcontainers-dataplane +helm install cbcontainers-agent ./cbcontainers-agent-chart ``` ## Customization From ee25a2a5dfde30a1702d42297b4e891bd0ff0689 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 13:25:54 +0300 Subject: [PATCH 03/65] Fix charts for main to be idetical to latest release v6.0.1 --- .../cbcontainers-operator-chart/templates/operator.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 045ead4e..af2868e3 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -6404,12 +6404,7 @@ spec: type: object status: description: CBContainersAgentStatus defines the observed state of CBContainersAgent - properties: - observedGeneration: - description: ObservedGeneration is the last Custom resource generation - that was fully reconciled. - format: int64 - type: integertype: object + type: object type: object served: true storage: true From 2a8595323a54434dae4f4c679ebe1649a5aa6693 Mon Sep 17 00:00:00 2001 From: BenRub Date: Wed, 30 Aug 2023 14:11:55 +0300 Subject: [PATCH 04/65] Update README.md --- charts/cbcontainers-agent/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index bdc9a802..12809ba2 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -25,7 +25,7 @@ After setting these required fields in a `values.yaml` file you can install the ```sh cd charts/cbcontainers-agent -helm install cbcontainers-agent ./cbcontainers-agent-chart +helm install cbcontainers-agent ./cbcontainers-agent-chart -n cbcontainers-dataplane ``` ## Customization From 45b920bdf788a82cd4636795309fb71d781c469c Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:18:59 +0300 Subject: [PATCH 05/65] Adding labels field to charts --- .../cbcontainers-agent-chart/example-values.yaml | 2 ++ .../cbcontainers-agent-chart/templates/containers-agent.yaml | 4 ++++ .../cbcontainers-agent/cbcontainers-agent-chart/values.yaml | 1 + 3 files changed, 7 insertions(+) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml index 1c33f775..3fb7cc11 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml @@ -23,6 +23,8 @@ gateways: coreEventsGatewayHost: events.gateway.com hardeningEventsGatewayHost: hardening.events.gateway.com runtimeEventsGatewayHost: runtime.events.gateway.com +labels: + my-key: my-value # components is the set of components that will be installed components: settings: diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml index 92e6402a..4521bb6e 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/containers-agent.yaml @@ -2,6 +2,10 @@ apiVersion: operator.containers.carbonblack.io/v1 kind: CBContainersAgent metadata: name: cbcontainers-agent + {{- with .Values.labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} spec: account: {{ required "orgKey is required" .Values.orgKey }} clusterName: "{{ required "clusterGroup is required" .Values.clusterGroup }}:{{ required "clusterName is required" .Values.clusterName }}" diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml index 42a89ab9..1f55fb00 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml @@ -16,6 +16,7 @@ gateways: coreEventsGatewayHost: "" hardeningEventsGatewayHost: "" runtimeEventsGatewayHost: "" +labels: components: cndr: enabled: true \ No newline at end of file From 9259740ecb40b2e8fa267ede60e0dfdd0f1eff39 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:46:36 +0300 Subject: [PATCH 06/65] Adding company code secret template to charts --- .../cbcontainers-agent-chart/example-values.yaml | 6 ++++-- .../templates/_helpers.tpl | 11 +++++++++++ .../cbcontainers-company-code-secret.yaml | 15 +++++++++++++++ .../cbcontainers-agent-chart/values.yaml | 4 +++- 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml index 3fb7cc11..cfdef89a 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/example-values.yaml @@ -6,7 +6,7 @@ accessToken: "" # orgKey is the ID of the organization account orgKey: "ABC123" # version is the version of the agent that will be installed -version: "3.0.0" +version: "3.0.1" # clusterGroup is the group that the cluster will belong to. clusterGroup: "default" # clusterName is the name that will be used for the cluster that the agent is installed on @@ -204,4 +204,6 @@ components: port: 7071 enabled: false cndr: - enabled: true \ No newline at end of file + enabled: true + # accessTokenSecretName is the name of the Kubernetes object of type Secret that holds the values of the Company Code + companyCodeSecretName: "my-company-code-secret-name" \ No newline at end of file diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl index efda2dbf..0a1a9a67 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/_helpers.tpl @@ -8,3 +8,14 @@ {{- end -}} {{- end -}} +{{/* Get the name of the secret that contains the company code */}} +{{- define "cbcontainers-agent.company-code-name" -}} +{{- $secret := . -}} +{{- if $secret -}} +"{{- $secret -}}" +{{- else -}} +"cbcontainers-company-code" +{{- end -}} +{{- end -}} + + diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml new file mode 100644 index 00000000..786fd88c --- /dev/null +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml @@ -0,0 +1,15 @@ +{{- /* +The Secret object will be rendered only if the accessToken value is provided. +this value is required in order for the agent components to work correctly +so not having the access token secret created here assumes that the user of the charts +created the secret in an alternative way +*/}} +{{- if .Values.companyCode -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cbcontainers-agent.company-code-name" .Values.components.cndr.companyCodeSecretName }} + namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} +data: + accessToken: {{ .Values.companyCode | b64enc }} +{{- end -}} diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml index 1f55fb00..8c232c21 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/values.yaml @@ -1,5 +1,7 @@ -# accessToken is the API token used by the agent to communicate with the backend +# Optional: accessToken is the API token used by the agent to communicate with the backend accessToken: "" +# Optional: companyCode is the Company codes used by the agent to install the Endpoints solution +companyCode: "" # orgKey is the ID of the organization account orgKey: "" # version is the version of the agent that will be installed From f6a045c094ca658fca3b109e26cd50d7de72c932 Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 16:52:31 +0300 Subject: [PATCH 07/65] Adding company code secret template to charts fix --- .../templates/cbcontainers-company-code-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml index 786fd88c..e1853758 100644 --- a/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml +++ b/charts/cbcontainers-agent/cbcontainers-agent-chart/templates/cbcontainers-company-code-secret.yaml @@ -11,5 +11,5 @@ metadata: name: {{ include "cbcontainers-agent.company-code-name" .Values.components.cndr.companyCodeSecretName }} namespace: {{ default "cbcontainers-dataplane" .Values.agentNamespace }} data: - accessToken: {{ .Values.companyCode | b64enc }} + companyCode: {{ .Values.companyCode | b64enc }} {{- end -}} From 84aa072cbda132fd7e2a32bc155dc1b67f06293a Mon Sep 17 00:00:00 2001 From: benrub Date: Wed, 30 Aug 2023 17:03:54 +0300 Subject: [PATCH 08/65] Add documentation for secrets with helm --- charts/cbcontainers-agent/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/charts/cbcontainers-agent/README.md b/charts/cbcontainers-agent/README.md index 12809ba2..301b09d7 100644 --- a/charts/cbcontainers-agent/README.md +++ b/charts/cbcontainers-agent/README.md @@ -49,6 +49,7 @@ If the secret is pre-created before deploying the agent, then `agentNamespace` h ### Secret creation +#### Carbon Black Api Key In order for the agent components to function correctly and be able to communicate with the CBC backend an access token is required. This token is located in a secret. @@ -58,6 +59,28 @@ If that secret does not exist, the operator will not start any of the agent comp If you want to create the secret as part of the chart installation provide the `accessToken` value to the chart. +*DO NOT* store the token in your source code + +Inject this value as part of your pipeline in a secure way! + This means storing the secret as plain text in your `values.yaml` file. If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `accessToken` value and the chart will not create the `Secret` objects. + +#### Carbon Black Company Codes +In order for the agent CNDR component to function correctly and be able to communicate with the CBC backend a company code is required. + +This code is located in a secret. +By default, the secret is named `"cbcontainers-company-code"`, but that is configurable via the `components.cndr.companyCodeSecretName` property. + +If that secret does not exist, the CNDR component will fail. + +If you want to create the secret as part of the chart installation provide the `companyCode` value to the chart. + +*DO NOT* store the code in your source code + +Inject this value as part of your pipeline in a secure way! + +This means storing the secret as plain text in your `values.yaml` file. + +If you prefer to create the `Secret` yourself in an alternative and more secure way, don't set the `companyCode` value and the chart will not create the `Secret` objects. \ No newline at end of file From 6109ee3af5b49289dfbe882e4614891c86643cf9 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 17 Jul 2023 15:09:09 +0300 Subject: [PATCH 09/65] config applier skeleton --- config_applier/applier.go | 164 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 config_applier/applier.go diff --git a/config_applier/applier.go b/config_applier/applier.go new file mode 100644 index 00000000..e7074771 --- /dev/null +++ b/config_applier/applier.go @@ -0,0 +1,164 @@ +package config_applier + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "math/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +// TODO: Use interfaces for dependencies? + +var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0"} + +var ( + tr = true + fal = false +) + +type Applier struct { + K8sClient client.Client + Logger logr.Logger +} + +type pendingChange struct { + ID string + Version *string + EnableClusterScanning *bool + EnableRuntime *bool +} + +func (applier *Applier) RunLoop() { + // TODO: stop on signal + for { + applier.Logger.Info("Running config check iteration") + + // TODO: Fix pointers vs non-pointers + change, err := applier.getPendingChange() + if err != nil { + // TODO + applier.Logger.Error(err, "failed to get existing CR") + continue + } + if change == nil { + applier.Logger.Info("No pending remote configuration changes found") + // TODO: Polling interval + time.Sleep(10 * time.Second) + continue + } + + applier.Logger.Info("Applying change", "change", change) + err = applier.applyChange(*change) + if err != nil { + // TODO + applier.Logger.Error(err, "failed to apply change") + // TODO: REport error ? + continue + } + + err = applier.acknowledgeChange(*change) + if err != nil { + // TODO + panic(err) + } + + time.Sleep(40 * time.Second) + } + + // TODO: Staggering in case of multiple changes? +} + +func (applier *Applier) getPendingChange() (*pendingChange, error) { + rand := randomChange() + return rand, nil +} + +func (applier *Applier) acknowledgeChange(change pendingChange) error { + + return nil +} + +func (applier *Applier) applyChange(change pendingChange) error { + cr, err := applier.getCR() + if err != nil { + return err + } + if cr == nil { + // TODO: Log + return nil + } + + // TODO: Validation! + if change.Version != nil { + cr.Spec.Version = *change.Version + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + + // Apply + // TODO: Handle Conflict response and retry + err = applier.K8sClient.Update(context.TODO(), cr) + return err +} + +func (applier *Applier) getCR() (*cbcontainersv1.CBContainersAgent, error) { + + // TODO: Copied from the controller + cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} + if err := applier.K8sClient.List(context.TODO(), cbContainersAgentsList); err != nil { + return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %v", err) + } + + if cbContainersAgentsList.Items == nil || len(cbContainersAgentsList.Items) == 0 { + return nil, nil + } + + if len(cbContainersAgentsList.Items) > 1 { + return nil, fmt.Errorf("there is more than 1 CBContainersAgent k8s object, please delete unwanted resources") + } + + return &cbContainersAgentsList.Items[0], nil +} + +func randomChange() *pendingChange { + csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) + if versionRand == len(versions) { + return nil + } + + changeVersion := &versions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = &tr + case 2, 4: + changeClusterScanning = &fal + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = &tr + case 2, 4: + changeRuntime = &fal + default: + changeRuntime = nil + } + + return &pendingChange{ + Version: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + } +} From cb2fb12b29946565247129ef5d371dbfdc53628c Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 20 Jul 2023 13:05:48 +0300 Subject: [PATCH 10/65] More work on the configuration applier loop. --- config_applier/applier.go | 139 ++++++++++++++++++++++++-------------- main.go | 31 +++++++-- 2 files changed, 115 insertions(+), 55 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index e7074771..6b1034df 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -11,6 +11,12 @@ import ( ) // TODO: Use interfaces for dependencies? +// TODO: Env_var to enable +// TODO: Configurable polling interval + +const ( + timeoutSingleIteration = time.Second * 30 +) var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0"} @@ -19,75 +25,111 @@ var ( fal = false ) +type APIGateway interface { + // Get Compatibility matrix + // Update status of change (ack/error) + // Get pending changes + // Set status for change (acknowledge/error) +} + type Applier struct { K8sClient client.Client Logger logr.Logger + Gateway APIGateway +} + +type pendingChangesResponse struct { + ConfigurationChanges []pendingChange `json:"configuration_changes"` } type pendingChange struct { - ID string - Version *string - EnableClusterScanning *bool - EnableRuntime *bool + ID string `json:"id"` + Version *string `json:"version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` +} + +type configurationChangeStatusRequest struct { + ID string `json:"id"` + Status string `json:"status"` + Reason string `json:"reason"` } -func (applier *Applier) RunLoop() { - // TODO: stop on signal +type changeStatus string + +var ( + statusPending changeStatus = "PENDING" + statusAcknowledged changeStatus = "ACKNOWLEDGED" + statusFailed changeStatus = "FAILED" +) + +func (applier *Applier) RunLoop(signalsContext context.Context) { + pollingSleepDuration := 20 * time.Second + pollingTimer := time.NewTicker(pollingSleepDuration) + defer pollingTimer.Stop() + for { - applier.Logger.Info("Running config check iteration") - - // TODO: Fix pointers vs non-pointers - change, err := applier.getPendingChange() - if err != nil { - // TODO - applier.Logger.Error(err, "failed to get existing CR") - continue - } - if change == nil { - applier.Logger.Info("No pending remote configuration changes found") - // TODO: Polling interval - time.Sleep(10 * time.Second) - continue + select { + case <-signalsContext.Done(): + applier.Logger.Info("Received cancel signal, turning off configuration applier") + return + case <-pollingTimer.C: + // Nothing to do; this is the polling sleep case } + // TODO: Pass context down? + applier.Logger.Info("RUNNING ITERATION") + applier.configCheckIteration(signalsContext) + } +} - applier.Logger.Info("Applying change", "change", change) - err = applier.applyChange(*change) - if err != nil { - // TODO - applier.Logger.Error(err, "failed to apply change") - // TODO: REport error ? - continue - } +func (applier *Applier) configCheckIteration(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) + defer cancel() - err = applier.acknowledgeChange(*change) - if err != nil { - // TODO - panic(err) - } + applier.Logger.Info("Checking for pending remote configuration changes...") - time.Sleep(40 * time.Second) + change, err := applier.getPendingChange(ctx) + if err != nil { + applier.Logger.Error(err, "Failed to get pending configuration changes") + return + } + if change == nil { + applier.Logger.Info("No pending remote configuration changes found") + return } - // TODO: Staggering in case of multiple changes? + applier.Logger.Info("Applying remote configuration change", "change", change) + err = applier.applyChange(ctx, change) + if err != nil { + applier.Logger.Error(err, "Failed to apply configuration change", "changeID", change.ID) + // Intentional fallthrough so we always update the status of the change on the backend + } + + if errStatusUpdate := applier.updateChangeStatus(ctx, change, err); errStatusUpdate != nil { + applier.Logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") + return + } + + return } -func (applier *Applier) getPendingChange() (*pendingChange, error) { +func (applier *Applier) getPendingChange(ctx context.Context) (*pendingChange, error) { rand := randomChange() return rand, nil } -func (applier *Applier) acknowledgeChange(change pendingChange) error { +func (applier *Applier) updateChangeStatus(ctx context.Context, change *pendingChange, err error) error { return nil } -func (applier *Applier) applyChange(change pendingChange) error { - cr, err := applier.getCR() +func (applier *Applier) applyChange(ctx context.Context, change *pendingChange) error { + cr, err := applier.getContainerAgentCR(ctx) if err != nil { return err } if cr == nil { - // TODO: Log + applier.Logger.Info("No CBContainersAgent instance found") return nil } @@ -102,17 +144,19 @@ func (applier *Applier) applyChange(change pendingChange) error { cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime } - // Apply // TODO: Handle Conflict response and retry - err = applier.K8sClient.Update(context.TODO(), cr) + err = applier.K8sClient.Update(ctx, cr) return err } -func (applier *Applier) getCR() (*cbcontainersv1.CBContainersAgent, error) { +// getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions +// if no resource is defined, nil is returned +// in case more than 1 resource is defined (which is not supported), only the first one is returned +func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { + // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance - // TODO: Copied from the controller cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} - if err := applier.K8sClient.List(context.TODO(), cbContainersAgentsList); err != nil { + if err := applier.K8sClient.List(ctx, cbContainersAgentsList); err != nil { return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %v", err) } @@ -120,10 +164,7 @@ func (applier *Applier) getCR() (*cbcontainersv1.CBContainersAgent, error) { return nil, nil } - if len(cbContainersAgentsList.Items) > 1 { - return nil, fmt.Errorf("there is more than 1 CBContainersAgent k8s object, please delete unwanted resources") - } - + // We don't log a warning if len >=2 as the controller already warns users about that return &cbContainersAgentsList.Items[0], nil } diff --git a/main.go b/main.go index fc23d85c..13c7b877 100644 --- a/main.go +++ b/main.go @@ -25,14 +25,16 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/state/operator" + "github.com/vmware/cbcontainers-operator/config_applier" "go.uber.org/zap/zapcore" + coreV1 "k8s.io/api/core/v1" "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" + "sync" "strings" - coreV1 "k8s.io/api/core/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/processors" @@ -167,11 +169,28 @@ func main() { os.Exit(1) } - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } + signalsContext := ctrl.SetupSignalHandler() + k8sClient := mgr.GetClient() + applier := config_applier.Applier{K8sClient: k8sClient, Logger: ctrl.Log.WithName("configurator")} + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + setupLog.Info("starting manager") + if err := mgr.Start(signalsContext); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } + }() + go func() { + defer wg.Done() + setupLog.Info("starting configuration monitor") + applier.RunLoop(signalsContext) + }() + + wg.Wait() } func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier string, k8sVersion string) { From 6af7c0b0ad46bfe76ffc14133eba3e6742055c71 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 11:37:48 +0300 Subject: [PATCH 11/65] Happy path test for config_applier --- config_applier/applier.go | 85 +++++++++++++------ config_applier/applier_test.go | 77 +++++++++++++++++ config_applier/mocks/generated.go | 3 + .../mocks/mock_configuration_api.go | 64 ++++++++++++++ 4 files changed, 203 insertions(+), 26 deletions(-) create mode 100644 config_applier/applier_test.go create mode 100644 config_applier/mocks/generated.go create mode 100644 config_applier/mocks/mock_configuration_api.go diff --git a/config_applier/applier.go b/config_applier/applier.go index 6b1034df..56fe55ae 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -7,6 +7,7 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "math/rand" "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" "time" ) @@ -25,41 +26,51 @@ var ( fal = false ) -type APIGateway interface { +type ConfigurationAPI interface { // Get Compatibility matrix // Update status of change (ack/error) // Get pending changes // Set status for change (acknowledge/error) + + GetConfigurationChanges() ([]ConfigurationChange, error) + UpdateConfigurationChangeStatus(ConfigurationChangeStatusUpdate) error } type Applier struct { K8sClient client.Client Logger logr.Logger - Gateway APIGateway + Api ConfigurationAPI } type pendingChangesResponse struct { - ConfigurationChanges []pendingChange `json:"configuration_changes"` + ConfigurationChanges []ConfigurationChange `json:"configuration_changes"` } -type pendingChange struct { +type ConfigurationChange struct { ID string `json:"id"` - Version *string `json:"version"` + Status string `json:"status"` + AgentVersion *string `json:"agent_version"` EnableClusterScanning *bool `json:"enable_cluster_scanning"` EnableRuntime *bool `json:"enable_runtime"` } -type configurationChangeStatusRequest struct { +type ConfigurationChangeStatusUpdate struct { ID string `json:"id"` Status string `json:"status"` Reason string `json:"reason"` + // AppliedGeneration tracks the generation of the Custom resource where the change was applied + AppliedGeneration int64 `json:"applied_generation"` + // AppliedTimestamp records when the change was applied in RFC3339 format + AppliedTimestamp string `json:"applied_timestamp"` + + // TODO: CLuster and group. Cluster identifier? } type changeStatus string var ( statusPending changeStatus = "PENDING" - statusAcknowledged changeStatus = "ACKNOWLEDGED" + statusAcknowledged changeStatus = "ACKNOWLEDGED" // TODO: Acknowledged or applied? statusFailed changeStatus = "FAILED" ) @@ -78,11 +89,11 @@ func (applier *Applier) RunLoop(signalsContext context.Context) { } // TODO: Pass context down? applier.Logger.Info("RUNNING ITERATION") - applier.configCheckIteration(signalsContext) + applier.RunIteration(signalsContext) // TODO!!! } } -func (applier *Applier) configCheckIteration(ctx context.Context) { +func (applier *Applier) RunIteration(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() @@ -99,13 +110,13 @@ func (applier *Applier) configCheckIteration(ctx context.Context) { } applier.Logger.Info("Applying remote configuration change", "change", change) - err = applier.applyChange(ctx, change) + cr, err := applier.applyChange(ctx, change) if err != nil { applier.Logger.Error(err, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough so we always update the status of the change on the backend } - if errStatusUpdate := applier.updateChangeStatus(ctx, change, err); errStatusUpdate != nil { + if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, err); errStatusUpdate != nil { applier.Logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") return } @@ -113,29 +124,44 @@ func (applier *Applier) configCheckIteration(ctx context.Context) { return } -func (applier *Applier) getPendingChange(ctx context.Context) (*pendingChange, error) { - rand := randomChange() - return rand, nil -} +func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { + changes, err := applier.Api.GetConfigurationChanges() + if err != nil { + return nil, err + } -func (applier *Applier) updateChangeStatus(ctx context.Context, change *pendingChange, err error) error { + for _, change := range changes { + if change.Status == string(statusPending) { + return &change, nil + } + } + return nil, nil +} - return nil +func (applier *Applier) updateChangeStatus(ctx context.Context, change *ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, err error) error { + statusUpdate := ConfigurationChangeStatusUpdate{ + ID: change.ID, + Status: string(statusAcknowledged), + Reason: "", // TODO + AppliedGeneration: cr.Generation, + AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), + } + return applier.Api.UpdateConfigurationChangeStatus(statusUpdate) } -func (applier *Applier) applyChange(ctx context.Context, change *pendingChange) error { +func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { cr, err := applier.getContainerAgentCR(ctx) if err != nil { - return err + return nil, err } if cr == nil { applier.Logger.Info("No CBContainersAgent instance found") - return nil + return nil, nil } // TODO: Validation! - if change.Version != nil { - cr.Spec.Version = *change.Version + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion } if change.EnableClusterScanning != nil { cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning @@ -144,9 +170,12 @@ func (applier *Applier) applyChange(ctx context.Context, change *pendingChange) cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime } + generationBefore := cr.ObjectMeta.Generation // TODO: Handle Conflict response and retry err = applier.K8sClient.Update(ctx, cr) - return err + generationAfter := cr.ObjectMeta.Generation + applier.Logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) + return cr, nil } // getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions @@ -168,8 +197,10 @@ func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv return &cbContainersAgentsList.Items[0], nil } -func randomChange() *pendingChange { +func RandomChange() *ConfigurationChange { csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) + + csRand, runtimeRand, versionRand = 1, 2, 3 if versionRand == len(versions) { return nil } @@ -197,9 +228,11 @@ func randomChange() *pendingChange { changeRuntime = nil } - return &pendingChange{ - Version: changeVersion, + return &ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, EnableClusterScanning: changeClusterScanning, EnableRuntime: changeRuntime, + Status: string(statusPending), } } diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go new file mode 100644 index 00000000..bc1b9232 --- /dev/null +++ b/config_applier/applier_test.go @@ -0,0 +1,77 @@ +package config_applier_test + +import ( + "context" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" + "github.com/vmware/cbcontainers-operator/config_applier" + mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "time" +) + +func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockK8sClient := mocks.NewMockClient(ctrl) + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + + applier := config_applier.Applier{ + K8sClient: mockK8sClient, + Logger: logr.Discard(), + Api: mockAPI, + } + + // Once a change appears + // It should find our CR + // It should be applied to the CR + // It should be ACKed with proper CR generation and ID + // TODO: Compatiblity check + + configChange := config_applier.RandomChange() + mockAPI.EXPECT().GetConfigurationChanges().Return([]config_applier.ConfigurationChange{*configChange}, nil) + + mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + }, + } + }) + + mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + Do(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + asCb.ObjectMeta.Generation++ + return nil + }) + + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any()).DoAndReturn(func(update config_applier.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, int64(2), update.AppliedGeneration) + assert.Equal(t, "ACKNOWLEDGED", update.Status) + assert.NotEmpty(t, update.AppliedTimestamp, "applied timestamp should be populated") + + parsedTime, err := time.Parse(time.RFC3339, update.AppliedTimestamp) + assert.NoError(t, err) + assert.True(t, time.Now().After(parsedTime)) + return nil + }) + + applier.RunIteration(context.Background()) +} + +// TODO: Any changes with status NOT pending are ignored diff --git a/config_applier/mocks/generated.go b/config_applier/mocks/generated.go new file mode 100644 index 00000000..96ace6a2 --- /dev/null +++ b/config_applier/mocks/generated.go @@ -0,0 +1,3 @@ +package mocks + +//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/config_applier ConfigurationAPI diff --git a/config_applier/mocks/mock_configuration_api.go b/config_applier/mocks/mock_configuration_api.go new file mode 100644 index 00000000..51ae387a --- /dev/null +++ b/config_applier/mocks/mock_configuration_api.go @@ -0,0 +1,64 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/config_applier (interfaces: ConfigurationAPI) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + config_applier "github.com/vmware/cbcontainers-operator/config_applier" +) + +// MockConfigurationAPI is a mock of ConfigurationAPI interface. +type MockConfigurationAPI struct { + ctrl *gomock.Controller + recorder *MockConfigurationAPIMockRecorder +} + +// MockConfigurationAPIMockRecorder is the mock recorder for MockConfigurationAPI. +type MockConfigurationAPIMockRecorder struct { + mock *MockConfigurationAPI +} + +// NewMockConfigurationAPI creates a new mock instance. +func NewMockConfigurationAPI(ctrl *gomock.Controller) *MockConfigurationAPI { + mock := &MockConfigurationAPI{ctrl: ctrl} + mock.recorder = &MockConfigurationAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConfigurationAPI) EXPECT() *MockConfigurationAPIMockRecorder { + return m.recorder +} + +// GetConfigurationChanges mocks base method. +func (m *MockConfigurationAPI) GetConfigurationChanges() ([]config_applier.ConfigurationChange, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfigurationChanges") + ret0, _ := ret[0].([]config_applier.ConfigurationChange) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfigurationChanges indicates an expected call of GetConfigurationChanges. +func (mr *MockConfigurationAPIMockRecorder) GetConfigurationChanges() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationAPI)(nil).GetConfigurationChanges)) +} + +// UpdateConfigurationChangeStatus mocks base method. +func (m *MockConfigurationAPI) UpdateConfigurationChangeStatus(arg0 config_applier.ConfigurationChangeStatusUpdate) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. +func (mr *MockConfigurationAPIMockRecorder) UpdateConfigurationChangeStatus(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationAPI)(nil).UpdateConfigurationChangeStatus), arg0) +} From 459c8057123c9cd0d88fe5bc1f77e9a5ef092b8d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 11:38:33 +0300 Subject: [PATCH 12/65] Pass context to API functions as they'll do IO --- config_applier/applier.go | 8 ++++---- config_applier/applier_test.go | 4 ++-- config_applier/mocks/mock_configuration_api.go | 17 +++++++++-------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 56fe55ae..776d9ac5 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -32,8 +32,8 @@ type ConfigurationAPI interface { // Get pending changes // Set status for change (acknowledge/error) - GetConfigurationChanges() ([]ConfigurationChange, error) - UpdateConfigurationChangeStatus(ConfigurationChangeStatusUpdate) error + GetConfigurationChanges(context.Context) ([]ConfigurationChange, error) + UpdateConfigurationChangeStatus(context.Context, ConfigurationChangeStatusUpdate) error } type Applier struct { @@ -125,7 +125,7 @@ func (applier *Applier) RunIteration(ctx context.Context) { } func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { - changes, err := applier.Api.GetConfigurationChanges() + changes, err := applier.Api.GetConfigurationChanges(ctx) if err != nil { return nil, err } @@ -146,7 +146,7 @@ func (applier *Applier) updateChangeStatus(ctx context.Context, change *Configur AppliedGeneration: cr.Generation, AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), } - return applier.Api.UpdateConfigurationChangeStatus(statusUpdate) + return applier.Api.UpdateConfigurationChangeStatus(ctx, statusUpdate) } func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index bc1b9232..07ef880a 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -35,7 +35,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { // TODO: Compatiblity check configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges().Return([]config_applier.ConfigurationChange{*configChange}, nil) + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -59,7 +59,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any()).DoAndReturn(func(update config_applier.ConfigurationChangeStatusUpdate) error { + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, int64(2), update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) diff --git a/config_applier/mocks/mock_configuration_api.go b/config_applier/mocks/mock_configuration_api.go index 51ae387a..b0dfc6fd 100644 --- a/config_applier/mocks/mock_configuration_api.go +++ b/config_applier/mocks/mock_configuration_api.go @@ -5,6 +5,7 @@ package mocks import ( + context "context" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -35,30 +36,30 @@ func (m *MockConfigurationAPI) EXPECT() *MockConfigurationAPIMockRecorder { } // GetConfigurationChanges mocks base method. -func (m *MockConfigurationAPI) GetConfigurationChanges() ([]config_applier.ConfigurationChange, error) { +func (m *MockConfigurationAPI) GetConfigurationChanges(arg0 context.Context) ([]config_applier.ConfigurationChange, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConfigurationChanges") + ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) ret0, _ := ret[0].([]config_applier.ConfigurationChange) ret1, _ := ret[1].(error) return ret0, ret1 } // GetConfigurationChanges indicates an expected call of GetConfigurationChanges. -func (mr *MockConfigurationAPIMockRecorder) GetConfigurationChanges() *gomock.Call { +func (mr *MockConfigurationAPIMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationAPI)(nil).GetConfigurationChanges)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationAPI)(nil).GetConfigurationChanges), arg0) } // UpdateConfigurationChangeStatus mocks base method. -func (m *MockConfigurationAPI) UpdateConfigurationChangeStatus(arg0 config_applier.ConfigurationChangeStatusUpdate) error { +func (m *MockConfigurationAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 config_applier.ConfigurationChangeStatusUpdate) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0) + ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. -func (mr *MockConfigurationAPIMockRecorder) UpdateConfigurationChangeStatus(arg0 interface{}) *gomock.Call { +func (mr *MockConfigurationAPIMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationAPI)(nil).UpdateConfigurationChangeStatus), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationAPI)(nil).UpdateConfigurationChangeStatus), arg0, arg1) } From fa1eea0c9b52d61e4ea97818f2c0d80c295f7225 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 13:58:09 +0300 Subject: [PATCH 13/65] More tests --- config_applier/applier.go | 13 +++---- config_applier/applier_test.go | 64 ++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 776d9ac5..235e263c 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -93,7 +93,7 @@ func (applier *Applier) RunLoop(signalsContext context.Context) { } } -func (applier *Applier) RunIteration(ctx context.Context) { +func (applier *Applier) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() @@ -102,26 +102,27 @@ func (applier *Applier) RunIteration(ctx context.Context) { change, err := applier.getPendingChange(ctx) if err != nil { applier.Logger.Error(err, "Failed to get pending configuration changes") - return + return err // TODO } + if change == nil { applier.Logger.Info("No pending remote configuration changes found") - return + return nil } applier.Logger.Info("Applying remote configuration change", "change", change) cr, err := applier.applyChange(ctx, change) if err != nil { applier.Logger.Error(err, "Failed to apply configuration change", "changeID", change.ID) - // Intentional fallthrough so we always update the status of the change on the backend + // Intentional fallthrough so we always update the status of the change on the backend, including failed status } if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, err); errStatusUpdate != nil { applier.Logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") - return + return errStatusUpdate // TODO } - return + return nil } func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 07ef880a..50589763 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -2,6 +2,7 @@ package config_applier_test import ( "context" + "errors" "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -71,7 +72,66 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - applier.RunIteration(context.Background()) + err := applier.RunIteration(context.Background()) + assert.NoError(t, err) } -// TODO: Any changes with status NOT pending are ignored +func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + name string + dataFromService []config_applier.ConfigurationChange + }{ + { + name: "empty list", + dataFromService: []config_applier.ConfigurationChange{}, + }, + { + name: "list is not empty but there are no PENDING changes", + dataFromService: []config_applier.ConfigurationChange{ + {ID: "123", Status: "non-existent"}, + {ID: "234", Status: "FAILED"}, + {ID: "345", Status: "ACKNOWLEDGED"}, + {ID: "456", Status: "SUCCEEDED"}, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + applier := config_applier.Applier{ + K8sClient: nil, + Logger: logr.Discard(), + Api: mockAPI, + } + + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) + + err := applier.RunIteration(context.Background()) + assert.NoError(t, err) + }) + } +} + +func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + applier := config_applier.Applier{ + K8sClient: nil, + Logger: logr.Discard(), + Api: mockAPI, + } + + errFromService := errors.New("some error") + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) + + returnedErr := applier.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} From 319826af77b16aea17f00d63c53a8aca6b736172 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 14:54:28 +0300 Subject: [PATCH 14/65] Send Failed status to backend when failing to list or update CR --- config_applier/applier.go | 48 ++++++++------ config_applier/applier_test.go | 118 ++++++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 20 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 235e263c..816afac2 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -99,10 +99,10 @@ func (applier *Applier) RunIteration(ctx context.Context) error { applier.Logger.Info("Checking for pending remote configuration changes...") - change, err := applier.getPendingChange(ctx) - if err != nil { - applier.Logger.Error(err, "Failed to get pending configuration changes") - return err // TODO + change, errGettingChanges := applier.getPendingChange(ctx) + if errGettingChanges != nil { + applier.Logger.Error(errGettingChanges, "Failed to get pending configuration changes") + return errGettingChanges // TODO } if change == nil { @@ -111,18 +111,18 @@ func (applier *Applier) RunIteration(ctx context.Context) error { } applier.Logger.Info("Applying remote configuration change", "change", change) - cr, err := applier.applyChange(ctx, change) - if err != nil { - applier.Logger.Error(err, "Failed to apply configuration change", "changeID", change.ID) + cr, errApplyingCR := applier.applyChange(ctx, change) + if errApplyingCR != nil { + applier.Logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough so we always update the status of the change on the backend, including failed status } - if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, err); errStatusUpdate != nil { - applier.Logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") + if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, errApplyingCR); errStatusUpdate != nil { + applier.Logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") return errStatusUpdate // TODO } - return nil + return errApplyingCR } func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { @@ -139,14 +139,24 @@ func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationCha return nil, nil } -func (applier *Applier) updateChangeStatus(ctx context.Context, change *ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, err error) error { - statusUpdate := ConfigurationChangeStatusUpdate{ - ID: change.ID, - Status: string(statusAcknowledged), - Reason: "", // TODO - AppliedGeneration: cr.Generation, - AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), +func (applier *Applier) updateChangeStatus(ctx context.Context, change *ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { + var statusUpdate ConfigurationChangeStatusUpdate + if encounteredError == nil { + statusUpdate = ConfigurationChangeStatusUpdate{ + ID: change.ID, + Status: string(statusAcknowledged), + Reason: "", // TODO + AppliedGeneration: cr.Generation, + AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), + } + } else { + statusUpdate = ConfigurationChangeStatusUpdate{ + ID: change.ID, + Status: string(statusFailed), + Reason: encounteredError.Error(), // TODO + } } + return applier.Api.UpdateConfigurationChangeStatus(ctx, statusUpdate) } @@ -176,7 +186,7 @@ func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationCh err = applier.K8sClient.Update(ctx, cr) generationAfter := cr.ObjectMeta.Generation applier.Logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) - return cr, nil + return cr, err } // getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions @@ -187,7 +197,7 @@ func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} if err := applier.K8sClient.List(ctx, cbContainersAgentsList); err != nil { - return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %v", err) + return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) } if cbContainersAgentsList.Items == nil || len(cbContainersAgentsList.Items) == 0 { diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 50589763..11a87859 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -12,10 +12,26 @@ import ( "github.com/vmware/cbcontainers-operator/config_applier" mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "testing" "time" ) +func setupApplier(ctrl *gomock.Controller, k8sClient client.Client, api config_applier.ConfigurationAPI) config_applier.Applier { + if k8sClient == nil { + k8sClient = mocks.NewMockClient(ctrl) + } + if api == nil { + api = mocksConfigApplier.NewMockConfigurationAPI(ctrl) + } + + return config_applier.Applier{ + K8sClient: k8sClient, + Logger: logr.Discard(), + Api: api, + } +} + func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -23,6 +39,8 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { mockK8sClient := mocks.NewMockClient(ctrl) mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + //applier := + applier := config_applier.Applier{ K8sClient: mockK8sClient, Logger: logr.Discard(), @@ -53,7 +71,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { }) mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - Do(func(_ context.Context, item any, _ ...any) error { + DoAndReturn(func(_ context.Context, item any, _ ...any) error { asCb, ok := item.(*cbcontainersv1.CBContainersAgent) require.True(t, ok) asCb.ObjectMeta.Generation++ @@ -135,3 +153,101 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } + +func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockClient(ctrl) + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + + applier := config_applier.Applier{ + K8sClient: mockClient, + Logger: logr.Discard(), + Api: mockAPI, + } + + configChange := config_applier.RandomChange() + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + + errFromService := errors.New("some error") + mockClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) + + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) + + //mockClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + // DoAndReturn(func(_ context.Context, item any, _ ...any) error { + // asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + // require.True(t, ok) + // asCb.ObjectMeta.Generation++ + // return nil + // }) + + returnedErr := applier.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} + +func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := mocks.NewMockClient(ctrl) + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + + applier := config_applier.Applier{ + K8sClient: mockClient, + Logger: logr.Discard(), + Api: mockAPI, + } + + configChange := config_applier.RandomChange() + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + + mockClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + }, + } + }) + + errFromService := errors.New("some error") + mockClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) + + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) + + returnedErr := applier.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} + +// Update fails, marks as failed + +// Update to backend fails, returns err + +// TODO: No CR, pending change -> nothing happens but warning? + +// Scheduler -> failed; increased retry From 0917e02dd084cf14e413a818993a7f4b72d41e5e Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 15:09:36 +0300 Subject: [PATCH 15/65] Treat missing CR when a change is pending as error --- config_applier/applier.go | 5 +- config_applier/applier_test.go | 91 ++++++++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 816afac2..03ff70c2 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -19,6 +19,8 @@ const ( timeoutSingleIteration = time.Second * 30 ) +// TODO: Log ChangeID on every log + var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0"} var ( @@ -166,8 +168,7 @@ func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationCh return nil, err } if cr == nil { - applier.Logger.Info("No CBContainersAgent instance found") - return nil, nil + return nil, fmt.Errorf("no CBContainerAgent instance found, cannot apply change") } // TODO: Validation! diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 11a87859..263ec690 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -184,14 +184,6 @@ func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { return nil }) - //mockClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - // DoAndReturn(func(_ context.Context, item any, _ ...any) error { - // asCb, ok := item.(*cbcontainersv1.CBContainersAgent) - // require.True(t, ok) - // asCb.ObjectMeta.Generation++ - // return nil - // }) - returnedErr := applier.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") @@ -244,10 +236,87 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } -// Update fails, marks as failed +func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockK8sClient := mocks.NewMockClient(ctrl) + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + + applier := config_applier.Applier{ + K8sClient: mockK8sClient, + Logger: logr.Discard(), + Api: mockAPI, + } + + configChange := config_applier.RandomChange() + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + + mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + }, + } + }) + + mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + asCb.ObjectMeta.Generation++ + return nil + }) + + errFromService := errors.New("some error") + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) + + returnedErr := applier.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, errFromService, returnedErr, "expected returned error to match or wrap error from service") +} + +func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() -// Update to backend fails, returns err + mockK8sClient := mocks.NewMockClient(ctrl) + mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) -// TODO: No CR, pending change -> nothing happens but warning? + applier := config_applier.Applier{ + K8sClient: mockK8sClient, + Logger: logr.Discard(), + Api: mockAPI, + } + + configChange := config_applier.RandomChange() + mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + + mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{} + }) + + mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) + + err := applier.RunIteration(context.Background()) + assert.Error(t, err) + // TODO: Specific error exposed for this? +} // Scheduler -> failed; increased retry From 551e1328d6bf8152269eafb561a35dd6fd0c63e6 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 16:31:01 +0300 Subject: [PATCH 16/65] Move "scheduling" logic to a separate struct for config applier --- config_applier/controller.go | 75 +++++++++++++++++++++++++++++++ config_applier/controller_test.go | 5 +++ 2 files changed, 80 insertions(+) create mode 100644 config_applier/controller.go create mode 100644 config_applier/controller_test.go diff --git a/config_applier/controller.go b/config_applier/controller.go new file mode 100644 index 00000000..4c48a16e --- /dev/null +++ b/config_applier/controller.go @@ -0,0 +1,75 @@ +package config_applier + +import ( + "context" + "github.com/go-logr/logr" + "math" + "time" +) + +type configurationApplier interface { + RunIteration(ctx context.Context) error +} + +type RemoteConfigurationController struct { + applier configurationApplier + logger logr.Logger +} + +func NewRemoteConfigurationController(applier configurationApplier, logger logr.Logger) *RemoteConfigurationController { + return &RemoteConfigurationController{applier: applier, logger: logger} +} + +func (controller *RemoteConfigurationController) RunLoop(signalsContext context.Context) { + // TODO: Parameters vs consts + + pollingSleepDuration := 20 * time.Second + pollingTimer := backoffTicker{ + Ticker: time.NewTicker(pollingSleepDuration), + sleepDuration: pollingSleepDuration, + maxRetries: 10, // 1024s or ~17minutes max + } + defer pollingTimer.Stop() + + for { + select { + case <-signalsContext.Done(): + controller.logger.Info("Received cancel signal, turning off configuration applier") + return + case <-pollingTimer.C: + // Nothing to do; this is the polling sleep case + } + err := controller.applier.RunIteration(signalsContext) + + if err != nil { + pollingTimer.resetErr() + } else { + pollingTimer.resetSuccess() + } + } +} + +// backoffTicker is a ticker with exponential backoff for errors and static backoff for success cases +// Note: When calling resetErr or resetSuccess, the ticker will wait the full sleep duration again +type backoffTicker struct { + *time.Ticker + + sleepDuration time.Duration + maxRetries int + + currentRetries int +} + +func (b *backoffTicker) resetErr() { + if b.currentRetries < b.maxRetries { + b.currentRetries++ + } + + nextSleepDuration := time.Duration(math.Pow(2.0, float64(b.currentRetries)))*time.Second + b.sleepDuration + b.Reset(nextSleepDuration) +} + +func (b *backoffTicker) resetSuccess() { + b.currentRetries = 0 + b.Reset(b.sleepDuration) +} diff --git a/config_applier/controller_test.go b/config_applier/controller_test.go new file mode 100644 index 00000000..6e733e45 --- /dev/null +++ b/config_applier/controller_test.go @@ -0,0 +1,5 @@ +package config_applier_test + +// No error -> normal waiting time +// Error encountered -> with backoff +// Respects cancellation From a7f9f72a800d0e3b523e3f322d8ffdf7f44c910e Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 18 Aug 2023 16:53:47 +0300 Subject: [PATCH 17/65] Use a DummyAPI implementation --- config_applier/applier.go | 101 +-------------------------------- config_applier/applier_test.go | 4 ++ config_applier/controller.go | 2 + config_applier/temp.go | 98 ++++++++++++++++++++++++++++++++ main.go | 13 +++-- 5 files changed, 114 insertions(+), 104 deletions(-) create mode 100644 config_applier/temp.go diff --git a/config_applier/applier.go b/config_applier/applier.go index 03ff70c2..27c7de33 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -5,15 +5,14 @@ import ( "fmt" "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "math/rand" "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" "time" ) // TODO: Use interfaces for dependencies? // TODO: Env_var to enable // TODO: Configurable polling interval +// TODO: Recover panics to avoid crashing the operator? const ( timeoutSingleIteration = time.Second * 30 @@ -21,13 +20,6 @@ const ( // TODO: Log ChangeID on every log -var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0"} - -var ( - tr = true - fal = false -) - type ConfigurationAPI interface { // Get Compatibility matrix // Update status of change (ack/error) @@ -44,57 +36,6 @@ type Applier struct { Api ConfigurationAPI } -type pendingChangesResponse struct { - ConfigurationChanges []ConfigurationChange `json:"configuration_changes"` -} - -type ConfigurationChange struct { - ID string `json:"id"` - Status string `json:"status"` - AgentVersion *string `json:"agent_version"` - EnableClusterScanning *bool `json:"enable_cluster_scanning"` - EnableRuntime *bool `json:"enable_runtime"` -} - -type ConfigurationChangeStatusUpdate struct { - ID string `json:"id"` - Status string `json:"status"` - Reason string `json:"reason"` - // AppliedGeneration tracks the generation of the Custom resource where the change was applied - AppliedGeneration int64 `json:"applied_generation"` - // AppliedTimestamp records when the change was applied in RFC3339 format - AppliedTimestamp string `json:"applied_timestamp"` - - // TODO: CLuster and group. Cluster identifier? -} - -type changeStatus string - -var ( - statusPending changeStatus = "PENDING" - statusAcknowledged changeStatus = "ACKNOWLEDGED" // TODO: Acknowledged or applied? - statusFailed changeStatus = "FAILED" -) - -func (applier *Applier) RunLoop(signalsContext context.Context) { - pollingSleepDuration := 20 * time.Second - pollingTimer := time.NewTicker(pollingSleepDuration) - defer pollingTimer.Stop() - - for { - select { - case <-signalsContext.Done(): - applier.Logger.Info("Received cancel signal, turning off configuration applier") - return - case <-pollingTimer.C: - // Nothing to do; this is the polling sleep case - } - // TODO: Pass context down? - applier.Logger.Info("RUNNING ITERATION") - applier.RunIteration(signalsContext) // TODO!!! - } -} - func (applier *Applier) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() @@ -208,43 +149,3 @@ func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv // We don't log a warning if len >=2 as the controller already warns users about that return &cbContainersAgentsList.Items[0], nil } - -func RandomChange() *ConfigurationChange { - csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) - - csRand, runtimeRand, versionRand = 1, 2, 3 - if versionRand == len(versions) { - return nil - } - - changeVersion := &versions[versionRand] - - var changeClusterScanning *bool - var changeRuntime *bool - - switch csRand % 5 { - case 1, 3: - changeClusterScanning = &tr - case 2, 4: - changeClusterScanning = &fal - default: - changeClusterScanning = nil - } - - switch runtimeRand % 5 { - case 1, 3: - changeRuntime = &tr - case 2, 4: - changeRuntime = &fal - default: - changeRuntime = nil - } - - return &ConfigurationChange{ - ID: strconv.Itoa(rand.Int()), - AgentVersion: changeVersion, - EnableClusterScanning: changeClusterScanning, - EnableRuntime: changeRuntime, - Status: string(statusPending), - } -} diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 263ec690..23d13c4c 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -17,6 +17,10 @@ import ( "time" ) +// TODO: Compatibility checks +// TODO: Adding CNDR to the config options +// TODO: Properly handle version + custom image to override the custom image + func setupApplier(ctrl *gomock.Controller, k8sClient client.Client, api config_applier.ConfigurationAPI) config_applier.Applier { if k8sClient == nil { k8sClient = mocks.NewMockClient(ctrl) diff --git a/config_applier/controller.go b/config_applier/controller.go index 4c48a16e..e551167e 100644 --- a/config_applier/controller.go +++ b/config_applier/controller.go @@ -42,8 +42,10 @@ func (controller *RemoteConfigurationController) RunLoop(signalsContext context. err := controller.applier.RunIteration(signalsContext) if err != nil { + controller.logger.Error(err, "Configuration applier iteration failed, will retry again") pollingTimer.resetErr() } else { + controller.logger.Info("Completed configuration applier iteration, sleeping") pollingTimer.resetSuccess() } } diff --git a/config_applier/temp.go b/config_applier/temp.go new file mode 100644 index 00000000..286c3915 --- /dev/null +++ b/config_applier/temp.go @@ -0,0 +1,98 @@ +package config_applier + +import ( + "context" + "math/rand" + "strconv" +) + +var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} + +var ( + tr = true + fal = false +) + +type DummyAPI struct { +} + +func (d DummyAPI) GetConfigurationChanges(ctx context.Context) ([]ConfigurationChange, error) { + c := RandomChange() + if c != nil { + return []ConfigurationChange{*RandomChange()}, nil + + } + return nil, nil +} + +func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update ConfigurationChangeStatusUpdate) error { + return nil +} + +func RandomChange() *ConfigurationChange { + csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) + + //csRand, runtimeRand, versionRand = 1, 2, 3 + if versionRand == len(versions) { + return nil + } + + changeVersion := &versions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = &tr + case 2, 4: + changeClusterScanning = &fal + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = &tr + case 2, 4: + changeRuntime = &fal + default: + changeRuntime = nil + } + + return &ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + Status: string(statusPending), + } +} + +type ConfigurationChange struct { + ID string `json:"id"` + Status string `json:"status"` + AgentVersion *string `json:"agent_version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` +} + +type ConfigurationChangeStatusUpdate struct { + ID string `json:"id"` + Status string `json:"status"` + Reason string `json:"reason"` + // AppliedGeneration tracks the generation of the Custom resource where the change was applied + AppliedGeneration int64 `json:"applied_generation"` + // AppliedTimestamp records when the change was applied in RFC3339 format + AppliedTimestamp string `json:"applied_timestamp"` + + // TODO: CLuster and group. Cluster identifier? +} + +type changeStatus string + +var ( + statusPending changeStatus = "PENDING" + statusAcknowledged changeStatus = "ACKNOWLEDGED" // TODO: Acknowledged or applied? + statusFailed changeStatus = "FAILED" +) diff --git a/main.go b/main.go index 13c7b877..5f7caf85 100644 --- a/main.go +++ b/main.go @@ -32,9 +32,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" - "sync" "strings" - + "sync" "github.com/vmware/cbcontainers-operator/cbcontainers/processors" @@ -169,9 +168,15 @@ func main() { os.Exit(1) } + // TODO: Prettify + // TODO: Check env var + signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() - applier := config_applier.Applier{K8sClient: k8sClient, Logger: ctrl.Log.WithName("configurator")} + log := ctrl.Log.WithName("configurator") + api := config_applier.DummyAPI{} + applier := &config_applier.Applier{K8sClient: k8sClient, Logger: log, Api: api} + applierController := config_applier.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup wg.Add(2) @@ -187,7 +192,7 @@ func main() { go func() { defer wg.Done() setupLog.Info("starting configuration monitor") - applier.RunLoop(signalsContext) + applierController.RunLoop(signalsContext) }() wg.Wait() From 9fe2033557ccbba254bcea9ec48dc14fcdd43465 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 22 Aug 2023 12:44:15 +0300 Subject: [PATCH 18/65] Small refactoring --- config_applier/applier.go | 38 +++-- config_applier/applier_test.go | 153 ++++++------------ config_applier/mocks/generated.go | 2 +- .../mocks/mock_configuration_api.go | 36 ++--- config_applier/temp.go | 9 ++ main.go | 2 +- 6 files changed, 103 insertions(+), 137 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 27c7de33..7ca68833 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -9,10 +9,10 @@ import ( "time" ) -// TODO: Use interfaces for dependencies? // TODO: Env_var to enable // TODO: Configurable polling interval // TODO: Recover panics to avoid crashing the operator? +// TODO: Respect proxy config const ( timeoutSingleIteration = time.Second * 30 @@ -20,7 +20,7 @@ const ( // TODO: Log ChangeID on every log -type ConfigurationAPI interface { +type ConfigurationChangesAPI interface { // Get Compatibility matrix // Update status of change (ack/error) // Get pending changes @@ -31,37 +31,43 @@ type ConfigurationAPI interface { } type Applier struct { - K8sClient client.Client - Logger logr.Logger - Api ConfigurationAPI + k8sClient client.Client + logger logr.Logger + changesAPI ConfigurationChangesAPI } +func NewApplier(k8sClient client.Client, api ConfigurationChangesAPI, logger logr.Logger) *Applier { + return &Applier{k8sClient: k8sClient, logger: logger, changesAPI: api} +} + +//func NewApplier(k8sClient client.) + func (applier *Applier) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() - applier.Logger.Info("Checking for pending remote configuration changes...") + applier.logger.Info("Checking for pending remote configuration changes...") change, errGettingChanges := applier.getPendingChange(ctx) if errGettingChanges != nil { - applier.Logger.Error(errGettingChanges, "Failed to get pending configuration changes") + applier.logger.Error(errGettingChanges, "Failed to get pending configuration changes") return errGettingChanges // TODO } if change == nil { - applier.Logger.Info("No pending remote configuration changes found") + applier.logger.Info("No pending remote configuration changes found") return nil } - applier.Logger.Info("Applying remote configuration change", "change", change) + applier.logger.Info("Applying remote configuration change", "change", change) cr, errApplyingCR := applier.applyChange(ctx, change) if errApplyingCR != nil { - applier.Logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) + applier.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough so we always update the status of the change on the backend, including failed status } if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, errApplyingCR); errStatusUpdate != nil { - applier.Logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") + applier.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") return errStatusUpdate // TODO } @@ -69,7 +75,7 @@ func (applier *Applier) RunIteration(ctx context.Context) error { } func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { - changes, err := applier.Api.GetConfigurationChanges(ctx) + changes, err := applier.changesAPI.GetConfigurationChanges(ctx) if err != nil { return nil, err } @@ -100,7 +106,7 @@ func (applier *Applier) updateChangeStatus(ctx context.Context, change *Configur } } - return applier.Api.UpdateConfigurationChangeStatus(ctx, statusUpdate) + return applier.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) } func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { @@ -125,9 +131,9 @@ func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationCh generationBefore := cr.ObjectMeta.Generation // TODO: Handle Conflict response and retry - err = applier.K8sClient.Update(ctx, cr) + err = applier.k8sClient.Update(ctx, cr) generationAfter := cr.ObjectMeta.Generation - applier.Logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) + applier.logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) return cr, err } @@ -138,7 +144,7 @@ func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} - if err := applier.K8sClient.List(ctx, cbContainersAgentsList); err != nil { + if err := applier.k8sClient.List(ctx, cbContainersAgentsList); err != nil { return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) } diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 23d13c4c..bd82614b 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -8,11 +8,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" + k8sMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" "github.com/vmware/cbcontainers-operator/config_applier" mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" "testing" "time" ) @@ -20,47 +19,38 @@ import ( // TODO: Compatibility checks // TODO: Adding CNDR to the config options // TODO: Properly handle version + custom image to override the custom image +// TODO: Check fields are applied to CR correctly -func setupApplier(ctrl *gomock.Controller, k8sClient client.Client, api config_applier.ConfigurationAPI) config_applier.Applier { - if k8sClient == nil { - k8sClient = mocks.NewMockClient(ctrl) - } - if api == nil { - api = mocksConfigApplier.NewMockConfigurationAPI(ctrl) - } +type applierMocks struct { + k8sClient *k8sMocks.MockClient + api *mocksConfigApplier.MockConfigurationChangesAPI +} - return config_applier.Applier{ - K8sClient: k8sClient, - Logger: logr.Discard(), - Api: api, +func setupApplier(ctrl *gomock.Controller) (*config_applier.Applier, applierMocks) { + k8sClient := k8sMocks.NewMockClient(ctrl) + api := mocksConfigApplier.NewMockConfigurationChangesAPI(ctrl) + + applier := config_applier.NewApplier(k8sClient, api, logr.Discard()) + mocksHolder := applierMocks{ + k8sClient: k8sClient, + api: api, } + + return applier, mocksHolder } func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockK8sClient := mocks.NewMockClient(ctrl) - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) - - //applier := + applier, mocks := setupApplier(ctrl) - applier := config_applier.Applier{ - K8sClient: mockK8sClient, - Logger: logr.Discard(), - Api: mockAPI, - } - - // Once a change appears - // It should find our CR - // It should be applied to the CR - // It should be ACKed with proper CR generation and ID // TODO: Compatiblity check - configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := config_applier.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) - mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{ { @@ -74,7 +64,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { } }) - mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, item any, _ ...any) error { asCb, ok := item.(*cbcontainersv1.CBContainersAgent) require.True(t, ok) @@ -82,7 +72,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, int64(2), update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) @@ -99,8 +89,6 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { } func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() testCases := []struct { name string @@ -123,15 +111,13 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { for _, tC := range testCases { t.Run(tC.name, func(t *testing.T) { - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) - applier := config_applier.Applier{ - K8sClient: nil, - Logger: logr.Discard(), - Api: mockAPI, - } + ctrl := gomock.NewController(t) + defer ctrl.Finish() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) + applier, mocks := setupApplier(ctrl) + + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) err := applier.RunIteration(context.Background()) assert.NoError(t, err) @@ -143,15 +129,10 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) ctrl := gomock.NewController(t) defer ctrl.Finish() - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) - applier := config_applier.Applier{ - K8sClient: nil, - Logger: logr.Discard(), - Api: mockAPI, - } + applier, mocks := setupApplier(ctrl) errFromService := errors.New("some error") - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) returnedErr := applier.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -162,22 +143,15 @@ func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockClient := mocks.NewMockClient(ctrl) - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) - - applier := config_applier.Applier{ - K8sClient: mockClient, - Logger: logr.Discard(), - Api: mockAPI, - } + applier, mocks := setupApplier(ctrl) - configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := config_applier.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) errFromService := errors.New("some error") - mockClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) @@ -197,19 +171,12 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockClient := mocks.NewMockClient(ctrl) - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) - - applier := config_applier.Applier{ - K8sClient: mockClient, - Logger: logr.Discard(), - Api: mockAPI, - } + applier, mocks := setupApplier(ctrl) - configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := config_applier.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) - mockClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{ { @@ -222,9 +189,9 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { }) errFromService := errors.New("some error") - mockClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) @@ -244,19 +211,12 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockK8sClient := mocks.NewMockClient(ctrl) - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + applier, mocks := setupApplier(ctrl) - applier := config_applier.Applier{ - K8sClient: mockK8sClient, - Logger: logr.Discard(), - Api: mockAPI, - } - - configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := config_applier.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) - mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{ { @@ -270,7 +230,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { } }) - mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, item any, _ ...any) error { asCb, ok := item.(*cbcontainersv1.CBContainersAgent) require.True(t, ok) @@ -279,7 +239,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { }) errFromService := errors.New("some error") - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) returnedErr := applier.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -290,24 +250,17 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockK8sClient := mocks.NewMockClient(ctrl) - mockAPI := mocksConfigApplier.NewMockConfigurationAPI(ctrl) + applier, mocks := setupApplier(ctrl) - applier := config_applier.Applier{ - K8sClient: mockK8sClient, - Logger: logr.Discard(), - Api: mockAPI, - } + configChange := config_applier.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) - configChange := config_applier.RandomChange() - mockAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) - - mockK8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{} }) - mockAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) @@ -322,5 +275,3 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { assert.Error(t, err) // TODO: Specific error exposed for this? } - -// Scheduler -> failed; increased retry diff --git a/config_applier/mocks/generated.go b/config_applier/mocks/generated.go index 96ace6a2..7a354304 100644 --- a/config_applier/mocks/generated.go +++ b/config_applier/mocks/generated.go @@ -1,3 +1,3 @@ package mocks -//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/config_applier ConfigurationAPI +//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/config_applier ConfigurationChangesAPI diff --git a/config_applier/mocks/mock_configuration_api.go b/config_applier/mocks/mock_configuration_api.go index b0dfc6fd..82b04050 100644 --- a/config_applier/mocks/mock_configuration_api.go +++ b/config_applier/mocks/mock_configuration_api.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/config_applier (interfaces: ConfigurationAPI) +// Source: github.com/vmware/cbcontainers-operator/config_applier (interfaces: ConfigurationChangesAPI) // Package mocks is a generated GoMock package. package mocks @@ -12,31 +12,31 @@ import ( config_applier "github.com/vmware/cbcontainers-operator/config_applier" ) -// MockConfigurationAPI is a mock of ConfigurationAPI interface. -type MockConfigurationAPI struct { +// MockConfigurationChangesAPI is a mock of ConfigurationChangesAPI interface. +type MockConfigurationChangesAPI struct { ctrl *gomock.Controller - recorder *MockConfigurationAPIMockRecorder + recorder *MockConfigurationChangesAPIMockRecorder } -// MockConfigurationAPIMockRecorder is the mock recorder for MockConfigurationAPI. -type MockConfigurationAPIMockRecorder struct { - mock *MockConfigurationAPI +// MockConfigurationChangesAPIMockRecorder is the mock recorder for MockConfigurationChangesAPI. +type MockConfigurationChangesAPIMockRecorder struct { + mock *MockConfigurationChangesAPI } -// NewMockConfigurationAPI creates a new mock instance. -func NewMockConfigurationAPI(ctrl *gomock.Controller) *MockConfigurationAPI { - mock := &MockConfigurationAPI{ctrl: ctrl} - mock.recorder = &MockConfigurationAPIMockRecorder{mock} +// NewMockConfigurationChangesAPI creates a new mock instance. +func NewMockConfigurationChangesAPI(ctrl *gomock.Controller) *MockConfigurationChangesAPI { + mock := &MockConfigurationChangesAPI{ctrl: ctrl} + mock.recorder = &MockConfigurationChangesAPIMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConfigurationAPI) EXPECT() *MockConfigurationAPIMockRecorder { +func (m *MockConfigurationChangesAPI) EXPECT() *MockConfigurationChangesAPIMockRecorder { return m.recorder } // GetConfigurationChanges mocks base method. -func (m *MockConfigurationAPI) GetConfigurationChanges(arg0 context.Context) ([]config_applier.ConfigurationChange, error) { +func (m *MockConfigurationChangesAPI) GetConfigurationChanges(arg0 context.Context) ([]config_applier.ConfigurationChange, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) ret0, _ := ret[0].([]config_applier.ConfigurationChange) @@ -45,13 +45,13 @@ func (m *MockConfigurationAPI) GetConfigurationChanges(arg0 context.Context) ([] } // GetConfigurationChanges indicates an expected call of GetConfigurationChanges. -func (mr *MockConfigurationAPIMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { +func (mr *MockConfigurationChangesAPIMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationAPI)(nil).GetConfigurationChanges), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationChangesAPI)(nil).GetConfigurationChanges), arg0) } // UpdateConfigurationChangeStatus mocks base method. -func (m *MockConfigurationAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 config_applier.ConfigurationChangeStatusUpdate) error { +func (m *MockConfigurationChangesAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 config_applier.ConfigurationChangeStatusUpdate) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) ret0, _ := ret[0].(error) @@ -59,7 +59,7 @@ func (m *MockConfigurationAPI) UpdateConfigurationChangeStatus(arg0 context.Cont } // UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. -func (mr *MockConfigurationAPIMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockConfigurationChangesAPIMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationAPI)(nil).UpdateConfigurationChangeStatus), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationChangesAPI)(nil).UpdateConfigurationChangeStatus), arg0, arg1) } diff --git a/config_applier/temp.go b/config_applier/temp.go index 286c3915..bd522f3a 100644 --- a/config_applier/temp.go +++ b/config_applier/temp.go @@ -29,6 +29,15 @@ func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update Co return nil } +func RandomNonNilChange() *ConfigurationChange { + for { + c := RandomChange() + if c != nil { + return c + } + } +} + func RandomChange() *ConfigurationChange { csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) diff --git a/main.go b/main.go index 5f7caf85..7d13cea2 100644 --- a/main.go +++ b/main.go @@ -175,7 +175,7 @@ func main() { k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") api := config_applier.DummyAPI{} - applier := &config_applier.Applier{K8sClient: k8sClient, Logger: log, Api: api} + applier := config_applier.NewApplier(k8sClient, api, log) applierController := config_applier.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup From aa5fd69d303afb2f3c7c8bc13e9fde75df172bf9 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 22 Aug 2023 17:12:20 +0300 Subject: [PATCH 19/65] Add test for feature toggles --- config_applier/applier.go | 6 ++ config_applier/applier_test.go | 145 +++++++++++++++++++++++++++++++++ config_applier/temp.go | 1 + 3 files changed, 152 insertions(+) diff --git a/config_applier/applier.go b/config_applier/applier.go index 7ca68833..9996008d 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -128,6 +128,12 @@ func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationCh if change.EnableRuntime != nil { cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime } + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } generationBefore := cr.ObjectMeta.Generation // TODO: Handle Conflict response and retry diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index bd82614b..c4e1b366 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -3,6 +3,7 @@ package config_applier_test import ( "context" "errors" + "fmt" "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -12,6 +13,8 @@ import ( "github.com/vmware/cbcontainers-operator/config_applier" mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "math/rand" + "strconv" "testing" "time" ) @@ -21,6 +24,19 @@ import ( // TODO: Properly handle version + custom image to override the custom image // TODO: Check fields are applied to CR correctly +// TODO: Reads cluster, etc from CR correctly? +// TODO: Respects proxy +// TODO: Updates img with version +// TODO: Review gomock.any usages here +// TODO: version tests + +var ( + trueV = true + truePtr = &trueV + falseV = false + falsePtr = &falseV +) + type applierMocks struct { k8sClient *k8sMocks.MockClient api *mocksConfigApplier.MockConfigurationChangesAPI @@ -125,6 +141,121 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { } } +func TestCRFieldsAreChangedCorrectlyBasedOnRemoteChange(t *testing.T) { + type appliedChangeTest struct { + name string + change config_applier.ConfigurationChange + initialCR cbcontainersv1.CBContainersAgent + assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) + } + + // generateFeatureToggleTestCases produces a set of tests for a single feature toggle in the requested change + // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) + generateFeatureToggleTestCases := + func(feature string, + changeFieldSelector func(*config_applier.ConfigurationChange) **bool, + crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { + + var result []appliedChangeTest + + for _, crState := range []*bool{truePtr, falsePtr, nil} { + cr := cbcontainersv1.CBContainersAgent{} + crFieldPtr := crFieldSelector(&cr) + *crFieldPtr = crState + + // Validate that each toggle state works (or doesn't do anything when it matches) + for _, changeState := range []*bool{truePtr, falsePtr} { + change := createPendingChange() + changeFieldPtr := changeFieldSelector(&change) + *changeFieldPtr = changeState + + expectedState := changeState // avoid closure issues + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("toggle feature (%s) from (%v) to (%v)", feature, prettyPrintBoolPtr(crState), prettyPrintBoolPtr(changeState)), + change: change, + initialCR: cr, + assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { + crFieldPostChangePtr := crFieldSelector(agent) + assert.Equal(t, expectedState, *crFieldPostChangePtr) + }, + }) + } + + // Validate that a change with the toggle unset does not modify the CR + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), + change: createPendingChange(), + initialCR: cr, + assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { + crFieldPostChangePtr := crFieldSelector(agent) + assert.Equal(t, *crFieldPtr, *crFieldPostChangePtr) + }, + }) + } + + return result + } + + var testCases []appliedChangeTest + + clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", + func(change *config_applier.ConfigurationChange) **bool { + return &change.EnableClusterScanning + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + return &agent.Spec.Components.ClusterScanning.Enabled + }) + + runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", + func(change *config_applier.ConfigurationChange) **bool { + return &change.EnableRuntime + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + return &agent.Spec.Components.RuntimeProtection.Enabled + }) + + cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", + func(change *config_applier.ConfigurationChange) **bool { + return &change.EnableCNDR + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + if agent.Spec.Components.Cndr == nil { + agent.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + return &agent.Spec.Components.Cndr.Enabled + }) + + testCases = append(testCases, clusterScannerToggleTestCases...) + testCases = append(testCases, runtimeToggleTestCases...) + testCases = append(testCases, cndrToggleTestCases...) + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + applier, mocks := setupApplier(ctrl) + + changesList := []config_applier.ConfigurationChange{testCase.change} + cr := testCase.initialCR + + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(changesList, nil) + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{cr} + }) + + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + + testCase.assertFinalCR(t, asCb) + return nil + }) + + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(nil) + + assert.NoError(t, applier.RunIteration(context.Background())) + }) + } +} + func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -275,3 +406,17 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { assert.Error(t, err) // TODO: Specific error exposed for this? } + +func createPendingChange() config_applier.ConfigurationChange { + return config_applier.ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + Status: "PENDING", + } +} + +func prettyPrintBoolPtr(v *bool) string { + if v == nil { + return "" + } + return fmt.Sprintf("%t", *v) +} diff --git a/config_applier/temp.go b/config_applier/temp.go index bd522f3a..b0fa9b22 100644 --- a/config_applier/temp.go +++ b/config_applier/temp.go @@ -84,6 +84,7 @@ type ConfigurationChange struct { AgentVersion *string `json:"agent_version"` EnableClusterScanning *bool `json:"enable_cluster_scanning"` EnableRuntime *bool `json:"enable_runtime"` + EnableCNDR *bool `json:"enable_cndr"` } type ConfigurationChangeStatusUpdate struct { From 43b671b7a3b46c0cc92a994332db293df4417f4b Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 23 Aug 2023 13:57:37 +0300 Subject: [PATCH 20/65] Add tests for the version field. Split the modification logic in a separate helper function to make testing easier --- config_applier/applier.go | 34 +---- config_applier/applier_test.go | 143 +----------------- config_applier/change_applier.go | 40 +++++ config_applier/change_applier_test.go | 210 ++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 167 deletions(-) create mode 100644 config_applier/change_applier.go create mode 100644 config_applier/change_applier_test.go diff --git a/config_applier/applier.go b/config_applier/applier.go index 9996008d..7f494c31 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -22,9 +22,6 @@ const ( type ConfigurationChangesAPI interface { // Get Compatibility matrix - // Update status of change (ack/error) - // Get pending changes - // Set status for change (acknowledge/error) GetConfigurationChanges(context.Context) ([]ConfigurationChange, error) UpdateConfigurationChangeStatus(context.Context, ConfigurationChangeStatusUpdate) error @@ -40,8 +37,6 @@ func NewApplier(k8sClient client.Client, api ConfigurationChangesAPI, logger log return &Applier{k8sClient: k8sClient, logger: logger, changesAPI: api} } -//func NewApplier(k8sClient client.) - func (applier *Applier) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() @@ -60,13 +55,13 @@ func (applier *Applier) RunIteration(ctx context.Context) error { } applier.logger.Info("Applying remote configuration change", "change", change) - cr, errApplyingCR := applier.applyChange(ctx, change) + cr, errApplyingCR := applier.applyChange(ctx, *change) if errApplyingCR != nil { applier.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough so we always update the status of the change on the backend, including failed status } - if errStatusUpdate := applier.updateChangeStatus(ctx, change, cr, errApplyingCR); errStatusUpdate != nil { + if errStatusUpdate := applier.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { applier.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") return errStatusUpdate // TODO } @@ -88,7 +83,7 @@ func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationCha return nil, nil } -func (applier *Applier) updateChangeStatus(ctx context.Context, change *ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { +func (applier *Applier) updateChangeStatus(ctx context.Context, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { var statusUpdate ConfigurationChangeStatusUpdate if encounteredError == nil { statusUpdate = ConfigurationChangeStatusUpdate{ @@ -109,7 +104,7 @@ func (applier *Applier) updateChangeStatus(ctx context.Context, change *Configur return applier.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) } -func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { +func (applier *Applier) applyChange(ctx context.Context, change ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { cr, err := applier.getContainerAgentCR(ctx) if err != nil { return nil, err @@ -118,27 +113,14 @@ func (applier *Applier) applyChange(ctx context.Context, change *ConfigurationCh return nil, fmt.Errorf("no CBContainerAgent instance found, cannot apply change") } - // TODO: Validation! - if change.AgentVersion != nil { - cr.Spec.Version = *change.AgentVersion - } - if change.EnableClusterScanning != nil { - cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning - } - if change.EnableRuntime != nil { - cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime - } - if change.EnableCNDR != nil { - if cr.Spec.Components.Cndr == nil { - cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} - } - cr.Spec.Components.Cndr.Enabled = change.EnableCNDR - } + applyChange(change, cr) generationBefore := cr.ObjectMeta.Generation - // TODO: Handle Conflict response and retry + err = applier.k8sClient.Update(ctx, cr) generationAfter := cr.ObjectMeta.Generation + + // TODO: remove applier.logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) return cr, err } diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index c4e1b366..6b9a6f36 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -3,7 +3,6 @@ package config_applier_test import ( "context" "errors" - "fmt" "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -13,8 +12,6 @@ import ( "github.com/vmware/cbcontainers-operator/config_applier" mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "math/rand" - "strconv" "testing" "time" ) @@ -26,16 +23,9 @@ import ( // TODO: Reads cluster, etc from CR correctly? // TODO: Respects proxy -// TODO: Updates img with version // TODO: Review gomock.any usages here // TODO: version tests - -var ( - trueV = true - truePtr = &trueV - falseV = false - falsePtr = &falseV -) +// TODO: Multiple changes are applied according to timestamp type applierMocks struct { k8sClient *k8sMocks.MockClient @@ -84,6 +74,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { DoAndReturn(func(_ context.Context, item any, _ ...any) error { asCb, ok := item.(*cbcontainersv1.CBContainersAgent) require.True(t, ok) + require.Equal(t, *configChange.AgentVersion, asCb.Spec.Version) asCb.ObjectMeta.Generation++ return nil }) @@ -105,7 +96,6 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { } func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { - testCases := []struct { name string dataFromService []config_applier.ConfigurationChange @@ -141,121 +131,6 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { } } -func TestCRFieldsAreChangedCorrectlyBasedOnRemoteChange(t *testing.T) { - type appliedChangeTest struct { - name string - change config_applier.ConfigurationChange - initialCR cbcontainersv1.CBContainersAgent - assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) - } - - // generateFeatureToggleTestCases produces a set of tests for a single feature toggle in the requested change - // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) - generateFeatureToggleTestCases := - func(feature string, - changeFieldSelector func(*config_applier.ConfigurationChange) **bool, - crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { - - var result []appliedChangeTest - - for _, crState := range []*bool{truePtr, falsePtr, nil} { - cr := cbcontainersv1.CBContainersAgent{} - crFieldPtr := crFieldSelector(&cr) - *crFieldPtr = crState - - // Validate that each toggle state works (or doesn't do anything when it matches) - for _, changeState := range []*bool{truePtr, falsePtr} { - change := createPendingChange() - changeFieldPtr := changeFieldSelector(&change) - *changeFieldPtr = changeState - - expectedState := changeState // avoid closure issues - result = append(result, appliedChangeTest{ - name: fmt.Sprintf("toggle feature (%s) from (%v) to (%v)", feature, prettyPrintBoolPtr(crState), prettyPrintBoolPtr(changeState)), - change: change, - initialCR: cr, - assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { - crFieldPostChangePtr := crFieldSelector(agent) - assert.Equal(t, expectedState, *crFieldPostChangePtr) - }, - }) - } - - // Validate that a change with the toggle unset does not modify the CR - result = append(result, appliedChangeTest{ - name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), - change: createPendingChange(), - initialCR: cr, - assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { - crFieldPostChangePtr := crFieldSelector(agent) - assert.Equal(t, *crFieldPtr, *crFieldPostChangePtr) - }, - }) - } - - return result - } - - var testCases []appliedChangeTest - - clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", - func(change *config_applier.ConfigurationChange) **bool { - return &change.EnableClusterScanning - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { - return &agent.Spec.Components.ClusterScanning.Enabled - }) - - runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", - func(change *config_applier.ConfigurationChange) **bool { - return &change.EnableRuntime - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { - return &agent.Spec.Components.RuntimeProtection.Enabled - }) - - cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", - func(change *config_applier.ConfigurationChange) **bool { - return &change.EnableCNDR - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { - if agent.Spec.Components.Cndr == nil { - agent.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} - } - return &agent.Spec.Components.Cndr.Enabled - }) - - testCases = append(testCases, clusterScannerToggleTestCases...) - testCases = append(testCases, runtimeToggleTestCases...) - testCases = append(testCases, cndrToggleTestCases...) - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - applier, mocks := setupApplier(ctrl) - - changesList := []config_applier.ConfigurationChange{testCase.change} - cr := testCase.initialCR - - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(changesList, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{cr} - }) - - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, item any, _ ...any) error { - asCb, ok := item.(*cbcontainersv1.CBContainersAgent) - require.True(t, ok) - - testCase.assertFinalCR(t, asCb) - return nil - }) - - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(nil) - - assert.NoError(t, applier.RunIteration(context.Background())) - }) - } -} - func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -406,17 +281,3 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { assert.Error(t, err) // TODO: Specific error exposed for this? } - -func createPendingChange() config_applier.ConfigurationChange { - return config_applier.ConfigurationChange{ - ID: strconv.Itoa(rand.Int()), - Status: "PENDING", - } -} - -func prettyPrintBoolPtr(v *bool) string { - if v == nil { - return "" - } - return fmt.Sprintf("%t", *v) -} diff --git a/config_applier/change_applier.go b/config_applier/change_applier.go new file mode 100644 index 00000000..f311b0b5 --- /dev/null +++ b/config_applier/change_applier.go @@ -0,0 +1,40 @@ +package config_applier + +import cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + +func applyChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { + // TODO: Validation? + + resetVersion := func(ptrToField *string) { + if ptrToField != nil && *ptrToField != "" { + *ptrToField = "" + } + } + + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion + + resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + if cr.Spec.Components.Cndr != nil { + resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) + } + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } +} diff --git a/config_applier/change_applier_test.go b/config_applier/change_applier_test.go new file mode 100644 index 00000000..828caa69 --- /dev/null +++ b/config_applier/change_applier_test.go @@ -0,0 +1,210 @@ +package config_applier + +import ( + "fmt" + "github.com/stretchr/testify/assert" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "testing" +) + +var ( + trueV = true + truePtr = &trueV + falseV = false + falsePtr = &falseV +) + +func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { + type appliedChangeTest struct { + name string + change ConfigurationChange + initialCR cbcontainersv1.CBContainersAgent + assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) + } + + // generateFeatureToggleTestCases produces a set of tests for a single feature toggle in the requested change + // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) + generateFeatureToggleTestCases := + func(feature string, + changeFieldSelector func(*ConfigurationChange) **bool, + crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { + + var result []appliedChangeTest + + for _, crState := range []*bool{truePtr, falsePtr, nil} { + cr := cbcontainersv1.CBContainersAgent{} + crFieldPtr := crFieldSelector(&cr) + *crFieldPtr = crState + + // Validate that each toggle state works (or doesn't do anything when it matches) + for _, changeState := range []*bool{truePtr, falsePtr} { + change := ConfigurationChange{} + changeFieldPtr := changeFieldSelector(&change) + *changeFieldPtr = changeState + + expectedState := changeState // avoid closure issues + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("toggle feature (%s) from (%v) to (%v)", feature, prettyPrintBoolPtr(crState), prettyPrintBoolPtr(changeState)), + change: change, + initialCR: cr, + assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { + crFieldPostChangePtr := crFieldSelector(agent) + assert.Equal(t, expectedState, *crFieldPostChangePtr) + }, + }) + } + + // Validate that a change with the toggle unset does not modify the CR + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), + change: ConfigurationChange{}, + initialCR: cr, + assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { + crFieldPostChangePtr := crFieldSelector(agent) + assert.Equal(t, *crFieldPtr, *crFieldPostChangePtr) + }, + }) + } + + return result + } + + var testCases []appliedChangeTest + + clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", + func(change *ConfigurationChange) **bool { + return &change.EnableClusterScanning + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + return &agent.Spec.Components.ClusterScanning.Enabled + }) + + runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", + func(change *ConfigurationChange) **bool { + return &change.EnableRuntime + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + return &agent.Spec.Components.RuntimeProtection.Enabled + }) + + cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", + func(change *ConfigurationChange) **bool { + return &change.EnableCNDR + }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + if agent.Spec.Components.Cndr == nil { + agent.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + return &agent.Spec.Components.Cndr.Enabled + }) + + testCases = append(testCases, clusterScannerToggleTestCases...) + testCases = append(testCases, runtimeToggleTestCases...) + testCases = append(testCases, cndrToggleTestCases...) + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + applyChange(testCase.change, &testCase.initialCR) + testCase.assertFinalCR(t, &testCase.initialCR) + }) + } +} + +func TestVersionIsAppliedCorrectly(t *testing.T) { + originalVersion := "my-version-42" + newVersion := "new-version" + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} + change := ConfigurationChange{AgentVersion: &newVersion} + + applyChange(change, &cr) + assert.Equal(t, newVersion, cr.Spec.Version) +} + +func TestMissingVersionDoesNotModifyCR(t *testing.T) { + originalVersion := "my-version-42" + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} + change := ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} + + applyChange(change, &cr) + assert.Equal(t, originalVersion, cr.Spec.Version) + +} + +func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { + cr := cbcontainersv1.CBContainersAgent{ + Spec: cbcontainersv1.CBContainersAgentSpec{ + Version: "some-version", + Components: cbcontainersv1.CBContainersComponentsSpec{ + Basic: cbcontainersv1.CBContainersBasicSpec{ + Enforcer: cbcontainersv1.CBContainersEnforcerSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-enforcer", + }, + }, + StateReporter: cbcontainersv1.CBContainersStateReporterSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-state-repoter", + }, + }, + Monitor: cbcontainersv1.CBContainersMonitorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-monitor", + }, + }, + }, + RuntimeProtection: cbcontainersv1.CBContainersRuntimeProtectionSpec{ + Resolver: cbcontainersv1.CBContainersRuntimeResolverSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-runtime-resolver", + }, + }, + Sensor: cbcontainersv1.CBContainersRuntimeSensorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-runtime-sensor", + }, + }, + }, + Cndr: &cbcontainersv1.CBContainersCndrSpec{ + Sensor: cbcontainersv1.CBContainersCndrSensorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-cndr-sensor", + }, + }, + }, + ClusterScanning: cbcontainersv1.CBContainersClusterScanningSpec{ + ClusterScannerAgent: cbcontainersv1.CBContainersClusterScannerAgentSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-cluster-scanning-agent", + }, + }, + ImageScanningReporter: cbcontainersv1.CBContainersImageScanningReporterSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-image-scanning-reporter", + }, + }, + }, + }, + }, + } + + newVersion := "new-version" + change := ConfigurationChange{AgentVersion: &newVersion} + + applyChange(change, &cr) + + assert.Equal(t, newVersion, cr.Spec.Version) + // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields + // => the operator will use the common version instead + assert.Empty(t, cr.Spec.Components.Basic.Monitor.Image.Tag) + assert.Empty(t, cr.Spec.Components.Basic.Enforcer.Image.Tag) + assert.Empty(t, cr.Spec.Components.Basic.StateReporter.Image.Tag) + assert.Empty(t, cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + assert.Empty(t, cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + assert.Empty(t, cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + assert.Empty(t, cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + assert.Empty(t, cr.Spec.Components.Cndr.Sensor.Image.Tag) +} + +func prettyPrintBoolPtr(v *bool) string { + if v == nil { + return "" + } + return fmt.Sprintf("%t", *v) +} From cfb3216bd07137832c367b71553f9bf889b5961d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 23 Aug 2023 17:09:56 +0300 Subject: [PATCH 21/65] Sort the change slice in case there are multiple pending changes --- config_applier/applier.go | 7 ++++- config_applier/applier_test.go | 50 ++++++++++++++++++++++++++++++++++ config_applier/temp.go | 1 + 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 7f494c31..9eefa430 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -6,6 +6,7 @@ import ( "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sort" "time" ) @@ -75,6 +76,10 @@ func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationCha return nil, err } + sort.SliceStable(changes, func(i, j int) bool { + return changes[i].Timestamp < changes[j].Timestamp + }) + for _, change := range changes { if change.Status == string(statusPending) { return &change, nil @@ -136,7 +141,7 @@ func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) } - if cbContainersAgentsList.Items == nil || len(cbContainersAgentsList.Items) == 0 { + if len(cbContainersAgentsList.Items) == 0 { return nil, nil } diff --git a/config_applier/applier_test.go b/config_applier/applier_test.go index 6b9a6f36..e3fe9192 100644 --- a/config_applier/applier_test.go +++ b/config_applier/applier_test.go @@ -131,6 +131,56 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { } } +func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + applier, mocks := setupApplier(ctrl) + + olderChange := config_applier.RandomNonNilChange() + newerChange := config_applier.RandomNonNilChange() + + expectedVersion := "version-for-older-change" + versionThatShouldNotBe := "version-for-newer-change" + olderChange.AgentVersion = &expectedVersion + newerChange.AgentVersion = &versionThatShouldNotBe + olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) + newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) + + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*newerChange, *olderChange}, nil) + + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + }, + } + }) + + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + + assert.Equal(t, expectedVersion, asCb.Spec.Version) + return nil + }) + + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, olderChange.ID, update.ID) + return nil + }) + + err := applier.RunIteration(context.Background()) + assert.NoError(t, err) +} + func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/config_applier/temp.go b/config_applier/temp.go index b0fa9b22..23ec9c31 100644 --- a/config_applier/temp.go +++ b/config_applier/temp.go @@ -85,6 +85,7 @@ type ConfigurationChange struct { EnableClusterScanning *bool `json:"enable_cluster_scanning"` EnableRuntime *bool `json:"enable_runtime"` EnableCNDR *bool `json:"enable_cndr"` + Timestamp string `json:"timestamp"` } type ConfigurationChangeStatusUpdate struct { From 3586a4c97cdc6a478de32a2a39ab714ce598826a Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 13:59:40 +0300 Subject: [PATCH 22/65] Clearing TODOs, small fixes --- config_applier/applier.go | 12 ++---------- config_applier/change_applier.go | 2 +- config_applier/change_applier_test.go | 8 ++++---- config_applier/temp.go | 12 ++++++++++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index 9eefa430..b18dede0 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -19,8 +19,6 @@ const ( timeoutSingleIteration = time.Second * 30 ) -// TODO: Log ChangeID on every log - type ConfigurationChangesAPI interface { // Get Compatibility matrix @@ -59,7 +57,7 @@ func (applier *Applier) RunIteration(ctx context.Context) error { cr, errApplyingCR := applier.applyChange(ctx, *change) if errApplyingCR != nil { applier.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) - // Intentional fallthrough so we always update the status of the change on the backend, including failed status + // Intentional fallthrough as we always update the status of the change on the backend, including failed status } if errStatusUpdate := applier.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { @@ -118,15 +116,9 @@ func (applier *Applier) applyChange(ctx context.Context, change ConfigurationCha return nil, fmt.Errorf("no CBContainerAgent instance found, cannot apply change") } - applyChange(change, cr) - - generationBefore := cr.ObjectMeta.Generation + applyChangesToCR(change, cr) err = applier.k8sClient.Update(ctx, cr) - generationAfter := cr.ObjectMeta.Generation - - // TODO: remove - applier.logger.Info("Updated object", "oldGeneration", generationBefore, "newGeneration", generationAfter, "err", err) return cr, err } diff --git a/config_applier/change_applier.go b/config_applier/change_applier.go index f311b0b5..e1e01a4b 100644 --- a/config_applier/change_applier.go +++ b/config_applier/change_applier.go @@ -2,7 +2,7 @@ package config_applier import cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" -func applyChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { +func applyChangesToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { // TODO: Validation? resetVersion := func(ptrToField *string) { diff --git a/config_applier/change_applier_test.go b/config_applier/change_applier_test.go index 828caa69..e0ff9a2f 100644 --- a/config_applier/change_applier_test.go +++ b/config_applier/change_applier_test.go @@ -101,7 +101,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - applyChange(testCase.change, &testCase.initialCR) + applyChangesToCR(testCase.change, &testCase.initialCR) testCase.assertFinalCR(t, &testCase.initialCR) }) } @@ -113,7 +113,7 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := ConfigurationChange{AgentVersion: &newVersion} - applyChange(change, &cr) + applyChangesToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) } @@ -122,7 +122,7 @@ func TestMissingVersionDoesNotModifyCR(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} - applyChange(change, &cr) + applyChangesToCR(change, &cr) assert.Equal(t, originalVersion, cr.Spec.Version) } @@ -187,7 +187,7 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { newVersion := "new-version" change := ConfigurationChange{AgentVersion: &newVersion} - applyChange(change, &cr) + applyChangesToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields diff --git a/config_applier/temp.go b/config_applier/temp.go index 23ec9c31..60f1140e 100644 --- a/config_applier/temp.go +++ b/config_applier/temp.go @@ -19,7 +19,7 @@ type DummyAPI struct { func (d DummyAPI) GetConfigurationChanges(ctx context.Context) ([]ConfigurationChange, error) { c := RandomChange() if c != nil { - return []ConfigurationChange{*RandomChange()}, nil + return []ConfigurationChange{*c}, nil } return nil, nil @@ -39,7 +39,7 @@ func RandomNonNilChange() *ConfigurationChange { } func RandomChange() *ConfigurationChange { - csRand, runtimeRand, versionRand := rand.Int(), rand.Int(), rand.Intn(len(versions)+1) + csRand, runtimeRand, cndrRand, versionRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)+1) //csRand, runtimeRand, versionRand = 1, 2, 3 if versionRand == len(versions) { @@ -50,6 +50,7 @@ func RandomChange() *ConfigurationChange { var changeClusterScanning *bool var changeRuntime *bool + var changeCNDR *bool switch csRand % 5 { case 1, 3: @@ -69,11 +70,18 @@ func RandomChange() *ConfigurationChange { changeRuntime = nil } + if changeVersion != nil && *changeVersion == "3.0.0" && cndrRand%2 == 0 { + changeCNDR = &tr + } else { + changeCNDR = &fal + } + return &ConfigurationChange{ ID: strconv.Itoa(rand.Int()), AgentVersion: changeVersion, EnableClusterScanning: changeClusterScanning, EnableRuntime: changeRuntime, + EnableCNDR: changeCNDR, Status: string(statusPending), } } From e2bb24b67a5c2831d9af7f7f03bd1b1c933f687f Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 14:09:43 +0300 Subject: [PATCH 23/65] Removing more TODOs --- config_applier/applier.go | 8 +++----- config_applier/controller.go | 16 +++++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index b18dede0..a54a8325 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -11,12 +11,10 @@ import ( ) // TODO: Env_var to enable -// TODO: Configurable polling interval -// TODO: Recover panics to avoid crashing the operator? // TODO: Respect proxy config const ( - timeoutSingleIteration = time.Second * 30 + timeoutSingleIteration = time.Second * 60 ) type ConfigurationChangesAPI interface { @@ -45,7 +43,7 @@ func (applier *Applier) RunIteration(ctx context.Context) error { change, errGettingChanges := applier.getPendingChange(ctx) if errGettingChanges != nil { applier.logger.Error(errGettingChanges, "Failed to get pending configuration changes") - return errGettingChanges // TODO + return errGettingChanges } if change == nil { @@ -62,7 +60,7 @@ func (applier *Applier) RunIteration(ctx context.Context) error { if errStatusUpdate := applier.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { applier.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") - return errStatusUpdate // TODO + return errStatusUpdate } return errApplyingCR diff --git a/config_applier/controller.go b/config_applier/controller.go index e551167e..4b65d821 100644 --- a/config_applier/controller.go +++ b/config_applier/controller.go @@ -7,6 +7,11 @@ import ( "time" ) +const ( + sleepDuration = 20 * time.Second + maxRetries = 10 // 1024s or ~17 minutes at peak +) + type configurationApplier interface { RunIteration(ctx context.Context) error } @@ -21,13 +26,10 @@ func NewRemoteConfigurationController(applier configurationApplier, logger logr. } func (controller *RemoteConfigurationController) RunLoop(signalsContext context.Context) { - // TODO: Parameters vs consts - - pollingSleepDuration := 20 * time.Second pollingTimer := backoffTicker{ - Ticker: time.NewTicker(pollingSleepDuration), - sleepDuration: pollingSleepDuration, - maxRetries: 10, // 1024s or ~17minutes max + Ticker: time.NewTicker(sleepDuration), + sleepDuration: sleepDuration, + maxRetries: maxRetries, } defer pollingTimer.Stop() @@ -42,7 +44,7 @@ func (controller *RemoteConfigurationController) RunLoop(signalsContext context. err := controller.applier.RunIteration(signalsContext) if err != nil { - controller.logger.Error(err, "Configuration applier iteration failed, will retry again") + controller.logger.Error(err, "Configuration applier iteration failed, it will be retried on next iteration period") pollingTimer.resetErr() } else { controller.logger.Info("Completed configuration applier iteration, sleeping") From cbbc1b8d86a18afc38a97dbb3b57a0e60f4e1518 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 14:18:50 +0300 Subject: [PATCH 24/65] Turn off configurator by default and use env var to enable --- config_applier/applier.go | 1 - main.go | 23 +++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/config_applier/applier.go b/config_applier/applier.go index a54a8325..eea1f7c2 100644 --- a/config_applier/applier.go +++ b/config_applier/applier.go @@ -10,7 +10,6 @@ import ( "time" ) -// TODO: Env_var to enable // TODO: Respect proxy config const ( diff --git a/main.go b/main.go index 7d13cea2..ad4aea6c 100644 --- a/main.go +++ b/main.go @@ -58,10 +58,12 @@ var ( ) const ( - NamespaceIdentifier = "default" - httpProxyEnv = "HTTP_PROXY" - httpsProxyEnv = "HTTPS_PROXY" - noProxyEnv = "NO_PROXY" + NamespaceIdentifier = "default" + httpProxyEnv = "HTTP_PROXY" + httpsProxyEnv = "HTTPS_PROXY" + noProxyEnv = "NO_PROXY" + namespaceEnv = "OPERATOR_NAMESPACE" + enableRemoteConfiguratorEnv = "ENABLE_REMOTE_CONFIGURATOR" ) func init() { @@ -114,7 +116,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog.Info("Getting the namespace where operator is running and which should host the agent") - operatorNamespace := os.Getenv("OPERATOR_NAMESPACE") + operatorNamespace := os.Getenv(namespaceEnv) if operatorNamespace == "" { setupLog.Info(fmt.Sprintf("Operator namespace variable was not found. Falling back to default %s", common.DataPlaneNamespaceName)) operatorNamespace = common.DataPlaneNamespaceName @@ -169,7 +171,6 @@ func main() { } // TODO: Prettify - // TODO: Check env var signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() @@ -191,8 +192,14 @@ func main() { }() go func() { defer wg.Done() - setupLog.Info("starting configuration monitor") - applierController.RunLoop(signalsContext) + + enableConfigurator := os.Getenv(enableRemoteConfiguratorEnv) + if enableConfigurator == "true" { + setupLog.Info("Starting remote configurator") + applierController.RunLoop(signalsContext) + } else { + setupLog.Info(fmt.Sprintf("Environment variable %s is not set to true, remote configuration feature will be disabled", enableRemoteConfiguratorEnv)) + } }() wg.Wait() From 17051ce11db823504404c3c29742bca9c3242a96 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 14:23:33 +0300 Subject: [PATCH 25/65] Renaming things --- main.go | 8 +- .../change_applier.go | 2 +- .../change_applier_test.go | 2 +- .../configurator.go | 48 +++++----- .../configurator_test.go | 95 +++++++++---------- .../controller.go | 2 +- .../controller_test.go | 2 +- .../mocks/generated.go | 0 .../mocks/mock_configuration_api.go | 4 +- .../temp.go | 2 +- 10 files changed, 82 insertions(+), 83 deletions(-) rename {config_applier => remote_configuration}/change_applier.go (97%) rename {config_applier => remote_configuration}/change_applier_test.go (99%) rename config_applier/applier.go => remote_configuration/configurator.go (56%) rename config_applier/applier_test.go => remote_configuration/configurator_test.go (75%) rename {config_applier => remote_configuration}/controller.go (98%) rename {config_applier => remote_configuration}/controller_test.go (74%) rename {config_applier => remote_configuration}/mocks/generated.go (100%) rename {config_applier => remote_configuration}/mocks/mock_configuration_api.go (93%) rename {config_applier => remote_configuration}/temp.go (98%) diff --git a/main.go b/main.go index ad4aea6c..0d8fe978 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/state/operator" - "github.com/vmware/cbcontainers-operator/config_applier" + "github.com/vmware/cbcontainers-operator/remote_configuration" "go.uber.org/zap/zapcore" coreV1 "k8s.io/api/core/v1" "os" @@ -175,9 +175,9 @@ func main() { signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") - api := config_applier.DummyAPI{} - applier := config_applier.NewApplier(k8sClient, api, log) - applierController := config_applier.NewRemoteConfigurationController(applier, log) + api := remote_configuration.DummyAPI{} + applier := remote_configuration.NewConfigurator(k8sClient, api, log) + applierController := remote_configuration.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup wg.Add(2) diff --git a/config_applier/change_applier.go b/remote_configuration/change_applier.go similarity index 97% rename from config_applier/change_applier.go rename to remote_configuration/change_applier.go index e1e01a4b..ff3c4e68 100644 --- a/config_applier/change_applier.go +++ b/remote_configuration/change_applier.go @@ -1,4 +1,4 @@ -package config_applier +package remote_configuration import cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" diff --git a/config_applier/change_applier_test.go b/remote_configuration/change_applier_test.go similarity index 99% rename from config_applier/change_applier_test.go rename to remote_configuration/change_applier_test.go index e0ff9a2f..251e1364 100644 --- a/config_applier/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -1,4 +1,4 @@ -package config_applier +package remote_configuration import ( "fmt" diff --git a/config_applier/applier.go b/remote_configuration/configurator.go similarity index 56% rename from config_applier/applier.go rename to remote_configuration/configurator.go index eea1f7c2..3b9f6577 100644 --- a/config_applier/applier.go +++ b/remote_configuration/configurator.go @@ -1,4 +1,4 @@ -package config_applier +package remote_configuration import ( "context" @@ -17,56 +17,56 @@ const ( ) type ConfigurationChangesAPI interface { - // Get Compatibility matrix + // TODO: Get Compatibility matrix GetConfigurationChanges(context.Context) ([]ConfigurationChange, error) UpdateConfigurationChangeStatus(context.Context, ConfigurationChangeStatusUpdate) error } -type Applier struct { +type Configurator struct { k8sClient client.Client logger logr.Logger changesAPI ConfigurationChangesAPI } -func NewApplier(k8sClient client.Client, api ConfigurationChangesAPI, logger logr.Logger) *Applier { - return &Applier{k8sClient: k8sClient, logger: logger, changesAPI: api} +func NewConfigurator(k8sClient client.Client, api ConfigurationChangesAPI, logger logr.Logger) *Configurator { + return &Configurator{k8sClient: k8sClient, logger: logger, changesAPI: api} } -func (applier *Applier) RunIteration(ctx context.Context) error { +func (configurator *Configurator) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() - applier.logger.Info("Checking for pending remote configuration changes...") + configurator.logger.Info("Checking for pending remote configuration changes...") - change, errGettingChanges := applier.getPendingChange(ctx) + change, errGettingChanges := configurator.getPendingChange(ctx) if errGettingChanges != nil { - applier.logger.Error(errGettingChanges, "Failed to get pending configuration changes") + configurator.logger.Error(errGettingChanges, "Failed to get pending configuration changes") return errGettingChanges } if change == nil { - applier.logger.Info("No pending remote configuration changes found") + configurator.logger.Info("No pending remote configuration changes found") return nil } - applier.logger.Info("Applying remote configuration change", "change", change) - cr, errApplyingCR := applier.applyChange(ctx, *change) + configurator.logger.Info("Applying remote configuration change", "change", change) + cr, errApplyingCR := configurator.applyChange(ctx, *change) if errApplyingCR != nil { - applier.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) + configurator.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough as we always update the status of the change on the backend, including failed status } - if errStatusUpdate := applier.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { - applier.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") + if errStatusUpdate := configurator.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { + configurator.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") return errStatusUpdate } return errApplyingCR } -func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { - changes, err := applier.changesAPI.GetConfigurationChanges(ctx) +func (configurator *Configurator) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { + changes, err := configurator.changesAPI.GetConfigurationChanges(ctx) if err != nil { return nil, err } @@ -83,7 +83,7 @@ func (applier *Applier) getPendingChange(ctx context.Context) (*ConfigurationCha return nil, nil } -func (applier *Applier) updateChangeStatus(ctx context.Context, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { +func (configurator *Configurator) updateChangeStatus(ctx context.Context, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { var statusUpdate ConfigurationChangeStatusUpdate if encounteredError == nil { statusUpdate = ConfigurationChangeStatusUpdate{ @@ -101,11 +101,11 @@ func (applier *Applier) updateChangeStatus(ctx context.Context, change Configura } } - return applier.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) + return configurator.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) } -func (applier *Applier) applyChange(ctx context.Context, change ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { - cr, err := applier.getContainerAgentCR(ctx) +func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { + cr, err := configurator.getContainerAgentCR(ctx) if err != nil { return nil, err } @@ -115,18 +115,18 @@ func (applier *Applier) applyChange(ctx context.Context, change ConfigurationCha applyChangesToCR(change, cr) - err = applier.k8sClient.Update(ctx, cr) + err = configurator.k8sClient.Update(ctx, cr) return cr, err } // getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions // if no resource is defined, nil is returned // in case more than 1 resource is defined (which is not supported), only the first one is returned -func (applier *Applier) getContainerAgentCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { +func (configurator *Configurator) getContainerAgentCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} - if err := applier.k8sClient.List(ctx, cbContainersAgentsList); err != nil { + if err := configurator.k8sClient.List(ctx, cbContainersAgentsList); err != nil { return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) } diff --git a/config_applier/applier_test.go b/remote_configuration/configurator_test.go similarity index 75% rename from config_applier/applier_test.go rename to remote_configuration/configurator_test.go index e3fe9192..975356f1 100644 --- a/config_applier/applier_test.go +++ b/remote_configuration/configurator_test.go @@ -1,4 +1,4 @@ -package config_applier_test +package remote_configuration_test import ( "context" @@ -9,8 +9,8 @@ import ( "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" k8sMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" - "github.com/vmware/cbcontainers-operator/config_applier" - mocksConfigApplier "github.com/vmware/cbcontainers-operator/config_applier/mocks" + "github.com/vmware/cbcontainers-operator/remote_configuration" + mocksConfigurator "github.com/vmware/cbcontainers-operator/remote_configuration/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" "time" @@ -24,37 +24,36 @@ import ( // TODO: Reads cluster, etc from CR correctly? // TODO: Respects proxy // TODO: Review gomock.any usages here -// TODO: version tests // TODO: Multiple changes are applied according to timestamp -type applierMocks struct { +type configuratorMocks struct { k8sClient *k8sMocks.MockClient - api *mocksConfigApplier.MockConfigurationChangesAPI + api *mocksConfigurator.MockConfigurationChangesAPI } -func setupApplier(ctrl *gomock.Controller) (*config_applier.Applier, applierMocks) { +func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { k8sClient := k8sMocks.NewMockClient(ctrl) - api := mocksConfigApplier.NewMockConfigurationChangesAPI(ctrl) + api := mocksConfigurator.NewMockConfigurationChangesAPI(ctrl) - applier := config_applier.NewApplier(k8sClient, api, logr.Discard()) - mocksHolder := applierMocks{ + configurator := remote_configuration.NewConfigurator(k8sClient, api, logr.Discard()) + mocksHolder := configuratorMocks{ k8sClient: k8sClient, api: api, } - return applier, mocksHolder + return configurator, mocksHolder } func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) // TODO: Compatiblity check - configChange := config_applier.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := remote_configuration.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -79,7 +78,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, int64(2), update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) @@ -91,22 +90,22 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - err := applier.RunIteration(context.Background()) + err := configurator.RunIteration(context.Background()) assert.NoError(t, err) } func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { testCases := []struct { name string - dataFromService []config_applier.ConfigurationChange + dataFromService []remote_configuration.ConfigurationChange }{ { name: "empty list", - dataFromService: []config_applier.ConfigurationChange{}, + dataFromService: []remote_configuration.ConfigurationChange{}, }, { name: "list is not empty but there are no PENDING changes", - dataFromService: []config_applier.ConfigurationChange{ + dataFromService: []remote_configuration.ConfigurationChange{ {ID: "123", Status: "non-existent"}, {ID: "234", Status: "FAILED"}, {ID: "345", Status: "ACKNOWLEDGED"}, @@ -120,12 +119,12 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) - err := applier.RunIteration(context.Background()) + err := configurator.RunIteration(context.Background()) assert.NoError(t, err) }) } @@ -135,10 +134,10 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) - olderChange := config_applier.RandomNonNilChange() - newerChange := config_applier.RandomNonNilChange() + olderChange := remote_configuration.RandomNonNilChange() + newerChange := remote_configuration.RandomNonNilChange() expectedVersion := "version-for-older-change" versionThatShouldNotBe := "version-for-newer-change" @@ -147,7 +146,7 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*newerChange, *olderChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*newerChange, *olderChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -172,12 +171,12 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { return nil }) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, olderChange.ID, update.ID) return nil }) - err := applier.RunIteration(context.Background()) + err := configurator.RunIteration(context.Background()) assert.NoError(t, err) } @@ -185,12 +184,12 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) errFromService := errors.New("some error") mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) - returnedErr := applier.RunIteration(context.Background()) + returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } @@ -199,16 +198,16 @@ func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) - configChange := config_applier.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := remote_configuration.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) errFromService := errors.New("some error") mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) assert.NotEmpty(t, update.Reason) @@ -218,7 +217,7 @@ func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { return nil }) - returnedErr := applier.RunIteration(context.Background()) + returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } @@ -227,10 +226,10 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) - configChange := config_applier.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := remote_configuration.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -248,7 +247,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) assert.NotEmpty(t, update.Reason) @@ -258,7 +257,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { return nil }) - returnedErr := applier.RunIteration(context.Background()) + returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } @@ -267,10 +266,10 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) - configChange := config_applier.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := remote_configuration.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -297,7 +296,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { errFromService := errors.New("some error") mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) - returnedErr := applier.RunIteration(context.Background()) + returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, errFromService, returnedErr, "expected returned error to match or wrap error from service") } @@ -306,10 +305,10 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - applier, mocks := setupApplier(ctrl) + configurator, mocks := setupConfigurator(ctrl) - configChange := config_applier.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]config_applier.ConfigurationChange{*configChange}, nil) + configChange := remote_configuration.RandomNonNilChange() + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { @@ -317,7 +316,7 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { }) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update config_applier.ConfigurationChangeStatusUpdate) error { + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) assert.NotEmpty(t, update.Reason) @@ -327,7 +326,7 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { return nil }) - err := applier.RunIteration(context.Background()) + err := configurator.RunIteration(context.Background()) assert.Error(t, err) // TODO: Specific error exposed for this? } diff --git a/config_applier/controller.go b/remote_configuration/controller.go similarity index 98% rename from config_applier/controller.go rename to remote_configuration/controller.go index 4b65d821..a2c35aee 100644 --- a/config_applier/controller.go +++ b/remote_configuration/controller.go @@ -1,4 +1,4 @@ -package config_applier +package remote_configuration import ( "context" diff --git a/config_applier/controller_test.go b/remote_configuration/controller_test.go similarity index 74% rename from config_applier/controller_test.go rename to remote_configuration/controller_test.go index 6e733e45..08735e67 100644 --- a/config_applier/controller_test.go +++ b/remote_configuration/controller_test.go @@ -1,4 +1,4 @@ -package config_applier_test +package remote_configuration_test // No error -> normal waiting time // Error encountered -> with backoff diff --git a/config_applier/mocks/generated.go b/remote_configuration/mocks/generated.go similarity index 100% rename from config_applier/mocks/generated.go rename to remote_configuration/mocks/generated.go diff --git a/config_applier/mocks/mock_configuration_api.go b/remote_configuration/mocks/mock_configuration_api.go similarity index 93% rename from config_applier/mocks/mock_configuration_api.go rename to remote_configuration/mocks/mock_configuration_api.go index 82b04050..2a5c9d40 100644 --- a/config_applier/mocks/mock_configuration_api.go +++ b/remote_configuration/mocks/mock_configuration_api.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/config_applier (interfaces: ConfigurationChangesAPI) +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ConfigurationChangesAPI) // Package mocks is a generated GoMock package. package mocks @@ -9,7 +9,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - config_applier "github.com/vmware/cbcontainers-operator/config_applier" + config_applier "github.com/vmware/cbcontainers-operator/remote_configuration" ) // MockConfigurationChangesAPI is a mock of ConfigurationChangesAPI interface. diff --git a/config_applier/temp.go b/remote_configuration/temp.go similarity index 98% rename from config_applier/temp.go rename to remote_configuration/temp.go index 60f1140e..098209c3 100644 --- a/config_applier/temp.go +++ b/remote_configuration/temp.go @@ -1,4 +1,4 @@ -package config_applier +package remote_configuration import ( "context" From 581d5fcfdb7ebbaabe8bb4919b4fd88ebdbb78c6 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 15:57:09 +0300 Subject: [PATCH 26/65] Refactor tests a bit and remove more TODOs --- remote_configuration/configurator_test.go | 175 ++++++++-------------- remote_configuration/temp.go | 4 +- 2 files changed, 64 insertions(+), 115 deletions(-) diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 975356f1..b03d6d50 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -17,14 +17,10 @@ import ( ) // TODO: Compatibility checks -// TODO: Adding CNDR to the config options -// TODO: Properly handle version + custom image to override the custom image -// TODO: Check fields are applied to CR correctly // TODO: Reads cluster, etc from CR correctly? // TODO: Respects proxy // TODO: Review gomock.any usages here -// TODO: Multiple changes are applied according to timestamp type configuratorMocks struct { k8sClient *k8sMocks.MockClient @@ -49,38 +45,23 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) + var initialGeneration, finalGeneration int64 = 1, 2 // TODO: Compatiblity check configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{ - { - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Generation: 1, - }, - Spec: cbcontainersv1.CBContainersAgentSpec{}, - Status: cbcontainersv1.CBContainersAgentStatus{}, - }, - } - }) + setupCRInK8S(mocks.k8sClient, &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}}) - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, item any, _ ...any) error { - asCb, ok := item.(*cbcontainersv1.CBContainersAgent) - require.True(t, ok) - require.Equal(t, *configChange.AgentVersion, asCb.Spec.Version) - asCb.ObjectMeta.Generation++ - return nil - }) + assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, *configChange.AgentVersion, agent.Spec.Version) + agent.ObjectMeta.Generation = finalGeneration + }) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, int64(2), update.AppliedGeneration) + assert.Equal(t, finalGeneration, update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) assert.NotEmpty(t, update.AppliedTimestamp, "applied timestamp should be populated") @@ -146,36 +127,19 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*newerChange, *olderChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) + setupCRInK8S(mocks.k8sClient, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{ - { - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Generation: 1, - }, - Spec: cbcontainersv1.CBContainersAgentSpec{}, - Status: cbcontainersv1.CBContainersAgentStatus{}, - }, - } - }) - - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, item any, _ ...any) error { - asCb, ok := item.(*cbcontainersv1.CBContainersAgent) - require.True(t, ok) + assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, expectedVersion, agent.Spec.Version) + }) - assert.Equal(t, expectedVersion, asCb.Spec.Version) + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, olderChange.ID, update.ID) return nil }) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, olderChange.ID, update.ID) - return nil - }) - err := configurator.RunIteration(context.Background()) assert.NoError(t, err) } @@ -190,6 +154,7 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) returnedErr := configurator.RunIteration(context.Background()) + assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } @@ -201,21 +166,12 @@ func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) errFromService := errors.New("some error") mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) - assert.NotEmpty(t, update.Reason) - assert.Equal(t, int64(0), update.AppliedGeneration) - assert.Empty(t, update.AppliedTimestamp) - - return nil - }) + assertChangeIsSetAsFailed(t, mocks.api, configChange) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -229,33 +185,14 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{ - { - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: cbcontainersv1.CBContainersAgentSpec{}, - Status: cbcontainersv1.CBContainersAgentStatus{}, - }, - } - }) + setupCRInK8S(mocks.k8sClient, nil) errFromService := errors.New("some error") mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) - assert.NotEmpty(t, update.Reason) - assert.Equal(t, int64(0), update.AppliedGeneration) - assert.Empty(t, update.AppliedTimestamp) - - return nil - }) + assertChangeIsSetAsFailed(t, mocks.api, configChange) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -269,29 +206,11 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{ - { - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Generation: 1, - }, - Spec: cbcontainersv1.CBContainersAgentSpec{}, - Status: cbcontainersv1.CBContainersAgentStatus{}, - }, - } - }) + setupCRInK8S(mocks.k8sClient, nil) - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, item any, _ ...any) error { - asCb, ok := item.(*cbcontainersv1.CBContainersAgent) - require.True(t, ok) - asCb.ObjectMeta.Generation++ - return nil - }) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) @@ -308,16 +227,50 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{*configChange}, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{} }) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + assertChangeIsSetAsFailed(t, mocks.api, configChange) + + err := configurator.RunIteration(context.Background()) + assert.Error(t, err) +} + +// setupCRInK8S ensures the mock client will return 1 agent item for List calls - either the provided one or an empty CR otherwise +func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAgent) { + if item == nil { + item = &cbcontainersv1.CBContainersAgent{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + } + } + mock.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{*item} + }) +} + +func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcontainersv1.CBContainersAgent)) { + mock.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + + assert(asCb) + return nil + }) +} + +func assertChangeIsSetAsFailed(t *testing.T, mock *mocksConfigurator.MockConfigurationChangesAPI, change remote_configuration.ConfigurationChange) { + mock.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, change.ID, update.ID) assert.Equal(t, "FAILED", update.Status) assert.NotEmpty(t, update.Reason) assert.Equal(t, int64(0), update.AppliedGeneration) @@ -325,8 +278,4 @@ func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { return nil }) - - err := configurator.RunIteration(context.Background()) - assert.Error(t, err) - // TODO: Specific error exposed for this? } diff --git a/remote_configuration/temp.go b/remote_configuration/temp.go index 098209c3..d694aa69 100644 --- a/remote_configuration/temp.go +++ b/remote_configuration/temp.go @@ -29,11 +29,11 @@ func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update Co return nil } -func RandomNonNilChange() *ConfigurationChange { +func RandomNonNilChange() ConfigurationChange { for { c := RandomChange() if c != nil { - return c + return *c } } } From c5a802239e4534327c0a9bc81e2fae74656e9a1d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 24 Aug 2023 16:20:56 +0300 Subject: [PATCH 27/65] Change the configurator to read the CR first , so it can extract the cluster name --- remote_configuration/configurator.go | 35 ++++++++------ remote_configuration/configurator_test.go | 59 +++++++++-------------- 2 files changed, 42 insertions(+), 52 deletions(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 3b9f6577..5aade01e 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -37,8 +37,18 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) defer cancel() - configurator.logger.Info("Checking for pending remote configuration changes...") + configurator.logger.Info("Checking for installed agent...") + cr, errGettingCR := configurator.getContainerAgentCR(ctx) + if errGettingCR != nil { + configurator.logger.Error(errGettingCR, "Failed to get CBContainerAgent resource, cannot continue") + return errGettingCR + } + if cr == nil { + configurator.logger.Info("No CBContainerAgent installed, there is nothing to configure") + return nil + } + configurator.logger.Info("Checking for pending remote configuration changes...") change, errGettingChanges := configurator.getPendingChange(ctx) if errGettingChanges != nil { configurator.logger.Error(errGettingChanges, "Failed to get pending configuration changes") @@ -50,8 +60,8 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return nil } - configurator.logger.Info("Applying remote configuration change", "change", change) - cr, errApplyingCR := configurator.applyChange(ctx, *change) + configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) + errApplyingCR := configurator.applyChange(ctx, *change, cr) if errApplyingCR != nil { configurator.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) // Intentional fallthrough as we always update the status of the change on the backend, including failed status @@ -62,6 +72,7 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return errStatusUpdate } + // If we failed to apply the CR, we still report this to the backend but want to return the apply error here to propagate properly return errApplyingCR } @@ -104,19 +115,11 @@ func (configurator *Configurator) updateChangeStatus(ctx context.Context, change return configurator.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) } -func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange) (*cbcontainersv1.CBContainersAgent, error) { - cr, err := configurator.getContainerAgentCR(ctx) - if err != nil { - return nil, err - } - if cr == nil { - return nil, fmt.Errorf("no CBContainerAgent instance found, cannot apply change") - } - - applyChangesToCR(change, cr) - - err = configurator.k8sClient.Update(ctx, cr) - return cr, err +// applyChange will sync the required changes and push them to the k8s api-server +// the input agent will be modified after this function and will no longer match the original +func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange, agent *cbcontainersv1.CBContainersAgent) error { + applyChangesToCR(change, agent) + return configurator.k8sClient.Update(ctx, agent) } // getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index b03d6d50..c4d5ab01 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -49,11 +49,11 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { // TODO: Compatiblity check + setupCRInK8S(mocks.k8sClient, &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}}) + configChange := remote_configuration.RandomNonNilChange() mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - setupCRInK8S(mocks.k8sClient, &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}}) - assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { assert.Equal(t, *configChange.AgentVersion, agent.Spec.Version) agent.ObjectMeta.Generation = finalGeneration @@ -102,6 +102,7 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) + setupCRInK8S(mocks.k8sClient, nil) mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) @@ -127,8 +128,8 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) setupCRInK8S(mocks.k8sClient, nil) + mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { assert.Equal(t, expectedVersion, agent.Spec.Version) @@ -150,6 +151,8 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) configurator, mocks := setupConfigurator(ctrl) + setupCRInK8S(mocks.k8sClient, nil) + errFromService := errors.New("some error") mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) @@ -159,20 +162,15 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } -func TestWhenGettingCRFromAPIServerFailsChangeIsUpdatedAsFailed(t *testing.T) { +func TestWhenGettingCRFromAPIServerFailsAnErrorIsReturned(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) - configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - errFromService := errors.New("some error") mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) - assertChangeIsSetAsFailed(t, mocks.api, configChange) - returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") @@ -184,15 +182,24 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) + setupCRInK8S(mocks.k8sClient, nil) + configChange := remote_configuration.RandomNonNilChange() mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - setupCRInK8S(mocks.k8sClient, nil) - errFromService := errors.New("some error") mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) - assertChangeIsSetAsFailed(t, mocks.api, configChange) + mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -205,11 +212,11 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) + setupCRInK8S(mocks.k8sClient, nil) + configChange := remote_configuration.RandomNonNilChange() mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - setupCRInK8S(mocks.k8sClient, nil) - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") @@ -220,24 +227,17 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { assert.ErrorIs(t, errFromService, returnedErr, "expected returned error to match or wrap error from service") } -func TestWhenThereIsNoCRInstalledChangeIsUpdatedAsFailed(t *testing.T) { +func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { ctrl := gomock.NewController(t) - defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) - configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{} }) - assertChangeIsSetAsFailed(t, mocks.api, configChange) - - err := configurator.RunIteration(context.Background()) - assert.Error(t, err) + assert.NoError(t, configurator.RunIteration(context.Background())) } // setupCRInK8S ensures the mock client will return 1 agent item for List calls - either the provided one or an empty CR otherwise @@ -266,16 +266,3 @@ func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcont return nil }) } - -func assertChangeIsSetAsFailed(t *testing.T, mock *mocksConfigurator.MockConfigurationChangesAPI, change remote_configuration.ConfigurationChange) { - mock.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, change.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) - assert.NotEmpty(t, update.Reason) - assert.Equal(t, int64(0), update.AppliedGeneration) - assert.Empty(t, update.AppliedTimestamp) - - return nil - }) -} From d1d1b05f12a469e490967dff94d95447b80676c6 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 25 Aug 2023 13:56:02 +0300 Subject: [PATCH 28/65] Validate change against sensor capabilities --- remote_configuration/change_applier.go | 69 +++++++++++++++++++- remote_configuration/change_applier_test.go | 72 +++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index ff3c4e68..51f38406 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -1,6 +1,73 @@ package remote_configuration -import cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" +import ( + "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// TODO: Move somewhere else + +type Sensor struct { + Version string `json:"version" yaml:"version"` + IsLatest bool `json:"is_latest" yaml:"isLatest"` + SupportsRuntime bool `json:"supports_runtime" yaml:"supportsRuntime"` + SupportsClusterScanning bool `json:"supports_cluster_scanning" yaml:"supportsClusterScanning"` + SupportsCndr bool `json:"supports_cndr" yaml:"supportsCndr"` +} + +type TODO struct { + OperatorVersion string + SensorData []Sensor + OperatorCompatibilityData models.OperatorCompatibility +} + +func (todo *TODO) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { + var versionToValidate string + + // If the change will be modifying the agent version as well, we need to check what the _new_ version supports + if change.AgentVersion != nil { + versionToValidate = *change.AgentVersion + } else { + // Otherwise the current agent must actually work with the requested features + versionToValidate = cr.Spec.Version + } + + sensor, err := todo.findMatchingSensor(versionToValidate) + if err != nil { + return false, err.Error() + } + + if change.EnableClusterScanning != nil && + *change.EnableClusterScanning == true && + !sensor.SupportsClusterScanning { + return false, fmt.Sprintf("sensor version %s does not support cluster scanning feature", versionToValidate) + } + + if change.EnableRuntime != nil && + *change.EnableRuntime == true && + !sensor.SupportsRuntime { + return false, fmt.Sprintf("sensor version %s does not support runtime protection feature", versionToValidate) + } + + if change.EnableCNDR != nil && + *change.EnableCNDR == true && + !sensor.SupportsCndr { + return false, fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", versionToValidate) + } + + return false, "" +} + +func (todo *TODO) findMatchingSensor(sensorVersion string) (*Sensor, error) { + for _, sensor := range todo.SensorData { + if sensor.Version == sensorVersion { + return &sensor, nil + } + } + + return nil, fmt.Errorf("could not find sensor metadata for version %s", sensorVersion) +} func applyChangesToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { // TODO: Validation? diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 251e1364..4445230a 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -7,6 +7,8 @@ import ( "testing" ) +// TODO: add secret detection + var ( trueV = true truePtr = &trueV @@ -14,6 +16,76 @@ var ( falsePtr = &falseV ) +func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change ConfigurationChange + sensorMeta Sensor + }{ + { + name: "cluster scanning", + change: ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: Sensor{ + SupportsClusterScanning: false, + }, + }, + { + name: "runtime protection", + change: ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: Sensor{ + SupportsRuntime: false, + }, + }, + { + name: "CNDR", + change: ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: Sensor{ + SupportsCndr: false, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := TODO{ + SensorData: []Sensor{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + valid, msg := target.ValidateChange(tC.change, cr) + + assert.False(t, valid) + assert.NotEmpty(t, msg) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s not supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + valid, msg := target.ValidateChange(tC.change, cr) + + assert.False(t, valid) + assert.NotEmpty(t, msg) + }) + } + // sensor does not support 1,2,3,4 features -> should return error + + // Must validate when NO version upgrade is done -> validates against current version + // When version upgrade is done -> validates against new version instead + + //a, b := target.ValidateChange() +} + func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { type appliedChangeTest struct { name string From c4f2d9cd59aa1768a1e0bb3c22c3edf0cb40dd93 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 25 Aug 2023 14:29:16 +0300 Subject: [PATCH 29/65] Validate operator and agent version compatibility --- cbcontainers/models/operator_compatibility.go | 2 +- remote_configuration/change_applier.go | 46 ++++++++++++------- remote_configuration/change_applier_test.go | 44 ++++++++++++++++-- 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/cbcontainers/models/operator_compatibility.go b/cbcontainers/models/operator_compatibility.go index d0fcc52d..1509d929 100644 --- a/cbcontainers/models/operator_compatibility.go +++ b/cbcontainers/models/operator_compatibility.go @@ -21,6 +21,6 @@ func (c OperatorCompatibility) CheckCompatibility(agentVersion string) error { return fmt.Errorf("agent version too low, downgrade the operator to use that agent version: min is [%s], desired is [%s]", c.MinAgent, agentVersion) } - // if we are here it means the operator and the agent version are compatibile + // if we are here it means the operator and the agent version are compatible return nil } diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index 51f38406..14409f4b 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -17,7 +17,6 @@ type Sensor struct { } type TODO struct { - OperatorVersion string SensorData []Sensor OperatorCompatibilityData models.OperatorCompatibility } @@ -33,42 +32,57 @@ func (todo *TODO) ValidateChange(change ConfigurationChange, cr *cbcontainersv1. versionToValidate = cr.Spec.Version } - sensor, err := todo.findMatchingSensor(versionToValidate) - if err != nil { + if sensorAndOperatorCompatible, msg := todo.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { + return false, msg + } + + return todo.validateSensorAndFeatureCompatibility(versionToValidate, change, cr) +} + +func (todo *TODO) findMatchingSensor(sensorVersion string) (*Sensor, string) { + for _, sensor := range todo.SensorData { + if sensor.Version == sensorVersion { + return &sensor, "" + } + } + + return nil, fmt.Sprintf("could not find sensor metadata for version %s", sensorVersion) +} + +func (todo *TODO) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { + if err := todo.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { return false, err.Error() } + return true, "" +} + +func (todo *TODO) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { + sensor, msg := todo.findMatchingSensor(targetVersion) + if sensor == nil { + return false, msg + } if change.EnableClusterScanning != nil && *change.EnableClusterScanning == true && !sensor.SupportsClusterScanning { - return false, fmt.Sprintf("sensor version %s does not support cluster scanning feature", versionToValidate) + return false, fmt.Sprintf("sensor version %s does not support cluster scanning feature", targetVersion) } if change.EnableRuntime != nil && *change.EnableRuntime == true && !sensor.SupportsRuntime { - return false, fmt.Sprintf("sensor version %s does not support runtime protection feature", versionToValidate) + return false, fmt.Sprintf("sensor version %s does not support runtime protection feature", targetVersion) } if change.EnableCNDR != nil && *change.EnableCNDR == true && !sensor.SupportsCndr { - return false, fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", versionToValidate) + return false, fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", targetVersion) } return false, "" } -func (todo *TODO) findMatchingSensor(sensorVersion string) (*Sensor, error) { - for _, sensor := range todo.SensorData { - if sensor.Version == sensorVersion { - return &sensor, nil - } - } - - return nil, fmt.Errorf("could not find sensor metadata for version %s", sensorVersion) -} - func applyChangesToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { // TODO: Validation? diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 4445230a..2fa7de74 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/stretchr/testify/assert" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" "testing" ) @@ -78,12 +79,47 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { assert.NotEmpty(t, msg) }) } - // sensor does not support 1,2,3,4 features -> should return error +} + +func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatiblity models.OperatorCompatibility + }{ + { + name: "sensor version is too high", + versionToApply: "5.0.0", + operatorCompatiblity: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: "4.0.0", + }, + }, + { + name: "sensor version is too low", + versionToApply: "0.9", + operatorCompatiblity: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := TODO{ + SensorData: []Sensor{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatiblity, + } - // Must validate when NO version upgrade is done -> validates against current version - // When version upgrade is done -> validates against new version instead + change := ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} - //a, b := target.ValidateChange() + valid, msg := target.ValidateChange(change, cr) + assert.False(t, valid) + assert.NotEmpty(t, msg) + }) + } } func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { From bd82a5c75d2d611f4ba8bfb9fe04b939337a41c1 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 25 Aug 2023 14:36:34 +0300 Subject: [PATCH 30/65] Tests for positive validation path --- remote_configuration/change_applier.go | 2 +- remote_configuration/change_applier_test.go | 133 +++++++++++++++++++- 2 files changed, 128 insertions(+), 7 deletions(-) diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index 14409f4b..cc28415f 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -80,7 +80,7 @@ func (todo *TODO) validateSensorAndFeatureCompatibility(targetVersion string, ch return false, fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", targetVersion) } - return false, "" + return true, "" } func applyChangesToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 2fa7de74..e2f5f704 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -81,16 +81,80 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { } } +func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change ConfigurationChange + sensorMeta Sensor + }{ + { + name: "cluster scanning", + change: ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: Sensor{ + SupportsClusterScanning: true, + }, + }, + { + name: "runtime protection", + change: ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: Sensor{ + SupportsRuntime: true, + }, + }, + { + name: "CNDR", + change: ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: Sensor{ + SupportsCndr: true, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := TODO{ + SensorData: []Sensor{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + valid, msg := target.ValidateChange(tC.change, cr) + + assert.True(t, valid) + assert.Empty(t, msg) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s is supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + valid, msg := target.ValidateChange(tC.change, cr) + + assert.True(t, valid) + assert.Empty(t, msg) + }) + } +} + func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { testCases := []struct { - name string - versionToApply string - operatorCompatiblity models.OperatorCompatibility + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility }{ { name: "sensor version is too high", versionToApply: "5.0.0", - operatorCompatiblity: models.OperatorCompatibility{ + operatorCompatibility: models.OperatorCompatibility{ MinAgent: models.AgentMinVersionNone, MaxAgent: "4.0.0", }, @@ -98,7 +162,7 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { { name: "sensor version is too low", versionToApply: "0.9", - operatorCompatiblity: models.OperatorCompatibility{ + operatorCompatibility: models.OperatorCompatibility{ MinAgent: "1.0.0", MaxAgent: models.AgentMaxVersionLatest, }, @@ -109,7 +173,7 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { t.Run(tC.name, func(t *testing.T) { target := TODO{ SensorData: []Sensor{{Version: tC.versionToApply}}, - OperatorCompatibilityData: tC.operatorCompatiblity, + OperatorCompatibilityData: tC.operatorCompatibility, } change := ConfigurationChange{AgentVersion: &tC.versionToApply} @@ -122,6 +186,63 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { } } +func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility + }{ + { + name: "sensor version is at lower end", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "5.0.0", + MaxAgent: "6.0.0", + }, + }, + { + name: "sensor version is at upper end", + versionToApply: "0.9", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "0.1.0", + MaxAgent: "0.9.0", + }, + }, + { + name: "sensor version is within range", + versionToApply: "2.3.4", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: "2.4", + }, + }, + { + name: "operator supports 'infinite' versions", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := TODO{ + SensorData: []Sensor{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatibility, + } + + change := ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} + + valid, msg := target.ValidateChange(change, cr) + assert.True(t, valid) + assert.Empty(t, msg) + }) + } +} + func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { type appliedChangeTest struct { name string From 3696680bcaaf83a487464d5332cf2c4b60366a52 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 4 Sep 2023 14:13:56 +0300 Subject: [PATCH 31/65] Bring the Validate and Apply change funcs under 1 struct and validate before applying --- remote_configuration/change_applier.go | 108 +++++++++--------- remote_configuration/change_applier_test.go | 120 +++++++++++++------- 2 files changed, 134 insertions(+), 94 deletions(-) diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index cc28415f..428ae893 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -9,19 +9,19 @@ import ( // TODO: Move somewhere else type Sensor struct { - Version string `json:"version" yaml:"version"` - IsLatest bool `json:"is_latest" yaml:"isLatest"` - SupportsRuntime bool `json:"supports_runtime" yaml:"supportsRuntime"` - SupportsClusterScanning bool `json:"supports_cluster_scanning" yaml:"supportsClusterScanning"` - SupportsCndr bool `json:"supports_cndr" yaml:"supportsCndr"` + Version string `json:"version"` + IsLatest bool `json:"is_latest" ` + SupportsRuntime bool `json:"supports_runtime"` + SupportsClusterScanning bool `json:"supports_cluster_scanning"` + SupportsCndr bool `json:"supports_cndr"` } -type TODO struct { +type CustomResourceChanger struct { SensorData []Sensor OperatorCompatibilityData models.OperatorCompatibility } -func (todo *TODO) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { +func (changer *CustomResourceChanger) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { var versionToValidate string // If the change will be modifying the agent version as well, we need to check what the _new_ version supports @@ -32,15 +32,56 @@ func (todo *TODO) ValidateChange(change ConfigurationChange, cr *cbcontainersv1. versionToValidate = cr.Spec.Version } - if sensorAndOperatorCompatible, msg := todo.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { + if sensorAndOperatorCompatible, msg := changer.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { return false, msg } - return todo.validateSensorAndFeatureCompatibility(versionToValidate, change, cr) + return changer.validateSensorAndFeatureCompatibility(versionToValidate, change) } -func (todo *TODO) findMatchingSensor(sensorVersion string) (*Sensor, string) { - for _, sensor := range todo.SensorData { +func (changer *CustomResourceChanger) ApplyChangeToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { + if isValid, msg := changer.ValidateChange(change, cr); !isValid { + return fmt.Errorf("provided change cannot be applied to the custom resource with reason (%s)", msg) + } + + resetVersion := func(ptrToField *string) { + if ptrToField != nil && *ptrToField != "" { + *ptrToField = "" + } + } + + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion + + resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + if cr.Spec.Components.Cndr != nil { + resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) + } + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } + + return nil +} + +func (changer *CustomResourceChanger) findMatchingSensor(sensorVersion string) (*Sensor, string) { + for _, sensor := range changer.SensorData { if sensor.Version == sensorVersion { return &sensor, "" } @@ -49,15 +90,15 @@ func (todo *TODO) findMatchingSensor(sensorVersion string) (*Sensor, string) { return nil, fmt.Sprintf("could not find sensor metadata for version %s", sensorVersion) } -func (todo *TODO) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { - if err := todo.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { +func (changer *CustomResourceChanger) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { + if err := changer.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { return false, err.Error() } return true, "" } -func (todo *TODO) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { - sensor, msg := todo.findMatchingSensor(targetVersion) +func (changer *CustomResourceChanger) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) (bool, string) { + sensor, msg := changer.findMatchingSensor(targetVersion) if sensor == nil { return false, msg } @@ -82,40 +123,3 @@ func (todo *TODO) validateSensorAndFeatureCompatibility(targetVersion string, ch return true, "" } - -func applyChangesToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { - // TODO: Validation? - - resetVersion := func(ptrToField *string) { - if ptrToField != nil && *ptrToField != "" { - *ptrToField = "" - } - } - - if change.AgentVersion != nil { - cr.Spec.Version = *change.AgentVersion - - resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) - if cr.Spec.Components.Cndr != nil { - resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) - } - } - if change.EnableClusterScanning != nil { - cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning - } - if change.EnableRuntime != nil { - cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime - } - if change.EnableCNDR != nil { - if cr.Spec.Components.Cndr == nil { - cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} - } - cr.Spec.Components.Cndr.Enabled = change.EnableCNDR - } -} diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index e2f5f704..858c6c16 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -1,10 +1,12 @@ -package remote_configuration +package remote_configuration_test import ( "fmt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/remote_configuration" "testing" ) @@ -20,33 +22,33 @@ var ( func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { testCases := []struct { name string - change ConfigurationChange - sensorMeta Sensor + change remote_configuration.ConfigurationChange + sensorMeta remote_configuration.Sensor }{ { name: "cluster scanning", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableClusterScanning: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsClusterScanning: false, }, }, { name: "runtime protection", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableRuntime: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsRuntime: false, }, }, { name: "CNDR", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableCNDR: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsCndr: false, }, }, @@ -55,8 +57,8 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { for _, tC := range testCases { version := "dummy-version" tC.sensorMeta.Version = version - target := TODO{ - SensorData: []Sensor{tC.sensorMeta}, + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{tC.sensorMeta}, } t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { @@ -84,33 +86,33 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { testCases := []struct { name string - change ConfigurationChange - sensorMeta Sensor + change remote_configuration.ConfigurationChange + sensorMeta remote_configuration.Sensor }{ { name: "cluster scanning", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableClusterScanning: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsClusterScanning: true, }, }, { name: "runtime protection", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableRuntime: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsRuntime: true, }, }, { name: "CNDR", - change: ConfigurationChange{ + change: remote_configuration.ConfigurationChange{ EnableCNDR: truePtr, }, - sensorMeta: Sensor{ + sensorMeta: remote_configuration.Sensor{ SupportsCndr: true, }, }, @@ -119,8 +121,8 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { for _, tC := range testCases { version := "dummy-version" tC.sensorMeta.Version = version - target := TODO{ - SensorData: []Sensor{tC.sensorMeta}, + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{tC.sensorMeta}, } t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { @@ -171,12 +173,12 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { for _, tC := range testCases { t.Run(tC.name, func(t *testing.T) { - target := TODO{ - SensorData: []Sensor{{Version: tC.versionToApply}}, + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{{Version: tC.versionToApply}}, OperatorCompatibilityData: tC.operatorCompatibility, } - change := ConfigurationChange{AgentVersion: &tC.versionToApply} + change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} valid, msg := target.ValidateChange(change, cr) @@ -228,12 +230,12 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { for _, tC := range testCases { t.Run(tC.name, func(t *testing.T) { - target := TODO{ - SensorData: []Sensor{{Version: tC.versionToApply}}, + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{{Version: tC.versionToApply}}, OperatorCompatibilityData: tC.operatorCompatibility, } - change := ConfigurationChange{AgentVersion: &tC.versionToApply} + change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} valid, msg := target.ValidateChange(change, cr) @@ -246,28 +248,30 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { type appliedChangeTest struct { name string - change ConfigurationChange + change remote_configuration.ConfigurationChange initialCR cbcontainersv1.CBContainersAgent assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) } + crVersion := "1.2.3" + // generateFeatureToggleTestCases produces a set of tests for a single feature toggle in the requested change // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) generateFeatureToggleTestCases := func(feature string, - changeFieldSelector func(*ConfigurationChange) **bool, + changeFieldSelector func(*remote_configuration.ConfigurationChange) **bool, crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { var result []appliedChangeTest for _, crState := range []*bool{truePtr, falsePtr, nil} { - cr := cbcontainersv1.CBContainersAgent{} + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} crFieldPtr := crFieldSelector(&cr) *crFieldPtr = crState // Validate that each toggle state works (or doesn't do anything when it matches) for _, changeState := range []*bool{truePtr, falsePtr} { - change := ConfigurationChange{} + change := remote_configuration.ConfigurationChange{} changeFieldPtr := changeFieldSelector(&change) *changeFieldPtr = changeState @@ -286,7 +290,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { // Validate that a change with the toggle unset does not modify the CR result = append(result, appliedChangeTest{ name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), - change: ConfigurationChange{}, + change: remote_configuration.ConfigurationChange{}, initialCR: cr, assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { crFieldPostChangePtr := crFieldSelector(agent) @@ -301,21 +305,21 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { var testCases []appliedChangeTest clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", - func(change *ConfigurationChange) **bool { + func(change *remote_configuration.ConfigurationChange) **bool { return &change.EnableClusterScanning }, func(agent *cbcontainersv1.CBContainersAgent) **bool { return &agent.Spec.Components.ClusterScanning.Enabled }) runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", - func(change *ConfigurationChange) **bool { + func(change *remote_configuration.ConfigurationChange) **bool { return &change.EnableRuntime }, func(agent *cbcontainersv1.CBContainersAgent) **bool { return &agent.Spec.Components.RuntimeProtection.Enabled }) cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", - func(change *ConfigurationChange) **bool { + func(change *remote_configuration.ConfigurationChange) **bool { return &change.EnableCNDR }, func(agent *cbcontainersv1.CBContainersAgent) **bool { if agent.Spec.Components.Cndr == nil { @@ -330,7 +334,20 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - applyChangesToCR(testCase.change, &testCase.initialCR) + + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{ + { + Version: crVersion, + SupportsRuntime: true, + SupportsCndr: true, + SupportsClusterScanning: true, + }, + }, + } + err := target.ApplyChangeToCR(testCase.change, &testCase.initialCR) + + require.NoError(t, err) testCase.assertFinalCR(t, &testCase.initialCR) }) } @@ -340,18 +357,22 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { originalVersion := "my-version-42" newVersion := "new-version" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} - change := ConfigurationChange{AgentVersion: &newVersion} + change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} + target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: newVersion}}} - applyChangesToCR(change, &cr) + err := target.ApplyChangeToCR(change, &cr) + require.NoError(t, err) assert.Equal(t, newVersion, cr.Spec.Version) } func TestMissingVersionDoesNotModifyCR(t *testing.T) { originalVersion := "my-version-42" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} - change := ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} + change := remote_configuration.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} + target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: originalVersion, SupportsRuntime: true}}} - applyChangesToCR(change, &cr) + err := target.ApplyChangeToCR(change, &cr) + require.NoError(t, err) assert.Equal(t, originalVersion, cr.Spec.Version) } @@ -414,9 +435,11 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { } newVersion := "new-version" - change := ConfigurationChange{AgentVersion: &newVersion} + change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} + target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: newVersion}}} - applyChangesToCR(change, &cr) + err := target.ApplyChangeToCR(change, &cr) + require.NoError(t, err) assert.Equal(t, newVersion, cr.Spec.Version) // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields @@ -431,6 +454,19 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { assert.Empty(t, cr.Spec.Components.Cndr.Sensor.Image.Tag) } +func TestInvalidChangeReturnsError(t *testing.T) { + version := "test" + change := remote_configuration.ConfigurationChange{EnableClusterScanning: truePtr} + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + target := remote_configuration.CustomResourceChanger{ + SensorData: []remote_configuration.Sensor{{Version: version, SupportsClusterScanning: false}}, + } + + err := target.ApplyChangeToCR(change, cr) + + assert.Error(t, err) +} + func prettyPrintBoolPtr(v *bool) string { if v == nil { return "" From 678dc9d83c80689eec137369bf7abf2c68dec86d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 4 Sep 2023 16:53:11 +0300 Subject: [PATCH 32/65] Adding change validation to the configuration --- cbcontainers/models/sensor_metadata.go | 9 ++ remote_configuration/configurator.go | 40 +++++--- remote_configuration/configurator_test.go | 96 +++++++++++++------ ..._applier.go => custom_resource_changer.go} | 74 ++++++-------- ...est.go => custom_resource_changer_test.go} | 73 ++++---------- remote_configuration/mocks/generated.go | 3 +- .../mocks/mock_change_validator.go | 51 ++++++++++ .../mocks/mock_configuration_api.go | 8 +- 8 files changed, 212 insertions(+), 142 deletions(-) create mode 100644 cbcontainers/models/sensor_metadata.go rename remote_configuration/{change_applier.go => custom_resource_changer.go} (62%) rename remote_configuration/{change_applier_test.go => custom_resource_changer_test.go} (84%) create mode 100644 remote_configuration/mocks/mock_change_validator.go diff --git a/cbcontainers/models/sensor_metadata.go b/cbcontainers/models/sensor_metadata.go new file mode 100644 index 00000000..f5c249b3 --- /dev/null +++ b/cbcontainers/models/sensor_metadata.go @@ -0,0 +1,9 @@ +package models + +type SensorMetadata struct { + Version string `json:"version"` + IsLatest bool `json:"is_latest" ` + SupportsRuntime bool `json:"supports_runtime"` + SupportsClusterScanning bool `json:"supports_cluster_scanning"` + SupportsCndr bool `json:"supports_cndr"` +} diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 5aade01e..56ce529e 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -18,19 +18,30 @@ const ( type ConfigurationChangesAPI interface { // TODO: Get Compatibility matrix + // TODO: Get sensor data GetConfigurationChanges(context.Context) ([]ConfigurationChange, error) UpdateConfigurationChangeStatus(context.Context, ConfigurationChangeStatusUpdate) error } +type ChangeValidator interface { + ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) +} + type Configurator struct { - k8sClient client.Client - logger logr.Logger - changesAPI ConfigurationChangesAPI + k8sClient client.Client + logger logr.Logger + changesAPI ConfigurationChangesAPI + changeValidator ChangeValidator } -func NewConfigurator(k8sClient client.Client, api ConfigurationChangesAPI, logger logr.Logger) *Configurator { - return &Configurator{k8sClient: k8sClient, logger: logger, changesAPI: api} +func NewConfigurator(k8sClient client.Client, configChangesAPI ConfigurationChangesAPI, changeValidator ChangeValidator, logger logr.Logger) *Configurator { + return &Configurator{ + k8sClient: k8sClient, + logger: logger, + changesAPI: configChangesAPI, + changeValidator: changeValidator, + } } func (configurator *Configurator) RunIteration(ctx context.Context) error { @@ -94,6 +105,18 @@ func (configurator *Configurator) getPendingChange(ctx context.Context) (*Config return nil, nil } +// applyChange will sync the required changes and push them to the k8s api-server +// the input agent will be modified after this function and will no longer match the original +func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange, agent *cbcontainersv1.CBContainersAgent) error { + if validChange, reason := configurator.changeValidator.ValidateChange(change, agent); !validChange { + return fmt.Errorf("provided change with ID (%s) is not applicable due to (%s)", change.ID, reason) + } + + ApplyChangeToCR(change, agent) + + return configurator.k8sClient.Update(ctx, agent) +} + func (configurator *Configurator) updateChangeStatus(ctx context.Context, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { var statusUpdate ConfigurationChangeStatusUpdate if encounteredError == nil { @@ -115,13 +138,6 @@ func (configurator *Configurator) updateChangeStatus(ctx context.Context, change return configurator.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) } -// applyChange will sync the required changes and push them to the k8s api-server -// the input agent will be modified after this function and will no longer match the original -func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange, agent *cbcontainersv1.CBContainersAgent) error { - applyChangesToCR(change, agent) - return configurator.k8sClient.Update(ctx, agent) -} - // getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions // if no resource is defined, nil is returned // in case more than 1 resource is defined (which is not supported), only the first one is returned diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index c4d5ab01..d9ce51e5 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -16,25 +16,31 @@ import ( "time" ) -// TODO: Compatibility checks +// TODO: What error data to show and what not? // TODO: Reads cluster, etc from CR correctly? // TODO: Respects proxy // TODO: Review gomock.any usages here +// TODO: error on compatiblity calls + type configuratorMocks struct { - k8sClient *k8sMocks.MockClient - api *mocksConfigurator.MockConfigurationChangesAPI + k8sClient *k8sMocks.MockClient + configChangesAPI *mocksConfigurator.MockConfigurationChangesAPI + changeValidator *mocksConfigurator.MockChangeValidator } func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { k8sClient := k8sMocks.NewMockClient(ctrl) - api := mocksConfigurator.NewMockConfigurationChangesAPI(ctrl) + configChangesAPI := mocksConfigurator.NewMockConfigurationChangesAPI(ctrl) + changeValidator := mocksConfigurator.NewMockChangeValidator(ctrl) + + configurator := remote_configuration.NewConfigurator(k8sClient, configChangesAPI, changeValidator, logr.Discard()) - configurator := remote_configuration.NewConfigurator(k8sClient, api, logr.Discard()) mocksHolder := configuratorMocks{ - k8sClient: k8sClient, - api: api, + k8sClient: k8sClient, + configChangesAPI: configChangesAPI, + changeValidator: changeValidator, } return configurator, mocksHolder @@ -47,19 +53,13 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) var initialGeneration, finalGeneration int64 = 1, 2 - // TODO: Compatiblity check - setupCRInK8S(mocks.k8sClient, &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}}) + setupChangeValidatorToAcceptAll(mocks.changeValidator) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - - assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { - assert.Equal(t, *configChange.AgentVersion, agent.Spec.Version) - agent.ObjectMeta.Generation = finalGeneration - }) + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, finalGeneration, update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) @@ -71,10 +71,42 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) + assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, *configChange.AgentVersion, agent.Spec.Version) + agent.ObjectMeta.Generation = finalGeneration + }) + err := configurator.RunIteration(context.Background()) assert.NoError(t, err) } +func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { + ctrl := gomock.NewController(t) + + configurator, mocks := setupConfigurator(ctrl) + + cr := setupCRInK8S(mocks.k8sClient, nil) + + configChange := remote_configuration.RandomNonNilChange() + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) + + mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(false, "your data is wrong pal") + + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) + + err := configurator.RunIteration(context.Background()) + assert.Error(t, err) +} + func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { testCases := []struct { name string @@ -103,8 +135,8 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) setupCRInK8S(mocks.k8sClient, nil) - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) err := configurator.RunIteration(context.Background()) assert.NoError(t, err) @@ -129,13 +161,15 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) setupCRInK8S(mocks.k8sClient, nil) - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) + setupChangeValidatorToAcceptAll(mocks.changeValidator) + + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { assert.Equal(t, expectedVersion, agent.Spec.Version) }) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, olderChange.ID, update.ID) return nil @@ -154,7 +188,7 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) setupCRInK8S(mocks.k8sClient, nil) errFromService := errors.New("some error") - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) returnedErr := configurator.RunIteration(context.Background()) @@ -183,14 +217,15 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) setupCRInK8S(mocks.k8sClient, nil) + setupChangeValidatorToAcceptAll(mocks.changeValidator) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) errFromService := errors.New("some error") mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) @@ -208,19 +243,20 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) - defer ctrl.Finish() + defer ctrl.Finish() // tODO: Remove all; redundant since 1.14 configurator, mocks := setupConfigurator(ctrl) setupCRInK8S(mocks.k8sClient, nil) + setupChangeValidatorToAcceptAll(mocks.changeValidator) configChange := remote_configuration.RandomNonNilChange() - mocks.api.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) + mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") - mocks.api.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) + mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -241,7 +277,7 @@ func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { } // setupCRInK8S ensures the mock client will return 1 agent item for List calls - either the provided one or an empty CR otherwise -func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAgent) { +func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAgent) *cbcontainersv1.CBContainersAgent { if item == nil { item = &cbcontainersv1.CBContainersAgent{ TypeMeta: metav1.TypeMeta{}, @@ -254,6 +290,8 @@ func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAg Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { list.Items = []cbcontainersv1.CBContainersAgent{*item} }) + + return item } func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcontainersv1.CBContainersAgent)) { @@ -266,3 +304,7 @@ func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcont return nil }) } + +func setupChangeValidatorToAcceptAll(mock *mocksConfigurator.MockChangeValidator) { + mock.EXPECT().ValidateChange(gomock.Any(), gomock.Any()).Return(true, "") +} diff --git a/remote_configuration/change_applier.go b/remote_configuration/custom_resource_changer.go similarity index 62% rename from remote_configuration/change_applier.go rename to remote_configuration/custom_resource_changer.go index 428ae893..90ae9717 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/custom_resource_changer.go @@ -6,44 +6,7 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/models" ) -// TODO: Move somewhere else - -type Sensor struct { - Version string `json:"version"` - IsLatest bool `json:"is_latest" ` - SupportsRuntime bool `json:"supports_runtime"` - SupportsClusterScanning bool `json:"supports_cluster_scanning"` - SupportsCndr bool `json:"supports_cndr"` -} - -type CustomResourceChanger struct { - SensorData []Sensor - OperatorCompatibilityData models.OperatorCompatibility -} - -func (changer *CustomResourceChanger) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { - var versionToValidate string - - // If the change will be modifying the agent version as well, we need to check what the _new_ version supports - if change.AgentVersion != nil { - versionToValidate = *change.AgentVersion - } else { - // Otherwise the current agent must actually work with the requested features - versionToValidate = cr.Spec.Version - } - - if sensorAndOperatorCompatible, msg := changer.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { - return false, msg - } - - return changer.validateSensorAndFeatureCompatibility(versionToValidate, change) -} - -func (changer *CustomResourceChanger) ApplyChangeToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { - if isValid, msg := changer.ValidateChange(change, cr); !isValid { - return fmt.Errorf("provided change cannot be applied to the custom resource with reason (%s)", msg) - } - +func ApplyChangeToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { resetVersion := func(ptrToField *string) { if ptrToField != nil && *ptrToField != "" { *ptrToField = "" @@ -76,12 +39,33 @@ func (changer *CustomResourceChanger) ApplyChangeToCR(change ConfigurationChange } cr.Spec.Components.Cndr.Enabled = change.EnableCNDR } +} + +type ConfigurationChangeValidator struct { + SensorData []models.SensorMetadata + OperatorCompatibilityData models.OperatorCompatibility +} + +func (validator *ConfigurationChangeValidator) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { + var versionToValidate string + + // If the change will be modifying the agent version as well, we need to check what the _new_ version supports + if change.AgentVersion != nil { + versionToValidate = *change.AgentVersion + } else { + // Otherwise the current agent must actually work with the requested features + versionToValidate = cr.Spec.Version + } + + if sensorAndOperatorCompatible, msg := validator.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { + return false, msg + } - return nil + return validator.validateSensorAndFeatureCompatibility(versionToValidate, change) } -func (changer *CustomResourceChanger) findMatchingSensor(sensorVersion string) (*Sensor, string) { - for _, sensor := range changer.SensorData { +func (validator *ConfigurationChangeValidator) findMatchingSensor(sensorVersion string) (*models.SensorMetadata, string) { + for _, sensor := range validator.SensorData { if sensor.Version == sensorVersion { return &sensor, "" } @@ -90,15 +74,15 @@ func (changer *CustomResourceChanger) findMatchingSensor(sensorVersion string) ( return nil, fmt.Sprintf("could not find sensor metadata for version %s", sensorVersion) } -func (changer *CustomResourceChanger) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { - if err := changer.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { +func (validator *ConfigurationChangeValidator) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { + if err := validator.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { return false, err.Error() } return true, "" } -func (changer *CustomResourceChanger) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) (bool, string) { - sensor, msg := changer.findMatchingSensor(targetVersion) +func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) (bool, string) { + sensor, msg := validator.findMatchingSensor(targetVersion) if sensor == nil { return false, msg } diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/custom_resource_changer_test.go similarity index 84% rename from remote_configuration/change_applier_test.go rename to remote_configuration/custom_resource_changer_test.go index 858c6c16..b7512a7e 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/custom_resource_changer_test.go @@ -3,7 +3,6 @@ package remote_configuration_test import ( "fmt" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" "github.com/vmware/cbcontainers-operator/remote_configuration" @@ -23,14 +22,14 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { testCases := []struct { name string change remote_configuration.ConfigurationChange - sensorMeta remote_configuration.Sensor + sensorMeta models.SensorMetadata }{ { name: "cluster scanning", change: remote_configuration.ConfigurationChange{ EnableClusterScanning: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsClusterScanning: false, }, }, @@ -39,7 +38,7 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { change: remote_configuration.ConfigurationChange{ EnableRuntime: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsRuntime: false, }, }, @@ -48,7 +47,7 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { change: remote_configuration.ConfigurationChange{ EnableCNDR: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsCndr: false, }, }, @@ -57,8 +56,8 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { for _, tC := range testCases { version := "dummy-version" tC.sensorMeta.Version = version - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{tC.sensorMeta}, + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, } t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { @@ -87,14 +86,14 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { testCases := []struct { name string change remote_configuration.ConfigurationChange - sensorMeta remote_configuration.Sensor + sensorMeta models.SensorMetadata }{ { name: "cluster scanning", change: remote_configuration.ConfigurationChange{ EnableClusterScanning: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsClusterScanning: true, }, }, @@ -103,7 +102,7 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { change: remote_configuration.ConfigurationChange{ EnableRuntime: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsRuntime: true, }, }, @@ -112,7 +111,7 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { change: remote_configuration.ConfigurationChange{ EnableCNDR: truePtr, }, - sensorMeta: remote_configuration.Sensor{ + sensorMeta: models.SensorMetadata{ SupportsCndr: true, }, }, @@ -121,8 +120,8 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { for _, tC := range testCases { version := "dummy-version" tC.sensorMeta.Version = version - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{tC.sensorMeta}, + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, } t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { @@ -173,8 +172,8 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { for _, tC := range testCases { t.Run(tC.name, func(t *testing.T) { - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{{Version: tC.versionToApply}}, + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, OperatorCompatibilityData: tC.operatorCompatibility, } @@ -230,8 +229,8 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { for _, tC := range testCases { t.Run(tC.name, func(t *testing.T) { - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{{Version: tC.versionToApply}}, + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, OperatorCompatibilityData: tC.operatorCompatibility, } @@ -334,20 +333,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{ - { - Version: crVersion, - SupportsRuntime: true, - SupportsCndr: true, - SupportsClusterScanning: true, - }, - }, - } - err := target.ApplyChangeToCR(testCase.change, &testCase.initialCR) - - require.NoError(t, err) + remote_configuration.ApplyChangeToCR(testCase.change, &testCase.initialCR) testCase.assertFinalCR(t, &testCase.initialCR) }) } @@ -358,10 +344,8 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { newVersion := "new-version" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} - target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: newVersion}}} - err := target.ApplyChangeToCR(change, &cr) - require.NoError(t, err) + remote_configuration.ApplyChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) } @@ -369,10 +353,8 @@ func TestMissingVersionDoesNotModifyCR(t *testing.T) { originalVersion := "my-version-42" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := remote_configuration.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} - target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: originalVersion, SupportsRuntime: true}}} - err := target.ApplyChangeToCR(change, &cr) - require.NoError(t, err) + remote_configuration.ApplyChangeToCR(change, &cr) assert.Equal(t, originalVersion, cr.Spec.Version) } @@ -436,10 +418,8 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { newVersion := "new-version" change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} - target := remote_configuration.CustomResourceChanger{SensorData: []remote_configuration.Sensor{{Version: newVersion}}} - err := target.ApplyChangeToCR(change, &cr) - require.NoError(t, err) + remote_configuration.ApplyChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields @@ -454,19 +434,6 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { assert.Empty(t, cr.Spec.Components.Cndr.Sensor.Image.Tag) } -func TestInvalidChangeReturnsError(t *testing.T) { - version := "test" - change := remote_configuration.ConfigurationChange{EnableClusterScanning: truePtr} - cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} - target := remote_configuration.CustomResourceChanger{ - SensorData: []remote_configuration.Sensor{{Version: version, SupportsClusterScanning: false}}, - } - - err := target.ApplyChangeToCR(change, cr) - - assert.Error(t, err) -} - func prettyPrintBoolPtr(v *bool) string { if v == nil { return "" diff --git a/remote_configuration/mocks/generated.go b/remote_configuration/mocks/generated.go index 7a354304..b1874522 100644 --- a/remote_configuration/mocks/generated.go +++ b/remote_configuration/mocks/generated.go @@ -1,3 +1,4 @@ package mocks -//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/config_applier ConfigurationChangesAPI +//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ConfigurationChangesAPI +//go:generate mockgen -destination mock_change_validator.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ChangeValidator diff --git a/remote_configuration/mocks/mock_change_validator.go b/remote_configuration/mocks/mock_change_validator.go new file mode 100644 index 00000000..1fe045cf --- /dev/null +++ b/remote_configuration/mocks/mock_change_validator.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ChangeValidator) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" +) + +// MockChangeValidator is a mock of ChangeValidator interface. +type MockChangeValidator struct { + ctrl *gomock.Controller + recorder *MockChangeValidatorMockRecorder +} + +// MockChangeValidatorMockRecorder is the mock recorder for MockChangeValidator. +type MockChangeValidatorMockRecorder struct { + mock *MockChangeValidator +} + +// NewMockChangeValidator creates a new mock instance. +func NewMockChangeValidator(ctrl *gomock.Controller) *MockChangeValidator { + mock := &MockChangeValidator{ctrl: ctrl} + mock.recorder = &MockChangeValidatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockChangeValidator) EXPECT() *MockChangeValidatorMockRecorder { + return m.recorder +} + +// ValidateChange mocks base method. +func (m *MockChangeValidator) ValidateChange(arg0 remote_configuration.ConfigurationChange, arg1 *v1.CBContainersAgent) (bool, string) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateChange", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(string) + return ret0, ret1 +} + +// ValidateChange indicates an expected call of ValidateChange. +func (mr *MockChangeValidatorMockRecorder) ValidateChange(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateChange", reflect.TypeOf((*MockChangeValidator)(nil).ValidateChange), arg0, arg1) +} diff --git a/remote_configuration/mocks/mock_configuration_api.go b/remote_configuration/mocks/mock_configuration_api.go index 2a5c9d40..1edc7b02 100644 --- a/remote_configuration/mocks/mock_configuration_api.go +++ b/remote_configuration/mocks/mock_configuration_api.go @@ -9,7 +9,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - config_applier "github.com/vmware/cbcontainers-operator/remote_configuration" + remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" ) // MockConfigurationChangesAPI is a mock of ConfigurationChangesAPI interface. @@ -36,10 +36,10 @@ func (m *MockConfigurationChangesAPI) EXPECT() *MockConfigurationChangesAPIMockR } // GetConfigurationChanges mocks base method. -func (m *MockConfigurationChangesAPI) GetConfigurationChanges(arg0 context.Context) ([]config_applier.ConfigurationChange, error) { +func (m *MockConfigurationChangesAPI) GetConfigurationChanges(arg0 context.Context) ([]remote_configuration.ConfigurationChange, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) - ret0, _ := ret[0].([]config_applier.ConfigurationChange) + ret0, _ := ret[0].([]remote_configuration.ConfigurationChange) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -51,7 +51,7 @@ func (mr *MockConfigurationChangesAPIMockRecorder) GetConfigurationChanges(arg0 } // UpdateConfigurationChangeStatus mocks base method. -func (m *MockConfigurationChangesAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 config_applier.ConfigurationChangeStatusUpdate) error { +func (m *MockConfigurationChangesAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 remote_configuration.ConfigurationChangeStatusUpdate) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) ret0, _ := ret[0].(error) From 0cfbcdadfc9c1e5581a39397127e7b56cde505a1 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 5 Sep 2023 10:34:50 +0300 Subject: [PATCH 33/65] Change validator interface and add fetcher to download metadata from the API --- remote_configuration/configurator.go | 8 ++- remote_configuration/configurator_test.go | 4 +- .../custom_resource_changer.go | 71 ++++++++++++++----- .../custom_resource_changer_test.go | 30 ++++---- .../mocks/mock_change_validator.go | 7 +- 5 files changed, 77 insertions(+), 43 deletions(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 56ce529e..df4a6cf1 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -12,6 +12,8 @@ import ( // TODO: Respect proxy config +// TODO: Split errors into visible and not visible + const ( timeoutSingleIteration = time.Second * 60 ) @@ -25,7 +27,7 @@ type ConfigurationChangesAPI interface { } type ChangeValidator interface { - ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) + ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error } type Configurator struct { @@ -108,8 +110,8 @@ func (configurator *Configurator) getPendingChange(ctx context.Context) (*Config // applyChange will sync the required changes and push them to the k8s api-server // the input agent will be modified after this function and will no longer match the original func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange, agent *cbcontainersv1.CBContainersAgent) error { - if validChange, reason := configurator.changeValidator.ValidateChange(change, agent); !validChange { - return fmt.Errorf("provided change with ID (%s) is not applicable due to (%s)", change.ID, reason) + if err := configurator.changeValidator.ValidateChange(change, agent); err != nil { + return err } ApplyChangeToCR(change, agent) diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index d9ce51e5..996f2f41 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -90,7 +90,7 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { configChange := remote_configuration.RandomNonNilChange() mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(false, "your data is wrong pal") + mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(errors.New("your data is wrong pal")) mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { @@ -306,5 +306,5 @@ func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcont } func setupChangeValidatorToAcceptAll(mock *mocksConfigurator.MockChangeValidator) { - mock.EXPECT().ValidateChange(gomock.Any(), gomock.Any()).Return(true, "") + mock.EXPECT().ValidateChange(gomock.Any(), gomock.Any()).Return(nil) } diff --git a/remote_configuration/custom_resource_changer.go b/remote_configuration/custom_resource_changer.go index 90ae9717..5a2a9800 100644 --- a/remote_configuration/custom_resource_changer.go +++ b/remote_configuration/custom_resource_changer.go @@ -41,12 +41,51 @@ func ApplyChangeToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainers } } +// TODO: Move + +type invalidChangeError struct { + msg string +} + +func (i invalidChangeError) Error() string { + return i.msg +} + +type SensorMetadataAPI interface { + GetSensorsMetadata() ([]models.SensorMetadata, error) + GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) +} + +type ConfigurationChangeFetcher struct { + operatorVersion string + api SensorMetadataAPI +} + +func (fetcher *ConfigurationChangeFetcher) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { + compatibilityMatrix, err := fetcher.api.GetCompatibilityMatrixEntryFor(fetcher.operatorVersion) + if err != nil { + return err + } + + sensors, err := fetcher.api.GetSensorsMetadata() + if err != nil { + return err + } + + validator := ConfigurationChangeValidator{ + SensorData: sensors, + OperatorCompatibilityData: *compatibilityMatrix, + } + + return validator.ValidateChange(change, cr) +} + type ConfigurationChangeValidator struct { SensorData []models.SensorMetadata OperatorCompatibilityData models.OperatorCompatibility } -func (validator *ConfigurationChangeValidator) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) (bool, string) { +func (validator *ConfigurationChangeValidator) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { var versionToValidate string // If the change will be modifying the agent version as well, we need to check what the _new_ version supports @@ -57,53 +96,53 @@ func (validator *ConfigurationChangeValidator) ValidateChange(change Configurati versionToValidate = cr.Spec.Version } - if sensorAndOperatorCompatible, msg := validator.validateOperatorAndSensorVersionCompatibility(versionToValidate); !sensorAndOperatorCompatible { - return false, msg + if err := validator.validateOperatorAndSensorVersionCompatibility(versionToValidate); err != nil { + return err } return validator.validateSensorAndFeatureCompatibility(versionToValidate, change) } -func (validator *ConfigurationChangeValidator) findMatchingSensor(sensorVersion string) (*models.SensorMetadata, string) { +func (validator *ConfigurationChangeValidator) findMatchingSensor(sensorVersion string) *models.SensorMetadata { for _, sensor := range validator.SensorData { if sensor.Version == sensorVersion { - return &sensor, "" + return &sensor } } - return nil, fmt.Sprintf("could not find sensor metadata for version %s", sensorVersion) + return nil } -func (validator *ConfigurationChangeValidator) validateOperatorAndSensorVersionCompatibility(sensorVersion string) (bool, string) { +func (validator *ConfigurationChangeValidator) validateOperatorAndSensorVersionCompatibility(sensorVersion string) error { if err := validator.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { - return false, err.Error() + return invalidChangeError{msg: err.Error()} } - return true, "" + return nil } -func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) (bool, string) { - sensor, msg := validator.findMatchingSensor(targetVersion) +func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) error { + sensor := validator.findMatchingSensor(targetVersion) if sensor == nil { - return false, msg + return fmt.Errorf("could not find sensor metadata for version %s", targetVersion) } if change.EnableClusterScanning != nil && *change.EnableClusterScanning == true && !sensor.SupportsClusterScanning { - return false, fmt.Sprintf("sensor version %s does not support cluster scanning feature", targetVersion) + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support cluster scanning feature", targetVersion)} } if change.EnableRuntime != nil && *change.EnableRuntime == true && !sensor.SupportsRuntime { - return false, fmt.Sprintf("sensor version %s does not support runtime protection feature", targetVersion) + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support runtime protection feature", targetVersion)} } if change.EnableCNDR != nil && *change.EnableCNDR == true && !sensor.SupportsCndr { - return false, fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", targetVersion) + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", targetVersion)} } - return true, "" + return nil } diff --git a/remote_configuration/custom_resource_changer_test.go b/remote_configuration/custom_resource_changer_test.go index b7512a7e..8601cc4c 100644 --- a/remote_configuration/custom_resource_changer_test.go +++ b/remote_configuration/custom_resource_changer_test.go @@ -64,20 +64,18 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { tC.change.AgentVersion = nil cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} - valid, msg := target.ValidateChange(tC.change, cr) + err := target.ValidateChange(tC.change, cr) - assert.False(t, valid) - assert.NotEmpty(t, msg) + assert.Error(t, err) }) t.Run(fmt.Sprintf("change also applies agent version, %s not supported by that version", tC.name), func(t *testing.T) { tC.change.AgentVersion = &version cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} - valid, msg := target.ValidateChange(tC.change, cr) + err := target.ValidateChange(tC.change, cr) - assert.False(t, valid) - assert.NotEmpty(t, msg) + assert.Error(t, err) }) } } @@ -128,20 +126,18 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { tC.change.AgentVersion = nil cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} - valid, msg := target.ValidateChange(tC.change, cr) + err := target.ValidateChange(tC.change, cr) - assert.True(t, valid) - assert.Empty(t, msg) + assert.NoError(t, err) }) t.Run(fmt.Sprintf("change also applies agent version, %s is supported by that version", tC.name), func(t *testing.T) { tC.change.AgentVersion = &version cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} - valid, msg := target.ValidateChange(tC.change, cr) + err := target.ValidateChange(tC.change, cr) - assert.True(t, valid) - assert.Empty(t, msg) + assert.NoError(t, err) }) } } @@ -180,9 +176,8 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} - valid, msg := target.ValidateChange(change, cr) - assert.False(t, valid) - assert.NotEmpty(t, msg) + err := target.ValidateChange(change, cr) + assert.Error(t, err) }) } } @@ -237,9 +232,8 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} - valid, msg := target.ValidateChange(change, cr) - assert.True(t, valid) - assert.Empty(t, msg) + err := target.ValidateChange(change, cr) + assert.NoError(t, err) }) } } diff --git a/remote_configuration/mocks/mock_change_validator.go b/remote_configuration/mocks/mock_change_validator.go index 1fe045cf..8ab92f57 100644 --- a/remote_configuration/mocks/mock_change_validator.go +++ b/remote_configuration/mocks/mock_change_validator.go @@ -36,12 +36,11 @@ func (m *MockChangeValidator) EXPECT() *MockChangeValidatorMockRecorder { } // ValidateChange mocks base method. -func (m *MockChangeValidator) ValidateChange(arg0 remote_configuration.ConfigurationChange, arg1 *v1.CBContainersAgent) (bool, string) { +func (m *MockChangeValidator) ValidateChange(arg0 remote_configuration.ConfigurationChange, arg1 *v1.CBContainersAgent) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ValidateChange", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(string) - return ret0, ret1 + ret0, _ := ret[0].(error) + return ret0 } // ValidateChange indicates an expected call of ValidateChange. From d3ed89c8309a624b8620fdaca3ba6363b829be40 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 5 Sep 2023 13:19:54 +0300 Subject: [PATCH 34/65] Add auth_provider and adjust test to match --- cbcontainers/state/operator/auth_provider.go | 43 +++++++++ controllers/cbcontainersagent_controller.go | 29 ++---- .../cbcontainersagent_controller_test.go | 96 +++++++++---------- controllers/mocks/generated.go | 3 +- .../mocks/mock_access_token_provider.go | 51 ++++++++++ controllers/mocks/mock_agent_processor.go | 51 ++++++++++ controllers/mocks/mock_cluster_processor.go | 51 ---------- 7 files changed, 205 insertions(+), 119 deletions(-) create mode 100644 cbcontainers/state/operator/auth_provider.go create mode 100644 controllers/mocks/mock_access_token_provider.go create mode 100644 controllers/mocks/mock_agent_processor.go delete mode 100644 controllers/mocks/mock_cluster_processor.go diff --git a/cbcontainers/state/operator/auth_provider.go b/cbcontainers/state/operator/auth_provider.go new file mode 100644 index 00000000..2e26a117 --- /dev/null +++ b/cbcontainers/state/operator/auth_provider.go @@ -0,0 +1,43 @@ +package operator + +import ( + "context" + "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type SecretAccessTokenProvider struct { + k8sClient client.Client +} + +func NewSecretAccessTokenProvider(k8sClient client.Client) *SecretAccessTokenProvider { + return &SecretAccessTokenProvider{k8sClient: k8sClient} +} + +// GetCBAccessToken will attempt to read the access token value from a secret in the deployed namespace +// The secret should be defined in the provided Custom resource +func (provider *SecretAccessTokenProvider) GetCBAccessToken( + ctx context.Context, + cbContainersCluster *cbcontainersv1.CBContainersAgent, + deployedNamespace string, +) (string, error) { + accessTokenSecretNamespacedName := types.NamespacedName{ + Name: cbContainersCluster.Spec.AccessTokenSecretName, + Namespace: deployedNamespace, + } + accessTokenSecret := &corev1.Secret{} + if err := provider.k8sClient.Get(ctx, accessTokenSecretNamespacedName, accessTokenSecret); err != nil { + return "", fmt.Errorf("couldn't find access token secret k8s object: %v", err) + } + + accessToken := string(accessTokenSecret.Data[commonState.AccessTokenSecretKeyName]) + if accessToken == "" { + return "", fmt.Errorf("the k8s secret %v is missing the key %v", accessTokenSecretNamespacedName, commonState.AccessTokenSecretKeyName) + } + + return accessToken, nil +} diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 54058136..3e1f7aae 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -29,11 +29,9 @@ import ( "github.com/go-logr/logr" "github.com/vmware/cbcontainers-operator/cbcontainers/models" applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,6 +52,10 @@ type AgentProcessor interface { Process(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (*models.RegistrySecretValues, error) } +type AccessTokenProvider interface { + GetCBAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, namespace string) (string, error) +} + type CBContainersAgentController struct { client.Client Log logr.Logger @@ -62,7 +64,8 @@ type CBContainersAgentController struct { StateApplier StateApplier K8sVersion string // Namespace is the kubernetes namespace for all agent components - Namespace string + Namespace string + AccessTokenProvider AccessTokenProvider } func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { @@ -124,10 +127,13 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re return ctrl.SetControllerReference(cbContainersAgent, controlledResource, r.Scheme) } - accessToken, err := r.getAccessToken(context.Background(), cbContainersAgent) + accessToken, err := r.AccessTokenProvider.GetCBAccessToken(ctx, cbContainersAgent, r.Namespace) if err != nil { return ctrl.Result{}, err } + if accessToken == "" { + return ctrl.Result{}, fmt.Errorf("CB access token has empty value, cannot continue") + } var registrySecret *models.RegistrySecretValues if cbContainersAgent.Spec.Components.Settings.ShouldCreateDefaultImagePullSecrets() { @@ -166,21 +172,6 @@ func (r *CBContainersAgentController) getRegistrySecretValues(ctx context.Contex return r.ClusterProcessor.Process(cbContainersCluster, accessToken) } -func (r *CBContainersAgentController) getAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent) (string, error) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: cbContainersCluster.Spec.AccessTokenSecretName, Namespace: r.Namespace} - accessTokenSecret := &corev1.Secret{} - if err := r.Get(ctx, accessTokenSecretNamespacedName, accessTokenSecret); err != nil { - return "", fmt.Errorf("couldn't find access token secret k8s object: %v", err) - } - - accessToken := string(accessTokenSecret.Data[commonState.AccessTokenSecretKeyName]) - if accessToken == "" { - return "", fmt.Errorf("the k8s secret %v is missing the key %v", accessTokenSecretNamespacedName, commonState.AccessTokenSecretKeyName) - } - - return accessToken, nil -} - func (r *CBContainersAgentController) updateCRStatus(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, agentStateWasChanged bool) error { // If we don't expect more changes (i.e. nothing changed in reality) and we haven't updated the status, we do so now. if !agentStateWasChanged && cbContainersCluster.Status.ObservedGeneration < cbContainersCluster.ObjectMeta.Generation { diff --git a/controllers/cbcontainersagent_controller_test.go b/controllers/cbcontainersagent_controller_test.go index 1eec9aab..06ff6a60 100644 --- a/controllers/cbcontainersagent_controller_test.go +++ b/controllers/cbcontainersagent_controller_test.go @@ -9,30 +9,28 @@ import ( "testing" "time" - logrTesting "github.com/go-logr/logr/testing" + logrTesting "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils" testUtilsMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" "github.com/vmware/cbcontainers-operator/controllers" "github.com/vmware/cbcontainers-operator/controllers/mocks" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrlRuntime "sigs.k8s.io/controller-runtime" ) type SetupClusterControllerTest func(*ClusterControllerTestMocks) type ClusterControllerTestMocks struct { - client *testUtilsMocks.MockClient - statusWriter *testUtilsMocks.MockStatusWriter - clusterProcessor *mocks.MockClusterProcessor - stateApplier *mocks.MockStateApplier - ctx context.Context + client *testUtilsMocks.MockClient + statusWriter *testUtilsMocks.MockStatusWriter + accessTokenProvider *mocks.MockAccessTokenProvider + mockAgentProcessor *mocks.MockAgentProcessor + stateApplier *mocks.MockStateApplier + ctx context.Context } const ( @@ -70,11 +68,12 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr mockK8SClient.EXPECT().Status().Return(mockStatusWriter).AnyTimes() mocksObjects := &ClusterControllerTestMocks{ - ctx: context.TODO(), - client: mockK8SClient, - statusWriter: mockStatusWriter, - clusterProcessor: mocks.NewMockClusterProcessor(ctrl), - stateApplier: mocks.NewMockStateApplier(ctrl), + ctx: context.TODO(), + client: mockK8SClient, + statusWriter: mockStatusWriter, + accessTokenProvider: mocks.NewMockAccessTokenProvider(ctrl), + mockAgentProcessor: mocks.NewMockAgentProcessor(ctrl), + stateApplier: mocks.NewMockStateApplier(ctrl), } for _, setup := range setups { @@ -83,12 +82,13 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr controller := &controllers.CBContainersAgentController{ Client: mocksObjects.client, - Log: logrTesting.NewTestLogger(t), + Log: logrTesting.New(t), Scheme: &runtime.Scheme{}, Namespace: agentNamespace, - ClusterProcessor: mocksObjects.clusterProcessor, - StateApplier: mocksObjects.stateApplier, + AccessTokenProvider: mocksObjects.accessTokenProvider, + ClusterProcessor: mocksObjects.mockAgentProcessor, + StateApplier: mocksObjects.stateApplier, } return controller.Reconcile(mocksObjects.ctx, ctrlRuntime.Request{}) @@ -109,15 +109,11 @@ func setupClusterCustomResource(items ...cbcontainersv1.CBContainersAgent) Setup } } -func setUpTokenSecretValues(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}). - Do(func(ctx context.Context, namespacedName types.NamespacedName, secret *corev1.Secret, _ ...interface{}) { - secret.Data = map[string][]byte{ - commonState.AccessTokenSecretKeyName: []byte(MyClusterTokenValue), - } - }). - Return(nil) +func setUpAccessToken(testMocks *ClusterControllerTestMocks) { + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return(MyClusterTokenValue, nil) } func TestListClusterResourcesErrorShouldReturnError(t *testing.T) { @@ -152,8 +148,10 @@ func TestFindingMoreThanOneClusterResourceShouldReturnError(t *testing.T) { func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource(), func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(fmt.Errorf("")) + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return("", fmt.Errorf("some error")) }) require.Error(t, err) @@ -161,8 +159,10 @@ func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { func TestTokenSecretWithoutTokenValueShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource(), func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(nil) + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return("", nil) }) require.Error(t, err) @@ -172,16 +172,16 @@ func TestClusterReconcile(t *testing.T) { secretValues := &models.RegistrySecretValues{Data: map[string][]byte{test_utils.RandomString(): {}}} t.Run("When processor returns error, reconcile should return error", func(t *testing.T) { - _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(nil, fmt.Errorf("")) + _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(nil, fmt.Errorf("")) }) require.Error(t, err) }) t.Run("When state applier returns error, reconcile should return error", func(t *testing.T) { - _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(false, fmt.Errorf("")) }) @@ -189,8 +189,8 @@ func TestClusterReconcile(t *testing.T) { }) t.Run("When state applier returns state was changed, reconcile should return Requeue true", func(t *testing.T) { - result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(true, nil) }) @@ -199,8 +199,8 @@ func TestClusterReconcile(t *testing.T) { }) t.Run("When state applier returns state was not changed, reconcile should return default Requeue", func(t *testing.T) { - result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(false, nil) }) @@ -217,8 +217,8 @@ func TestStatusUpdates(t *testing.T) { resourceWithStatus.ObjectMeta.Generation = 2 resourceWithStatus.Status.ObservedGeneration = 1 - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceWithStatus.Spec), secretValues, gomock.Any()).Return(true, nil) testMocks.statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).MaxTimes(0) }) @@ -232,8 +232,8 @@ func TestStatusUpdates(t *testing.T) { resourceWithStatus.ObjectMeta.Generation = 1 resourceWithStatus.Status.ObservedGeneration = 1 - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceWithStatus.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).MaxTimes(0) }) @@ -250,8 +250,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Times(1).Return(nil) }) @@ -268,8 +268,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Return(k8sErrors.NewConflict(schema.GroupResource{}, "conflict", nil)) }) @@ -286,8 +286,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Return(fmt.Errorf("some error")) }) diff --git a/controllers/mocks/generated.go b/controllers/mocks/generated.go index c343b9b6..6a139b21 100644 --- a/controllers/mocks/generated.go +++ b/controllers/mocks/generated.go @@ -1,4 +1,5 @@ package mocks //go:generate mockgen -destination mock_state_applier.go -package mocks github.com/vmware/cbcontainers-operator/controllers StateApplier -//go:generate mockgen -destination mock_cluster_processor.go -package mocks github.com/vmware/cbcontainers-operator/controllers ClusterProcessor +//go:generate mockgen -destination mock_agent_processor.go -package mocks github.com/vmware/cbcontainers-operator/controllers AgentProcessor +//go:generate mockgen -destination mock_access_token_provider.go -package mocks github.com/vmware/cbcontainers-operator/controllers AccessTokenProvider diff --git a/controllers/mocks/mock_access_token_provider.go b/controllers/mocks/mock_access_token_provider.go new file mode 100644 index 00000000..68cca361 --- /dev/null +++ b/controllers/mocks/mock_access_token_provider.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: AccessTokenProvider) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" +) + +// MockAccessTokenProvider is a mock of AccessTokenProvider interface. +type MockAccessTokenProvider struct { + ctrl *gomock.Controller + recorder *MockAccessTokenProviderMockRecorder +} + +// MockAccessTokenProviderMockRecorder is the mock recorder for MockAccessTokenProvider. +type MockAccessTokenProviderMockRecorder struct { + mock *MockAccessTokenProvider +} + +// NewMockAccessTokenProvider creates a new mock instance. +func NewMockAccessTokenProvider(ctrl *gomock.Controller) *MockAccessTokenProvider { + mock := &MockAccessTokenProvider{ctrl: ctrl} + mock.recorder = &MockAccessTokenProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessTokenProvider) EXPECT() *MockAccessTokenProviderMockRecorder { + return m.recorder +} + +// GetCBAccessToken mocks base method. +func (m *MockAccessTokenProvider) GetCBAccessToken(arg0 context.Context, arg1 *v1.CBContainersAgent, arg2 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCBAccessToken", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCBAccessToken indicates an expected call of GetCBAccessToken. +func (mr *MockAccessTokenProviderMockRecorder) GetCBAccessToken(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCBAccessToken", reflect.TypeOf((*MockAccessTokenProvider)(nil).GetCBAccessToken), arg0, arg1, arg2) +} diff --git a/controllers/mocks/mock_agent_processor.go b/controllers/mocks/mock_agent_processor.go new file mode 100644 index 00000000..ba7d7aa7 --- /dev/null +++ b/controllers/mocks/mock_agent_processor.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: AgentProcessor) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + models "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// MockAgentProcessor is a mock of AgentProcessor interface. +type MockAgentProcessor struct { + ctrl *gomock.Controller + recorder *MockAgentProcessorMockRecorder +} + +// MockAgentProcessorMockRecorder is the mock recorder for MockAgentProcessor. +type MockAgentProcessorMockRecorder struct { + mock *MockAgentProcessor +} + +// NewMockAgentProcessor creates a new mock instance. +func NewMockAgentProcessor(ctrl *gomock.Controller) *MockAgentProcessor { + mock := &MockAgentProcessor{ctrl: ctrl} + mock.recorder = &MockAgentProcessorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAgentProcessor) EXPECT() *MockAgentProcessorMockRecorder { + return m.recorder +} + +// Process mocks base method. +func (m *MockAgentProcessor) Process(arg0 *v1.CBContainersAgent, arg1 string) (*models.RegistrySecretValues, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Process", arg0, arg1) + ret0, _ := ret[0].(*models.RegistrySecretValues) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Process indicates an expected call of Process. +func (mr *MockAgentProcessorMockRecorder) Process(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Process", reflect.TypeOf((*MockAgentProcessor)(nil).Process), arg0, arg1) +} diff --git a/controllers/mocks/mock_cluster_processor.go b/controllers/mocks/mock_cluster_processor.go deleted file mode 100644 index a3f416c3..00000000 --- a/controllers/mocks/mock_cluster_processor.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: ClusterProcessor) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - models "github.com/vmware/cbcontainers-operator/cbcontainers/models" -) - -// MockClusterProcessor is a mock of ClusterProcessor interface. -type MockClusterProcessor struct { - ctrl *gomock.Controller - recorder *MockClusterProcessorMockRecorder -} - -// MockClusterProcessorMockRecorder is the mock recorder for MockClusterProcessor. -type MockClusterProcessorMockRecorder struct { - mock *MockClusterProcessor -} - -// NewMockClusterProcessor creates a new mock instance. -func NewMockClusterProcessor(ctrl *gomock.Controller) *MockClusterProcessor { - mock := &MockClusterProcessor{ctrl: ctrl} - mock.recorder = &MockClusterProcessorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClusterProcessor) EXPECT() *MockClusterProcessorMockRecorder { - return m.recorder -} - -// Process mocks base method. -func (m *MockClusterProcessor) Process(arg0 *v1.CBContainersAgent, arg1 string) (*models.RegistrySecretValues, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Process", arg0, arg1) - ret0, _ := ret[0].(*models.RegistrySecretValues) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Process indicates an expected call of Process. -func (mr *MockClusterProcessorMockRecorder) Process(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Process", reflect.TypeOf((*MockClusterProcessor)(nil).Process), arg0, arg1) -} From 0a6551cc363a0989fd2e7753f6b3e92a0a79dda5 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 5 Sep 2023 14:07:13 +0300 Subject: [PATCH 35/65] Create AccessTokenProvider in main.go --- main.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 0d8fe978..9f3f727d 100644 --- a/main.go +++ b/main.go @@ -147,13 +147,14 @@ func main() { cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") if err = (&controllers.CBContainersAgentController{ - Client: mgr.GetClient(), - Log: cbContainersAgentLogger, - Scheme: mgr.GetScheme(), - K8sVersion: k8sVersion, - Namespace: operatorNamespace, - ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier), - StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), + Client: mgr.GetClient(), + Log: cbContainersAgentLogger, + Scheme: mgr.GetScheme(), + K8sVersion: k8sVersion, + Namespace: operatorNamespace, + AccessTokenProvider: operator.NewSecretAccessTokenProvider(mgr.GetClient()), + ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier), + StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent") os.Exit(1) From a95ceee3ae065729a8bf44f4988a109885539dfb Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 7 Sep 2023 14:16:29 +0300 Subject: [PATCH 36/65] Extract the ApiGateway from processors.* and use it in configurator as well. --- .../communication/gateway/api_gateway.go | 14 ++ .../gateway}/default_gateway_creator.go | 7 +- .../models/remote_configuration_changes.go | 23 ++ cbcontainers/processors/agent_processor.go | 8 +- main.go | 19 +- remote_configuration/configurator.go | 199 +++++++++++++---- remote_configuration/configurator_test.go | 209 +++++++++++------- .../custom_resource_changer.go | 33 +-- .../custom_resource_changer_test.go | 40 ++-- remote_configuration/mocks/generated.go | 5 +- .../mocks/mock_access_token_provider.go | 51 +++++ .../mocks/mock_api_gateway.go | 95 ++++++++ .../mocks/mock_change_validator.go | 50 ----- .../mocks/mock_configuration_api.go | 65 ------ .../mocks/mock_resource_syncer.go | 67 ++++++ remote_configuration/temp.go | 35 +-- 16 files changed, 588 insertions(+), 332 deletions(-) rename cbcontainers/{processors => communication/gateway}/default_gateway_creator.go (78%) create mode 100644 cbcontainers/models/remote_configuration_changes.go create mode 100644 remote_configuration/mocks/mock_access_token_provider.go create mode 100644 remote_configuration/mocks/mock_api_gateway.go delete mode 100644 remote_configuration/mocks/mock_change_validator.go delete mode 100644 remote_configuration/mocks/mock_configuration_api.go create mode 100644 remote_configuration/mocks/mock_resource_syncer.go diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index 2eff48c1..541345b6 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -1,6 +1,7 @@ package gateway import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -165,3 +166,16 @@ func (gateway *ApiGateway) GetCompatibilityMatrixEntryFor(operatorVersion string return r, nil } + +func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) { + // TODO + return nil, nil +} + +func (gateway *ApiGateway) GetConfigurationChanges(context.Context) ([]models.ConfigurationChange, error) { + return nil, nil +} + +func (gateway *ApiGateway) UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error { + return nil +} diff --git a/cbcontainers/processors/default_gateway_creator.go b/cbcontainers/communication/gateway/default_gateway_creator.go similarity index 78% rename from cbcontainers/processors/default_gateway_creator.go rename to cbcontainers/communication/gateway/default_gateway_creator.go index daa1e718..6e7c30e3 100644 --- a/cbcontainers/processors/default_gateway_creator.go +++ b/cbcontainers/communication/gateway/default_gateway_creator.go @@ -1,8 +1,7 @@ -package processors +package gateway import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" ) type DefaultGatewayCreator struct { @@ -12,9 +11,9 @@ func NewDefaultGatewayCreator() *DefaultGatewayCreator { return &DefaultGatewayCreator{} } -func (creator *DefaultGatewayCreator) CreateGateway(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) { +func (creator *DefaultGatewayCreator) CreateGateway(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (*ApiGateway, error) { spec := cbContainersAgent.Spec - builder := gateway.NewBuilder(spec.Account, spec.ClusterName, accessToken, spec.Gateways.ApiGateway.Host, cbContainersAgent.ObjectMeta.Labels). + builder := NewBuilder(spec.Account, spec.ClusterName, accessToken, spec.Gateways.ApiGateway.Host, cbContainersAgent.ObjectMeta.Labels). SetURLComponents(spec.Gateways.ApiGateway.Scheme, spec.Gateways.ApiGateway.Port, spec.Gateways.ApiGateway.Adapter). SetTLSInsecureSkipVerify(spec.Gateways.GatewayTLS.InsecureSkipVerify). SetTLSRootCAsBundle(spec.Gateways.GatewayTLS.RootCAsBundle) diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go new file mode 100644 index 00000000..22598656 --- /dev/null +++ b/cbcontainers/models/remote_configuration_changes.go @@ -0,0 +1,23 @@ +package models + +type ConfigurationChange struct { + ID string `json:"id"` + Status string `json:"status"` + AgentVersion *string `json:"agent_version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` + EnableCNDR *bool `json:"enable_cndr"` + Timestamp string `json:"timestamp"` +} + +type ConfigurationChangeStatusUpdate struct { + ID string `json:"id"` + Status string `json:"status"` + Reason string `json:"reason"` + // AppliedGeneration tracks the generation of the Custom resource where the change was applied + AppliedGeneration int64 `json:"applied_generation"` + // AppliedTimestamp records when the change was applied in RFC3339 format + AppliedTimestamp string `json:"applied_timestamp"` + + // TODO: CLuster and group. Cluster identifier? +} diff --git a/cbcontainers/processors/agent_processor.go b/cbcontainers/processors/agent_processor.go index 5e8deaba..07fc6717 100644 --- a/cbcontainers/processors/agent_processor.go +++ b/cbcontainers/processors/agent_processor.go @@ -16,9 +16,7 @@ type APIGateway interface { GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) } -type APIGatewayCreator interface { - CreateGateway(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) -} +type APIGatewayCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) type OperatorVersionProvider interface { GetOperatorVersion() (string, error) @@ -79,7 +77,7 @@ func (processor *AgentProcessor) initializeIfNeeded(cbContainersCluster *cbconta } processor.log.Info("Initializing AgentProcessor components") - gateway, err := processor.gatewayCreator.CreateGateway(cbContainersCluster, accessToken) + gateway, err := processor.gatewayCreator(cbContainersCluster, accessToken) if err != nil { return err } @@ -118,7 +116,7 @@ func (processor *AgentProcessor) checkCompatibility(cbContainersAgent *cbcontain } return err } - gateway, err := processor.gatewayCreator.CreateGateway(cbContainersAgent, accessToken) + gateway, err := processor.gatewayCreator(cbContainersAgent, accessToken) if err != nil { processor.log.Error(err, "skipping compatibility check, error while building API gateway") // if there is an error while building the gateway log it and skip the check diff --git a/main.go b/main.go index 9f3f727d..eb997194 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" "github.com/vmware/cbcontainers-operator/cbcontainers/state" "github.com/vmware/cbcontainers-operator/cbcontainers/state/agent_applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" @@ -145,6 +146,11 @@ func main() { clusterIdentifier, k8sVersion := extractConfigurationVariables(mgr) + // TODO: improve + var processorGatewayCreator processors.APIGatewayCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return gateway.NewDefaultGatewayCreator().CreateGateway(cbContainersCluster, accessToken) + } + cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") if err = (&controllers.CBContainersAgentController{ Client: mgr.GetClient(), @@ -153,7 +159,7 @@ func main() { K8sVersion: k8sVersion, Namespace: operatorNamespace, AccessTokenProvider: operator.NewSecretAccessTokenProvider(mgr.GetClient()), - ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier), + ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processorGatewayCreator, operator.NewEnvVersionProvider(), clusterIdentifier), StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent") @@ -176,8 +182,15 @@ func main() { signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") - api := remote_configuration.DummyAPI{} - applier := remote_configuration.NewConfigurator(k8sClient, api, log) + syncer := remote_configuration.NewChangeSyncerImpl(k8sClient) + versionReader := operator.NewEnvVersionProvider() + operatorVersion, err := versionReader.GetOperatorVersion() + if err != nil { + // TODO + panic(err) + } + + applier := remote_configuration.NewConfigurator(remote_configuration.CBGatewayCreator, log, operator.NewSecretAccessTokenProvider(k8sClient), syncer, operatorVersion, operatorNamespace) applierController := remote_configuration.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index df4a6cf1..5c25355c 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" "sigs.k8s.io/controller-runtime/pkg/client" "sort" "time" @@ -18,31 +20,59 @@ const ( timeoutSingleIteration = time.Second * 60 ) -type ConfigurationChangesAPI interface { - // TODO: Get Compatibility matrix - // TODO: Get sensor data +type ApiGateway interface { + GetSensorMetadata() ([]models.SensorMetadata, error) + GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) - GetConfigurationChanges(context.Context) ([]ConfigurationChange, error) - UpdateConfigurationChangeStatus(context.Context, ConfigurationChangeStatusUpdate) error + GetConfigurationChanges(context.Context) ([]models.ConfigurationChange, error) + UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error } +type AccessTokenProvider interface { + GetCBAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, deployedNamespace string) (string, error) +} + +// CBGatewayCreator creates an implementation of ApiGateway that talks to the real backend +func CBGatewayCreator(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) { + creator := gateway.DefaultGatewayCreator{} + return creator.CreateGateway(cbContainersCluster, accessToken) +} + +type ApiCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) + type ChangeValidator interface { - ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error + ValidateChange(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error +} + +type CustomResourceSyncer interface { + GetCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) + ApplyChangeToCR(ctx context.Context, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator ChangeValidator) error } type Configurator struct { - k8sClient client.Client - logger logr.Logger - changesAPI ConfigurationChangesAPI - changeValidator ChangeValidator + logger logr.Logger + accessTokenProvider AccessTokenProvider + apiCreator ApiCreator + operatorVersion string + deployedNamespace string + syncer CustomResourceSyncer } -func NewConfigurator(k8sClient client.Client, configChangesAPI ConfigurationChangesAPI, changeValidator ChangeValidator, logger logr.Logger) *Configurator { +func NewConfigurator( + gatewayCreator ApiCreator, + logger logr.Logger, + accessTokenProvider AccessTokenProvider, + syncer CustomResourceSyncer, + operatorVersion string, + deployedNamespace string, +) *Configurator { return &Configurator{ - k8sClient: k8sClient, - logger: logger, - changesAPI: configChangesAPI, - changeValidator: changeValidator, + logger: logger, + apiCreator: gatewayCreator, + accessTokenProvider: accessTokenProvider, + operatorVersion: operatorVersion, + deployedNamespace: deployedNamespace, + syncer: syncer, } } @@ -51,18 +81,23 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { defer cancel() configurator.logger.Info("Checking for installed agent...") - cr, errGettingCR := configurator.getContainerAgentCR(ctx) - if errGettingCR != nil { - configurator.logger.Error(errGettingCR, "Failed to get CBContainerAgent resource, cannot continue") - return errGettingCR + cr, err := configurator.syncer.GetCR(ctx) + if err != nil { + configurator.logger.Error(err, "Failed to get CBContainerAgent resource, cannot continue") + return err } if cr == nil { configurator.logger.Info("No CBContainerAgent installed, there is nothing to configure") return nil } + apiGateway, err := configurator.createAPIGateway(ctx, cr) + if err != nil { + return err // TODO: ! + } + configurator.logger.Info("Checking for pending remote configuration changes...") - change, errGettingChanges := configurator.getPendingChange(ctx) + change, errGettingChanges := configurator.getPendingChange(ctx, apiGateway) if errGettingChanges != nil { configurator.logger.Error(errGettingChanges, "Failed to get pending configuration changes") return errGettingChanges @@ -74,23 +109,29 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { } configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) - errApplyingCR := configurator.applyChange(ctx, *change, cr) + validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) + if err != nil { + return err // TODO + } + + errApplyingCR := configurator.syncer.ApplyChangeToCR(ctx, *change, cr, validator) if errApplyingCR != nil { - configurator.logger.Error(errApplyingCR, "Failed to apply configuration change", "changeID", change.ID) - // Intentional fallthrough as we always update the status of the change on the backend, including failed status + // TODO: Validation err? + + // Intentional fallthrough to ack errors as well } - if errStatusUpdate := configurator.updateChangeStatus(ctx, *change, cr, errApplyingCR); errStatusUpdate != nil { - configurator.logger.Error(errStatusUpdate, "Failed to update the status of a configuration change; it might be re-applied again in the future") - return errStatusUpdate + if err := configurator.updateChangeStatus(ctx, apiGateway, *change, cr, errApplyingCR); err != nil { + configurator.logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") + return err } // If we failed to apply the CR, we still report this to the backend but want to return the apply error here to propagate properly return errApplyingCR } -func (configurator *Configurator) getPendingChange(ctx context.Context) (*ConfigurationChange, error) { - changes, err := configurator.changesAPI.GetConfigurationChanges(ctx) +func (configurator *Configurator) getPendingChange(ctx context.Context, apiGateway ApiGateway) (*models.ConfigurationChange, error) { + changes, err := apiGateway.GetConfigurationChanges(ctx) if err != nil { return nil, err } @@ -107,22 +148,16 @@ func (configurator *Configurator) getPendingChange(ctx context.Context) (*Config return nil, nil } -// applyChange will sync the required changes and push them to the k8s api-server -// the input agent will be modified after this function and will no longer match the original -func (configurator *Configurator) applyChange(ctx context.Context, change ConfigurationChange, agent *cbcontainersv1.CBContainersAgent) error { - if err := configurator.changeValidator.ValidateChange(change, agent); err != nil { - return err - } - - ApplyChangeToCR(change, agent) - - return configurator.k8sClient.Update(ctx, agent) -} - -func (configurator *Configurator) updateChangeStatus(ctx context.Context, change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, encounteredError error) error { - var statusUpdate ConfigurationChangeStatusUpdate +func (configurator *Configurator) updateChangeStatus( + ctx context.Context, + apiGateway ApiGateway, + change models.ConfigurationChange, + cr *cbcontainersv1.CBContainersAgent, + encounteredError error, +) error { + var statusUpdate models.ConfigurationChangeStatusUpdate if encounteredError == nil { - statusUpdate = ConfigurationChangeStatusUpdate{ + statusUpdate = models.ConfigurationChangeStatusUpdate{ ID: change.ID, Status: string(statusAcknowledged), Reason: "", // TODO @@ -130,24 +165,40 @@ func (configurator *Configurator) updateChangeStatus(ctx context.Context, change AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), } } else { - statusUpdate = ConfigurationChangeStatusUpdate{ + statusUpdate = models.ConfigurationChangeStatusUpdate{ ID: change.ID, Status: string(statusFailed), Reason: encounteredError.Error(), // TODO } } - return configurator.changesAPI.UpdateConfigurationChangeStatus(ctx, statusUpdate) + return apiGateway.UpdateConfigurationChangeStatus(ctx, statusUpdate) } -// getContainerAgentCR loads exactly 0 or 1 CBContainersAgent definitions +func (configurator *Configurator) createAPIGateway(ctx context.Context, cr *cbcontainersv1.CBContainersAgent) (ApiGateway, error) { + accessToken, err := configurator.accessTokenProvider.GetCBAccessToken(ctx, cr, configurator.deployedNamespace) + if err != nil { + return nil, err + } + return configurator.apiCreator(cr, accessToken) +} + +type ChangeSyncerImpl struct { + k8sClient client.Client +} + +func NewChangeSyncerImpl(k8sClient client.Client) *ChangeSyncerImpl { + return &ChangeSyncerImpl{k8sClient: k8sClient} +} + +// GetCR loads exactly 0 or 1 CBContainersAgent definitions // if no resource is defined, nil is returned -// in case more than 1 resource is defined (which is not supported), only the first one is returned -func (configurator *Configurator) getContainerAgentCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { +// in case more than 1 resource is defined (which is not generally supported), only the first one is returned +func (s *ChangeSyncerImpl) GetCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} - if err := configurator.k8sClient.List(ctx, cbContainersAgentsList); err != nil { + if err := s.k8sClient.List(ctx, cbContainersAgentsList); err != nil { return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) } @@ -158,3 +209,53 @@ func (configurator *Configurator) getContainerAgentCR(ctx context.Context) (*cbc // We don't log a warning if len >=2 as the controller already warns users about that return &cbContainersAgentsList.Items[0], nil } + +func (s *ChangeSyncerImpl) ApplyChangeToCR(ctx context.Context, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator ChangeValidator) error { + if err := validator.ValidateChange(change, cr); err != nil { + return err // TODO + } + + s.applyChangeImpl(change, cr) + + return s.k8sClient.Update(ctx, cr) +} + +// TODO receiver + +func (s *ChangeSyncerImpl) applyChangeImpl(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { + resetVersion := func(ptrToField *string) { + if ptrToField != nil && *ptrToField != "" { + *ptrToField = "" + } + } + + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion + + resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + if cr.Spec.Components.Cndr != nil { + resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) + } + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } +} + +// applyChange will sync the required changes and push them to the k8s api-server +// the input agent will be modified after this function and will no longer match the original diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 996f2f41..124aa049 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" k8sMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" "github.com/vmware/cbcontainers-operator/remote_configuration" mocksConfigurator "github.com/vmware/cbcontainers-operator/remote_configuration/mocks" @@ -16,6 +17,7 @@ import ( "time" ) +// TODO: Add back the .finish for older mockgens // TODO: What error data to show and what not? // TODO: Reads cluster, etc from CR correctly? @@ -25,22 +27,55 @@ import ( // TODO: error on compatiblity calls type configuratorMocks struct { - k8sClient *k8sMocks.MockClient - configChangesAPI *mocksConfigurator.MockConfigurationChangesAPI - changeValidator *mocksConfigurator.MockChangeValidator + k8sClient *k8sMocks.MockClient + apiGateway *mocksConfigurator.MockApiGateway + accessTokenProvider *mocksConfigurator.MockAccessTokenProvider + syncer *mocksConfigurator.MockCustomResourceSyncer + + stubAccessToken string + stubOperatorVersion string + stubNamespace string + + testCtx context.Context } +// setupConfigurator TODO func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { k8sClient := k8sMocks.NewMockClient(ctrl) - configChangesAPI := mocksConfigurator.NewMockConfigurationChangesAPI(ctrl) - changeValidator := mocksConfigurator.NewMockChangeValidator(ctrl) + apiGateway := mocksConfigurator.NewMockApiGateway(ctrl) + accessTokenProvider := mocksConfigurator.NewMockAccessTokenProvider(ctrl) + syncer := mocksConfigurator.NewMockCustomResourceSyncer(ctrl) + + var mockAPIProvider remote_configuration.ApiCreator = func( + cbContainersCluster *cbcontainersv1.CBContainersAgent, + accessToken string, + ) (remote_configuration.ApiGateway, error) { + return apiGateway, nil + } - configurator := remote_configuration.NewConfigurator(k8sClient, configChangesAPI, changeValidator, logr.Discard()) + namespace := "namespace-name" + accessToken := "access-token" + operatorVersion := "1.2.3" + accessTokenProvider.EXPECT().GetCBAccessToken(gomock.Any(), gomock.Any(), namespace).Return(accessToken, nil).AnyTimes() + + configurator := remote_configuration.NewConfigurator( + mockAPIProvider, + logr.Discard(), + accessTokenProvider, + syncer, + operatorVersion, + namespace, + ) mocksHolder := configuratorMocks{ - k8sClient: k8sClient, - configChangesAPI: configChangesAPI, - changeValidator: changeValidator, + k8sClient: k8sClient, + apiGateway: apiGateway, + accessTokenProvider: accessTokenProvider, + syncer: syncer, + stubAccessToken: accessToken, + stubOperatorVersion: operatorVersion, + stubNamespace: namespace, + testCtx: context.Background(), // TODO: Remove? } return configurator, mocksHolder @@ -53,13 +88,15 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) var initialGeneration, finalGeneration int64 = 1, 2 - setupCRInK8S(mocks.k8sClient, &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}}) - setupChangeValidatorToAcceptAll(mocks.changeValidator) + cr := &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}} + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(cr, nil) + + // TODO: Generation configChange := remote_configuration.RandomNonNilChange() - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, finalGeneration, update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) @@ -71,54 +108,66 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { - assert.Equal(t, *configChange.AgentVersion, agent.Spec.Version) - agent.ObjectMeta.Generation = finalGeneration - }) - - err := configurator.RunIteration(context.Background()) - assert.NoError(t, err) -} - -func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { - ctrl := gomock.NewController(t) - - configurator, mocks := setupConfigurator(ctrl) - - cr := setupCRInK8S(mocks.k8sClient, nil) - - configChange := remote_configuration.RandomNonNilChange() - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - - mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(errors.New("your data is wrong pal")) - - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { - assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) - assert.NotEmpty(t, update.Reason) - assert.Equal(t, int64(0), update.AppliedGeneration) - assert.Empty(t, update.AppliedTimestamp) + mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) + mocks.syncer. + EXPECT(). + ApplyChangeToCR(gomock.Any(), configChange, cr, gomock.Any()). + DoAndReturn(func(_ context.Context, _ models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, _ any) error { + // We simulate the generation bump that we expect k8s to do + cr.Generation = finalGeneration return nil }) err := configurator.RunIteration(context.Background()) - assert.Error(t, err) + assert.NoError(t, err) } +// TODO: reintroduce + +//func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { +// ctrl := gomock.NewController(t) +// +// configurator, mocks := setupConfigurator(ctrl) +// +// cr := setupCRInK8S(mocks.k8sClient, nil) +// if cr != nil { +// // Placeholder +// } +// +// configChange := remote_configuration.RandomNonNilChange() +// mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) +// +// //mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(errors.New("your data is wrong pal")) +// +// mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). +// DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { +// assert.Equal(t, configChange.ID, update.ID) +// assert.Equal(t, "FAILED", update.Status) +// assert.NotEmpty(t, update.Reason) +// assert.Equal(t, int64(0), update.AppliedGeneration) +// assert.Empty(t, update.AppliedTimestamp) +// +// return nil +// }) +// +// err := configurator.RunIteration(context.Background()) +// assert.Error(t, err) +//} + func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { testCases := []struct { name string - dataFromService []remote_configuration.ConfigurationChange + dataFromService []models.ConfigurationChange }{ { name: "empty list", - dataFromService: []remote_configuration.ConfigurationChange{}, + dataFromService: []models.ConfigurationChange{}, }, { name: "list is not empty but there are no PENDING changes", - dataFromService: []remote_configuration.ConfigurationChange{ + dataFromService: []models.ConfigurationChange{ {ID: "123", Status: "non-existent"}, {ID: "234", Status: "FAILED"}, {ID: "345", Status: "ACKNOWLEDGED"}, @@ -134,9 +183,9 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - setupCRInK8S(mocks.k8sClient, nil) - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) err := configurator.RunIteration(context.Background()) assert.NoError(t, err) @@ -160,17 +209,15 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) - setupCRInK8S(mocks.k8sClient, nil) - setupChangeValidatorToAcceptAll(mocks.changeValidator) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{newerChange, olderChange}, nil) + mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{newerChange, olderChange}, nil) + mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), olderChange, gomock.Any(), gomock.Any()).Return(nil).Times(1) - assertUpdateCR(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { - assert.Equal(t, expectedVersion, agent.Spec.Version) - }) - - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, olderChange.ID, update.ID) return nil }) @@ -185,10 +232,10 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) configurator, mocks := setupConfigurator(ctrl) - setupCRInK8S(mocks.k8sClient, nil) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) errFromService := errors.New("some error") - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) returnedErr := configurator.RunIteration(context.Background()) @@ -202,12 +249,12 @@ func TestWhenGettingCRFromAPIServerFailsAnErrorIsReturned(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - errFromService := errors.New("some error") - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) + errFromK8S := errors.New("some error") + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(nil, errFromK8S) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) - assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") + assert.ErrorIs(t, returnedErr, errFromK8S, "expected returned error to match or wrap error from service") } func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { @@ -216,17 +263,18 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - setupCRInK8S(mocks.k8sClient, nil) - setupChangeValidatorToAcceptAll(mocks.changeValidator) - configChange := remote_configuration.RandomNonNilChange() - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) + + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) errFromService := errors.New("some error") - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) + mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), configChange, gomock.Any(), gomock.Any()).Return(errFromService) - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, update remote_configuration.ConfigurationChangeStatusUpdate) error { + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, "FAILED", update.Status) assert.NotEmpty(t, update.Reason) @@ -243,20 +291,20 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) - defer ctrl.Finish() // tODO: Remove all; redundant since 1.14 + defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) - setupCRInK8S(mocks.k8sClient, nil) - setupChangeValidatorToAcceptAll(mocks.changeValidator) - configChange := remote_configuration.RandomNonNilChange() - mocks.configChangesAPI.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]remote_configuration.ConfigurationChange{configChange}, nil) - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) + mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), configChange, gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") - mocks.configChangesAPI.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) @@ -268,10 +316,7 @@ func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { - list.Items = []cbcontainersv1.CBContainersAgent{} - }) + mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(nil, nil) assert.NoError(t, configurator.RunIteration(context.Background())) } @@ -304,7 +349,3 @@ func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcont return nil }) } - -func setupChangeValidatorToAcceptAll(mock *mocksConfigurator.MockChangeValidator) { - mock.EXPECT().ValidateChange(gomock.Any(), gomock.Any()).Return(nil) -} diff --git a/remote_configuration/custom_resource_changer.go b/remote_configuration/custom_resource_changer.go index 5a2a9800..bcc16467 100644 --- a/remote_configuration/custom_resource_changer.go +++ b/remote_configuration/custom_resource_changer.go @@ -6,7 +6,7 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/models" ) -func ApplyChangeToCR(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { +func ApplyChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { resetVersion := func(ptrToField *string) { if ptrToField != nil && *ptrToField != "" { *ptrToField = "" @@ -51,33 +51,22 @@ func (i invalidChangeError) Error() string { return i.msg } -type SensorMetadataAPI interface { - GetSensorsMetadata() ([]models.SensorMetadata, error) - GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) -} - -type ConfigurationChangeFetcher struct { - operatorVersion string - api SensorMetadataAPI -} - -func (fetcher *ConfigurationChangeFetcher) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { - compatibilityMatrix, err := fetcher.api.GetCompatibilityMatrixEntryFor(fetcher.operatorVersion) +func NewConfigurationChangeValidator(operatorVersion string, api ApiGateway) (*ConfigurationChangeValidator, error) { + compatibilityMatrix, err := api.GetCompatibilityMatrixEntryFor(operatorVersion) if err != nil { - return err + return nil, err } - sensors, err := fetcher.api.GetSensorsMetadata() + sensors, err := api.GetSensorMetadata() if err != nil { - return err + return nil, err } - validator := ConfigurationChangeValidator{ + // TODO: Dereference + return &ConfigurationChangeValidator{ SensorData: sensors, OperatorCompatibilityData: *compatibilityMatrix, - } - - return validator.ValidateChange(change, cr) + }, nil } type ConfigurationChangeValidator struct { @@ -85,7 +74,7 @@ type ConfigurationChangeValidator struct { OperatorCompatibilityData models.OperatorCompatibility } -func (validator *ConfigurationChangeValidator) ValidateChange(change ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { +func (validator *ConfigurationChangeValidator) ValidateChange(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { var versionToValidate string // If the change will be modifying the agent version as well, we need to check what the _new_ version supports @@ -120,7 +109,7 @@ func (validator *ConfigurationChangeValidator) validateOperatorAndSensorVersionC return nil } -func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change ConfigurationChange) error { +func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change models.ConfigurationChange) error { sensor := validator.findMatchingSensor(targetVersion) if sensor == nil { return fmt.Errorf("could not find sensor metadata for version %s", targetVersion) diff --git a/remote_configuration/custom_resource_changer_test.go b/remote_configuration/custom_resource_changer_test.go index 8601cc4c..5bf63db3 100644 --- a/remote_configuration/custom_resource_changer_test.go +++ b/remote_configuration/custom_resource_changer_test.go @@ -21,12 +21,12 @@ var ( func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { testCases := []struct { name string - change remote_configuration.ConfigurationChange + change models.ConfigurationChange sensorMeta models.SensorMetadata }{ { name: "cluster scanning", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableClusterScanning: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -35,7 +35,7 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { }, { name: "runtime protection", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableRuntime: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -44,7 +44,7 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { }, { name: "CNDR", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableCNDR: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -83,12 +83,12 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { testCases := []struct { name string - change remote_configuration.ConfigurationChange + change models.ConfigurationChange sensorMeta models.SensorMetadata }{ { name: "cluster scanning", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableClusterScanning: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -97,7 +97,7 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { }, { name: "runtime protection", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableRuntime: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -106,7 +106,7 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { }, { name: "CNDR", - change: remote_configuration.ConfigurationChange{ + change: models.ConfigurationChange{ EnableCNDR: truePtr, }, sensorMeta: models.SensorMetadata{ @@ -173,7 +173,7 @@ func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { OperatorCompatibilityData: tC.operatorCompatibility, } - change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} err := target.ValidateChange(change, cr) @@ -229,7 +229,7 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { OperatorCompatibilityData: tC.operatorCompatibility, } - change := remote_configuration.ConfigurationChange{AgentVersion: &tC.versionToApply} + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} cr := &cbcontainersv1.CBContainersAgent{} err := target.ValidateChange(change, cr) @@ -241,7 +241,7 @@ func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { type appliedChangeTest struct { name string - change remote_configuration.ConfigurationChange + change models.ConfigurationChange initialCR cbcontainersv1.CBContainersAgent assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) } @@ -252,7 +252,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) generateFeatureToggleTestCases := func(feature string, - changeFieldSelector func(*remote_configuration.ConfigurationChange) **bool, + changeFieldSelector func(*models.ConfigurationChange) **bool, crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { var result []appliedChangeTest @@ -264,7 +264,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { // Validate that each toggle state works (or doesn't do anything when it matches) for _, changeState := range []*bool{truePtr, falsePtr} { - change := remote_configuration.ConfigurationChange{} + change := models.ConfigurationChange{} changeFieldPtr := changeFieldSelector(&change) *changeFieldPtr = changeState @@ -283,7 +283,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { // Validate that a change with the toggle unset does not modify the CR result = append(result, appliedChangeTest{ name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), - change: remote_configuration.ConfigurationChange{}, + change: models.ConfigurationChange{}, initialCR: cr, assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { crFieldPostChangePtr := crFieldSelector(agent) @@ -298,21 +298,21 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { var testCases []appliedChangeTest clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", - func(change *remote_configuration.ConfigurationChange) **bool { + func(change *models.ConfigurationChange) **bool { return &change.EnableClusterScanning }, func(agent *cbcontainersv1.CBContainersAgent) **bool { return &agent.Spec.Components.ClusterScanning.Enabled }) runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", - func(change *remote_configuration.ConfigurationChange) **bool { + func(change *models.ConfigurationChange) **bool { return &change.EnableRuntime }, func(agent *cbcontainersv1.CBContainersAgent) **bool { return &agent.Spec.Components.RuntimeProtection.Enabled }) cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", - func(change *remote_configuration.ConfigurationChange) **bool { + func(change *models.ConfigurationChange) **bool { return &change.EnableCNDR }, func(agent *cbcontainersv1.CBContainersAgent) **bool { if agent.Spec.Components.Cndr == nil { @@ -337,7 +337,7 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { originalVersion := "my-version-42" newVersion := "new-version" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} - change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} + change := models.ConfigurationChange{AgentVersion: &newVersion} remote_configuration.ApplyChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) @@ -346,7 +346,7 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { func TestMissingVersionDoesNotModifyCR(t *testing.T) { originalVersion := "my-version-42" cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} - change := remote_configuration.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} + change := models.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} remote_configuration.ApplyChangeToCR(change, &cr) assert.Equal(t, originalVersion, cr.Spec.Version) @@ -411,7 +411,7 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { } newVersion := "new-version" - change := remote_configuration.ConfigurationChange{AgentVersion: &newVersion} + change := models.ConfigurationChange{AgentVersion: &newVersion} remote_configuration.ApplyChangeToCR(change, &cr) diff --git a/remote_configuration/mocks/generated.go b/remote_configuration/mocks/generated.go index b1874522..1766d559 100644 --- a/remote_configuration/mocks/generated.go +++ b/remote_configuration/mocks/generated.go @@ -1,4 +1,5 @@ package mocks -//go:generate mockgen -destination mock_configuration_api.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ConfigurationChangesAPI -//go:generate mockgen -destination mock_change_validator.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ChangeValidator +//go:generate mockgen -destination mock_api_gateway.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ApiGateway +//go:generate mockgen -destination mock_access_token_provider.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration AccessTokenProvider +//go:generate mockgen -destination mock_resource_syncer.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration CustomResourceSyncer diff --git a/remote_configuration/mocks/mock_access_token_provider.go b/remote_configuration/mocks/mock_access_token_provider.go new file mode 100644 index 00000000..2c3af28a --- /dev/null +++ b/remote_configuration/mocks/mock_access_token_provider.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: AccessTokenProvider) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" +) + +// MockAccessTokenProvider is a mock of AccessTokenProvider interface. +type MockAccessTokenProvider struct { + ctrl *gomock.Controller + recorder *MockAccessTokenProviderMockRecorder +} + +// MockAccessTokenProviderMockRecorder is the mock recorder for MockAccessTokenProvider. +type MockAccessTokenProviderMockRecorder struct { + mock *MockAccessTokenProvider +} + +// NewMockAccessTokenProvider creates a new mock instance. +func NewMockAccessTokenProvider(ctrl *gomock.Controller) *MockAccessTokenProvider { + mock := &MockAccessTokenProvider{ctrl: ctrl} + mock.recorder = &MockAccessTokenProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessTokenProvider) EXPECT() *MockAccessTokenProviderMockRecorder { + return m.recorder +} + +// GetCBAccessToken mocks base method. +func (m *MockAccessTokenProvider) GetCBAccessToken(arg0 context.Context, arg1 *v1.CBContainersAgent, arg2 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCBAccessToken", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCBAccessToken indicates an expected call of GetCBAccessToken. +func (mr *MockAccessTokenProviderMockRecorder) GetCBAccessToken(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCBAccessToken", reflect.TypeOf((*MockAccessTokenProvider)(nil).GetCBAccessToken), arg0, arg1, arg2) +} diff --git a/remote_configuration/mocks/mock_api_gateway.go b/remote_configuration/mocks/mock_api_gateway.go new file mode 100644 index 00000000..8e4458d2 --- /dev/null +++ b/remote_configuration/mocks/mock_api_gateway.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ApiGateway) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + models "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// MockApiGateway is a mock of ApiGateway interface. +type MockApiGateway struct { + ctrl *gomock.Controller + recorder *MockApiGatewayMockRecorder +} + +// MockApiGatewayMockRecorder is the mock recorder for MockApiGateway. +type MockApiGatewayMockRecorder struct { + mock *MockApiGateway +} + +// NewMockApiGateway creates a new mock instance. +func NewMockApiGateway(ctrl *gomock.Controller) *MockApiGateway { + mock := &MockApiGateway{ctrl: ctrl} + mock.recorder = &MockApiGatewayMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockApiGateway) EXPECT() *MockApiGatewayMockRecorder { + return m.recorder +} + +// GetCompatibilityMatrixEntryFor mocks base method. +func (m *MockApiGateway) GetCompatibilityMatrixEntryFor(arg0 string) (*models.OperatorCompatibility, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCompatibilityMatrixEntryFor", arg0) + ret0, _ := ret[0].(*models.OperatorCompatibility) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCompatibilityMatrixEntryFor indicates an expected call of GetCompatibilityMatrixEntryFor. +func (mr *MockApiGatewayMockRecorder) GetCompatibilityMatrixEntryFor(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompatibilityMatrixEntryFor", reflect.TypeOf((*MockApiGateway)(nil).GetCompatibilityMatrixEntryFor), arg0) +} + +// GetConfigurationChanges mocks base method. +func (m *MockApiGateway) GetConfigurationChanges(arg0 context.Context) ([]models.ConfigurationChange, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) + ret0, _ := ret[0].([]models.ConfigurationChange) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfigurationChanges indicates an expected call of GetConfigurationChanges. +func (mr *MockApiGatewayMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockApiGateway)(nil).GetConfigurationChanges), arg0) +} + +// GetSensorMetadata mocks base method. +func (m *MockApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSensorMetadata") + ret0, _ := ret[0].([]models.SensorMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSensorMetadata indicates an expected call of GetSensorMetadata. +func (mr *MockApiGatewayMockRecorder) GetSensorMetadata() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSensorMetadata", reflect.TypeOf((*MockApiGateway)(nil).GetSensorMetadata)) +} + +// UpdateConfigurationChangeStatus mocks base method. +func (m *MockApiGateway) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 models.ConfigurationChangeStatusUpdate) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. +func (mr *MockApiGatewayMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockApiGateway)(nil).UpdateConfigurationChangeStatus), arg0, arg1) +} diff --git a/remote_configuration/mocks/mock_change_validator.go b/remote_configuration/mocks/mock_change_validator.go deleted file mode 100644 index 8ab92f57..00000000 --- a/remote_configuration/mocks/mock_change_validator.go +++ /dev/null @@ -1,50 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ChangeValidator) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" -) - -// MockChangeValidator is a mock of ChangeValidator interface. -type MockChangeValidator struct { - ctrl *gomock.Controller - recorder *MockChangeValidatorMockRecorder -} - -// MockChangeValidatorMockRecorder is the mock recorder for MockChangeValidator. -type MockChangeValidatorMockRecorder struct { - mock *MockChangeValidator -} - -// NewMockChangeValidator creates a new mock instance. -func NewMockChangeValidator(ctrl *gomock.Controller) *MockChangeValidator { - mock := &MockChangeValidator{ctrl: ctrl} - mock.recorder = &MockChangeValidatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockChangeValidator) EXPECT() *MockChangeValidatorMockRecorder { - return m.recorder -} - -// ValidateChange mocks base method. -func (m *MockChangeValidator) ValidateChange(arg0 remote_configuration.ConfigurationChange, arg1 *v1.CBContainersAgent) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateChange", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateChange indicates an expected call of ValidateChange. -func (mr *MockChangeValidatorMockRecorder) ValidateChange(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateChange", reflect.TypeOf((*MockChangeValidator)(nil).ValidateChange), arg0, arg1) -} diff --git a/remote_configuration/mocks/mock_configuration_api.go b/remote_configuration/mocks/mock_configuration_api.go deleted file mode 100644 index 1edc7b02..00000000 --- a/remote_configuration/mocks/mock_configuration_api.go +++ /dev/null @@ -1,65 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ConfigurationChangesAPI) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" -) - -// MockConfigurationChangesAPI is a mock of ConfigurationChangesAPI interface. -type MockConfigurationChangesAPI struct { - ctrl *gomock.Controller - recorder *MockConfigurationChangesAPIMockRecorder -} - -// MockConfigurationChangesAPIMockRecorder is the mock recorder for MockConfigurationChangesAPI. -type MockConfigurationChangesAPIMockRecorder struct { - mock *MockConfigurationChangesAPI -} - -// NewMockConfigurationChangesAPI creates a new mock instance. -func NewMockConfigurationChangesAPI(ctrl *gomock.Controller) *MockConfigurationChangesAPI { - mock := &MockConfigurationChangesAPI{ctrl: ctrl} - mock.recorder = &MockConfigurationChangesAPIMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConfigurationChangesAPI) EXPECT() *MockConfigurationChangesAPIMockRecorder { - return m.recorder -} - -// GetConfigurationChanges mocks base method. -func (m *MockConfigurationChangesAPI) GetConfigurationChanges(arg0 context.Context) ([]remote_configuration.ConfigurationChange, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) - ret0, _ := ret[0].([]remote_configuration.ConfigurationChange) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetConfigurationChanges indicates an expected call of GetConfigurationChanges. -func (mr *MockConfigurationChangesAPIMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockConfigurationChangesAPI)(nil).GetConfigurationChanges), arg0) -} - -// UpdateConfigurationChangeStatus mocks base method. -func (m *MockConfigurationChangesAPI) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 remote_configuration.ConfigurationChangeStatusUpdate) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. -func (mr *MockConfigurationChangesAPIMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockConfigurationChangesAPI)(nil).UpdateConfigurationChangeStatus), arg0, arg1) -} diff --git a/remote_configuration/mocks/mock_resource_syncer.go b/remote_configuration/mocks/mock_resource_syncer.go new file mode 100644 index 00000000..7e935dd7 --- /dev/null +++ b/remote_configuration/mocks/mock_resource_syncer.go @@ -0,0 +1,67 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: CustomResourceSyncer) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + models "github.com/vmware/cbcontainers-operator/cbcontainers/models" + remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" +) + +// MockCustomResourceSyncer is a mock of CustomResourceSyncer interface. +type MockCustomResourceSyncer struct { + ctrl *gomock.Controller + recorder *MockCustomResourceSyncerMockRecorder +} + +// MockCustomResourceSyncerMockRecorder is the mock recorder for MockCustomResourceSyncer. +type MockCustomResourceSyncerMockRecorder struct { + mock *MockCustomResourceSyncer +} + +// NewMockCustomResourceSyncer creates a new mock instance. +func NewMockCustomResourceSyncer(ctrl *gomock.Controller) *MockCustomResourceSyncer { + mock := &MockCustomResourceSyncer{ctrl: ctrl} + mock.recorder = &MockCustomResourceSyncerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCustomResourceSyncer) EXPECT() *MockCustomResourceSyncerMockRecorder { + return m.recorder +} + +// ApplyChangeToCR mocks base method. +func (m *MockCustomResourceSyncer) ApplyChangeToCR(arg0 context.Context, arg1 models.ConfigurationChange, arg2 *v1.CBContainersAgent, arg3 remote_configuration.ChangeValidator) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyChangeToCR", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// ApplyChangeToCR indicates an expected call of ApplyChangeToCR. +func (mr *MockCustomResourceSyncerMockRecorder) ApplyChangeToCR(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyChangeToCR", reflect.TypeOf((*MockCustomResourceSyncer)(nil).ApplyChangeToCR), arg0, arg1, arg2, arg3) +} + +// GetCR mocks base method. +func (m *MockCustomResourceSyncer) GetCR(arg0 context.Context) (*v1.CBContainersAgent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCR", arg0) + ret0, _ := ret[0].(*v1.CBContainersAgent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCR indicates an expected call of GetCR. +func (mr *MockCustomResourceSyncerMockRecorder) GetCR(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCR", reflect.TypeOf((*MockCustomResourceSyncer)(nil).GetCR), arg0) +} diff --git a/remote_configuration/temp.go b/remote_configuration/temp.go index d694aa69..681d7050 100644 --- a/remote_configuration/temp.go +++ b/remote_configuration/temp.go @@ -2,6 +2,7 @@ package remote_configuration import ( "context" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" "math/rand" "strconv" ) @@ -16,20 +17,20 @@ var ( type DummyAPI struct { } -func (d DummyAPI) GetConfigurationChanges(ctx context.Context) ([]ConfigurationChange, error) { +func (d DummyAPI) GetConfigurationChanges(ctx context.Context) ([]models.ConfigurationChange, error) { c := RandomChange() if c != nil { - return []ConfigurationChange{*c}, nil + return []models.ConfigurationChange{*c}, nil } return nil, nil } -func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update ConfigurationChangeStatusUpdate) error { +func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update models.ConfigurationChangeStatusUpdate) error { return nil } -func RandomNonNilChange() ConfigurationChange { +func RandomNonNilChange() models.ConfigurationChange { for { c := RandomChange() if c != nil { @@ -38,7 +39,7 @@ func RandomNonNilChange() ConfigurationChange { } } -func RandomChange() *ConfigurationChange { +func RandomChange() *models.ConfigurationChange { csRand, runtimeRand, cndrRand, versionRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)+1) //csRand, runtimeRand, versionRand = 1, 2, 3 @@ -76,7 +77,7 @@ func RandomChange() *ConfigurationChange { changeCNDR = &fal } - return &ConfigurationChange{ + return &models.ConfigurationChange{ ID: strconv.Itoa(rand.Int()), AgentVersion: changeVersion, EnableClusterScanning: changeClusterScanning, @@ -86,28 +87,6 @@ func RandomChange() *ConfigurationChange { } } -type ConfigurationChange struct { - ID string `json:"id"` - Status string `json:"status"` - AgentVersion *string `json:"agent_version"` - EnableClusterScanning *bool `json:"enable_cluster_scanning"` - EnableRuntime *bool `json:"enable_runtime"` - EnableCNDR *bool `json:"enable_cndr"` - Timestamp string `json:"timestamp"` -} - -type ConfigurationChangeStatusUpdate struct { - ID string `json:"id"` - Status string `json:"status"` - Reason string `json:"reason"` - // AppliedGeneration tracks the generation of the Custom resource where the change was applied - AppliedGeneration int64 `json:"applied_generation"` - // AppliedTimestamp records when the change was applied in RFC3339 format - AppliedTimestamp string `json:"applied_timestamp"` - - // TODO: CLuster and group. Cluster identifier? -} - type changeStatus string var ( From 24763858dc7995bade1d19d03fae1dec44f803eb Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 12:36:57 +0300 Subject: [PATCH 37/65] Go back to the previous version as it looked simpler --- remote_configuration/change_applier.go | 43 ++++ ...changer_test.go => change_applier_test.go} | 243 +----------------- remote_configuration/configurator.go | 131 +++------- remote_configuration/configurator_test.go | 151 ++++++----- remote_configuration/mocks/generated.go | 1 - .../mocks/mock_resource_syncer.go | 67 ----- ...stom_resource_changer.go => validation.go} | 35 --- remote_configuration/validation_test.go | 239 +++++++++++++++++ 8 files changed, 411 insertions(+), 499 deletions(-) create mode 100644 remote_configuration/change_applier.go rename remote_configuration/{custom_resource_changer_test.go => change_applier_test.go} (54%) delete mode 100644 remote_configuration/mocks/mock_resource_syncer.go rename remote_configuration/{custom_resource_changer.go => validation.go} (70%) create mode 100644 remote_configuration/validation_test.go diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go new file mode 100644 index 00000000..d4c5ed62 --- /dev/null +++ b/remote_configuration/change_applier.go @@ -0,0 +1,43 @@ +package remote_configuration + +import ( + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +type ChangeApplier struct{} + +func (applier ChangeApplier) ApplyConfigChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { + resetVersion := func(ptrToField *string) { + if ptrToField != nil && *ptrToField != "" { + *ptrToField = "" + } + } + + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion + + resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) + resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + if cr.Spec.Components.Cndr != nil { + resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) + } + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } +} diff --git a/remote_configuration/custom_resource_changer_test.go b/remote_configuration/change_applier_test.go similarity index 54% rename from remote_configuration/custom_resource_changer_test.go rename to remote_configuration/change_applier_test.go index 5bf63db3..37674d36 100644 --- a/remote_configuration/custom_resource_changer_test.go +++ b/remote_configuration/change_applier_test.go @@ -9,235 +9,6 @@ import ( "testing" ) -// TODO: add secret detection - -var ( - trueV = true - truePtr = &trueV - falseV = false - falsePtr = &falseV -) - -func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { - testCases := []struct { - name string - change models.ConfigurationChange - sensorMeta models.SensorMetadata - }{ - { - name: "cluster scanning", - change: models.ConfigurationChange{ - EnableClusterScanning: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsClusterScanning: false, - }, - }, - { - name: "runtime protection", - change: models.ConfigurationChange{ - EnableRuntime: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsRuntime: false, - }, - }, - { - name: "CNDR", - change: models.ConfigurationChange{ - EnableCNDR: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsCndr: false, - }, - }, - } - - for _, tC := range testCases { - version := "dummy-version" - tC.sensorMeta.Version = version - target := remote_configuration.ConfigurationChangeValidator{ - SensorData: []models.SensorMetadata{tC.sensorMeta}, - } - - t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { - tC.change.AgentVersion = nil - cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} - - err := target.ValidateChange(tC.change, cr) - - assert.Error(t, err) - }) - - t.Run(fmt.Sprintf("change also applies agent version, %s not supported by that version", tC.name), func(t *testing.T) { - tC.change.AgentVersion = &version - cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} - - err := target.ValidateChange(tC.change, cr) - - assert.Error(t, err) - }) - } -} - -func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { - testCases := []struct { - name string - change models.ConfigurationChange - sensorMeta models.SensorMetadata - }{ - { - name: "cluster scanning", - change: models.ConfigurationChange{ - EnableClusterScanning: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsClusterScanning: true, - }, - }, - { - name: "runtime protection", - change: models.ConfigurationChange{ - EnableRuntime: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsRuntime: true, - }, - }, - { - name: "CNDR", - change: models.ConfigurationChange{ - EnableCNDR: truePtr, - }, - sensorMeta: models.SensorMetadata{ - SupportsCndr: true, - }, - }, - } - - for _, tC := range testCases { - version := "dummy-version" - tC.sensorMeta.Version = version - target := remote_configuration.ConfigurationChangeValidator{ - SensorData: []models.SensorMetadata{tC.sensorMeta}, - } - - t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { - tC.change.AgentVersion = nil - cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} - - err := target.ValidateChange(tC.change, cr) - - assert.NoError(t, err) - }) - - t.Run(fmt.Sprintf("change also applies agent version, %s is supported by that version", tC.name), func(t *testing.T) { - tC.change.AgentVersion = &version - cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} - - err := target.ValidateChange(tC.change, cr) - - assert.NoError(t, err) - }) - } -} - -func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { - testCases := []struct { - name string - versionToApply string - operatorCompatibility models.OperatorCompatibility - }{ - { - name: "sensor version is too high", - versionToApply: "5.0.0", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: models.AgentMinVersionNone, - MaxAgent: "4.0.0", - }, - }, - { - name: "sensor version is too low", - versionToApply: "0.9", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: "1.0.0", - MaxAgent: models.AgentMaxVersionLatest, - }, - }, - } - - for _, tC := range testCases { - t.Run(tC.name, func(t *testing.T) { - target := remote_configuration.ConfigurationChangeValidator{ - SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, - OperatorCompatibilityData: tC.operatorCompatibility, - } - - change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} - cr := &cbcontainersv1.CBContainersAgent{} - - err := target.ValidateChange(change, cr) - assert.Error(t, err) - }) - } -} - -func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { - testCases := []struct { - name string - versionToApply string - operatorCompatibility models.OperatorCompatibility - }{ - { - name: "sensor version is at lower end", - versionToApply: "5.0.0", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: "5.0.0", - MaxAgent: "6.0.0", - }, - }, - { - name: "sensor version is at upper end", - versionToApply: "0.9", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: "0.1.0", - MaxAgent: "0.9.0", - }, - }, - { - name: "sensor version is within range", - versionToApply: "2.3.4", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: "1.0.0", - MaxAgent: "2.4", - }, - }, - { - name: "operator supports 'infinite' versions", - versionToApply: "5.0.0", - operatorCompatibility: models.OperatorCompatibility{ - MinAgent: models.AgentMinVersionNone, - MaxAgent: models.AgentMaxVersionLatest, - }, - }, - } - - for _, tC := range testCases { - t.Run(tC.name, func(t *testing.T) { - target := remote_configuration.ConfigurationChangeValidator{ - SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, - OperatorCompatibilityData: tC.operatorCompatibility, - } - - change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} - cr := &cbcontainersv1.CBContainersAgent{} - - err := target.ValidateChange(change, cr) - assert.NoError(t, err) - }) - } -} - func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { type appliedChangeTest struct { name string @@ -327,7 +98,9 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - remote_configuration.ApplyChangeToCR(testCase.change, &testCase.initialCR) + target := remote_configuration.ChangeApplier{} + + target.ApplyConfigChangeToCR(testCase.change, &testCase.initialCR) testCase.assertFinalCR(t, &testCase.initialCR) }) } @@ -339,7 +112,9 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := models.ConfigurationChange{AgentVersion: &newVersion} - remote_configuration.ApplyChangeToCR(change, &cr) + target := remote_configuration.ChangeApplier{} + + target.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) } @@ -348,7 +123,8 @@ func TestMissingVersionDoesNotModifyCR(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := models.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} - remote_configuration.ApplyChangeToCR(change, &cr) + target := remote_configuration.ChangeApplier{} + target.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, originalVersion, cr.Spec.Version) } @@ -413,8 +189,9 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { newVersion := "new-version" change := models.ConfigurationChange{AgentVersion: &newVersion} - remote_configuration.ApplyChangeToCR(change, &cr) + target := remote_configuration.ChangeApplier{} + target.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields // => the operator will use the common version instead diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 5c25355c..a0383577 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -40,39 +40,30 @@ func CBGatewayCreator(cbContainersCluster *cbcontainersv1.CBContainersAgent, acc type ApiCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) -type ChangeValidator interface { - ValidateChange(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error -} - -type CustomResourceSyncer interface { - GetCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) - ApplyChangeToCR(ctx context.Context, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator ChangeValidator) error -} - type Configurator struct { logger logr.Logger accessTokenProvider AccessTokenProvider apiCreator ApiCreator operatorVersion string deployedNamespace string - syncer CustomResourceSyncer + k8sClient client.Client } func NewConfigurator( + k8sClient client.Client, gatewayCreator ApiCreator, logger logr.Logger, accessTokenProvider AccessTokenProvider, - syncer CustomResourceSyncer, operatorVersion string, deployedNamespace string, ) *Configurator { return &Configurator{ + k8sClient: k8sClient, logger: logger, apiCreator: gatewayCreator, accessTokenProvider: accessTokenProvider, operatorVersion: operatorVersion, deployedNamespace: deployedNamespace, - syncer: syncer, } } @@ -81,7 +72,7 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { defer cancel() configurator.logger.Info("Checking for installed agent...") - cr, err := configurator.syncer.GetCR(ctx) + cr, err := configurator.getCR(ctx) if err != nil { configurator.logger.Error(err, "Failed to get CBContainerAgent resource, cannot continue") return err @@ -108,18 +99,15 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return nil } + // TODO: This is ugly... configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) if err != nil { return err // TODO } - errApplyingCR := configurator.syncer.ApplyChangeToCR(ctx, *change, cr, validator) - if errApplyingCR != nil { - // TODO: Validation err? - - // Intentional fallthrough to ack errors as well - } + errApplyingCR := configurator.applyChangeToCR(ctx, apiGateway, *change, cr, validator) + // TODO: Explain if err := configurator.updateChangeStatus(ctx, apiGateway, *change, cr, errApplyingCR); err != nil { configurator.logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") @@ -130,6 +118,25 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return errApplyingCR } +// getCR loads exactly 0 or 1 CBContainersAgent definitions +// if no resource is defined, nil is returned +// in case more than 1 resource is defined (which is not generally supported), only the first one is returned +func (configurator *Configurator) getCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { + // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance + + cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} + if err := configurator.k8sClient.List(ctx, cbContainersAgentsList); err != nil { + return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) + } + + if len(cbContainersAgentsList.Items) == 0 { + return nil, nil + } + + // We don't log a warning if len >=2 as the controller already warns users about that + return &cbContainersAgentsList.Items[0], nil +} + func (configurator *Configurator) getPendingChange(ctx context.Context, apiGateway ApiGateway) (*models.ConfigurationChange, error) { changes, err := apiGateway.GetConfigurationChanges(ctx) if err != nil { @@ -148,6 +155,15 @@ func (configurator *Configurator) getPendingChange(ctx context.Context, apiGatew return nil, nil } +func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGateway ApiGateway, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator *ConfigurationChangeValidator) error { + if err := validator.ValidateChange(change, cr); err != nil { + return err + } + c := ChangeApplier{} + c.ApplyConfigChangeToCR(change, cr) + return configurator.k8sClient.Update(ctx, cr) +} + func (configurator *Configurator) updateChangeStatus( ctx context.Context, apiGateway ApiGateway, @@ -182,80 +198,3 @@ func (configurator *Configurator) createAPIGateway(ctx context.Context, cr *cbco } return configurator.apiCreator(cr, accessToken) } - -type ChangeSyncerImpl struct { - k8sClient client.Client -} - -func NewChangeSyncerImpl(k8sClient client.Client) *ChangeSyncerImpl { - return &ChangeSyncerImpl{k8sClient: k8sClient} -} - -// GetCR loads exactly 0 or 1 CBContainersAgent definitions -// if no resource is defined, nil is returned -// in case more than 1 resource is defined (which is not generally supported), only the first one is returned -func (s *ChangeSyncerImpl) GetCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { - // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance - - cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} - if err := s.k8sClient.List(ctx, cbContainersAgentsList); err != nil { - return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) - } - - if len(cbContainersAgentsList.Items) == 0 { - return nil, nil - } - - // We don't log a warning if len >=2 as the controller already warns users about that - return &cbContainersAgentsList.Items[0], nil -} - -func (s *ChangeSyncerImpl) ApplyChangeToCR(ctx context.Context, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator ChangeValidator) error { - if err := validator.ValidateChange(change, cr); err != nil { - return err // TODO - } - - s.applyChangeImpl(change, cr) - - return s.k8sClient.Update(ctx, cr) -} - -// TODO receiver - -func (s *ChangeSyncerImpl) applyChangeImpl(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { - resetVersion := func(ptrToField *string) { - if ptrToField != nil && *ptrToField != "" { - *ptrToField = "" - } - } - - if change.AgentVersion != nil { - cr.Spec.Version = *change.AgentVersion - - resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) - if cr.Spec.Components.Cndr != nil { - resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) - } - } - if change.EnableClusterScanning != nil { - cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning - } - if change.EnableRuntime != nil { - cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime - } - if change.EnableCNDR != nil { - if cr.Spec.Components.Cndr == nil { - cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} - } - cr.Spec.Components.Cndr.Enabled = change.EnableCNDR - } -} - -// applyChange will sync the required changes and push them to the k8s api-server -// the input agent will be modified after this function and will no longer match the original diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 124aa049..f2772437 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -30,13 +30,10 @@ type configuratorMocks struct { k8sClient *k8sMocks.MockClient apiGateway *mocksConfigurator.MockApiGateway accessTokenProvider *mocksConfigurator.MockAccessTokenProvider - syncer *mocksConfigurator.MockCustomResourceSyncer stubAccessToken string stubOperatorVersion string stubNamespace string - - testCtx context.Context } // setupConfigurator TODO @@ -44,7 +41,6 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura k8sClient := k8sMocks.NewMockClient(ctrl) apiGateway := mocksConfigurator.NewMockApiGateway(ctrl) accessTokenProvider := mocksConfigurator.NewMockAccessTokenProvider(ctrl) - syncer := mocksConfigurator.NewMockCustomResourceSyncer(ctrl) var mockAPIProvider remote_configuration.ApiCreator = func( cbContainersCluster *cbcontainersv1.CBContainersAgent, @@ -59,10 +55,10 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura accessTokenProvider.EXPECT().GetCBAccessToken(gomock.Any(), gomock.Any(), namespace).Return(accessToken, nil).AnyTimes() configurator := remote_configuration.NewConfigurator( + k8sClient, mockAPIProvider, logr.Discard(), accessTokenProvider, - syncer, operatorVersion, namespace, ) @@ -71,11 +67,9 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura k8sClient: k8sClient, apiGateway: apiGateway, accessTokenProvider: accessTokenProvider, - syncer: syncer, stubAccessToken: accessToken, stubOperatorVersion: operatorVersion, stubNamespace: namespace, - testCtx: context.Background(), // TODO: Remove? } return configurator, mocksHolder @@ -86,16 +80,19 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) - var initialGeneration, finalGeneration int64 = 1, 2 + // Setup stub data + var initialGeneration, finalGeneration int64 = 1, 2 + expectedAgentVersion := "3.0.0" cr := &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}} - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(cr, nil) - - // TODO: Generation - configChange := remote_configuration.RandomNonNilChange() + configChange.AgentVersion = &expectedAgentVersion + + setupCRInK8S(mocks.k8sClient, cr) + setupValidCompatibilityData(mocks.apiGateway, expectedAgentVersion, mocks.stubOperatorVersion) mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + // Setup mock assertions mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, finalGeneration, update.AppliedGeneration) @@ -108,17 +105,10 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { return nil }) - mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) - mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) - - mocks.syncer. - EXPECT(). - ApplyChangeToCR(gomock.Any(), configChange, cr, gomock.Any()). - DoAndReturn(func(_ context.Context, _ models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, _ any) error { - // We simulate the generation bump that we expect k8s to do - cr.Generation = finalGeneration - return nil - }) + setupUpdateCRMock(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, expectedAgentVersion, agent.Spec.Version) + agent.ObjectMeta.Generation = finalGeneration + }) err := configurator.RunIteration(context.Background()) assert.NoError(t, err) @@ -126,35 +116,43 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { // TODO: reintroduce -//func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { -// ctrl := gomock.NewController(t) -// -// configurator, mocks := setupConfigurator(ctrl) -// -// cr := setupCRInK8S(mocks.k8sClient, nil) -// if cr != nil { -// // Placeholder -// } -// -// configChange := remote_configuration.RandomNonNilChange() -// mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) -// -// //mocks.changeValidator.EXPECT().ValidateChange(configChange, cr).Return(errors.New("your data is wrong pal")) -// -// mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). -// DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { -// assert.Equal(t, configChange.ID, update.ID) -// assert.Equal(t, "FAILED", update.Status) -// assert.NotEmpty(t, update.Reason) -// assert.Equal(t, int64(0), update.AppliedGeneration) -// assert.Empty(t, update.AppliedTimestamp) -// -// return nil -// }) -// -// err := configurator.RunIteration(context.Background()) -// assert.Error(t, err) -//} +func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { + ctrl := gomock.NewController(t) + + configurator, mocks := setupConfigurator(ctrl) + + cr := &cbcontainersv1.CBContainersAgent{} + maxAgentVersionForOperator := "4.0.0" + agentVersion := "5.0.0" + configChange := remote_configuration.RandomNonNilChange() + configChange.AgentVersion = &agentVersion + + setupCRInK8S(mocks.k8sClient, cr) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + + // Setup invalid compatibility; no need to do full verification here - this is what the validator tests are for + // We just want to check that _some_ validation happens + mocks.apiGateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{{Version: agentVersion}}, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentVersion(maxAgentVersionForOperator), + }, nil) + + // Setup mock assertions + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, "FAILED", update.Status) + assert.NotEmpty(t, update.Reason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + + return nil + }) + + err := configurator.RunIteration(context.Background()) + assert.Error(t, err) +} func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { testCases := []struct { @@ -183,7 +181,7 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + setupCRInK8S(mocks.k8sClient, nil) mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) @@ -209,12 +207,13 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + setupCRInK8S(mocks.k8sClient, nil) mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{newerChange, olderChange}, nil) - mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) - mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) + setupValidCompatibilityData(mocks.apiGateway, expectedVersion, mocks.stubOperatorVersion) - mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), olderChange, gomock.Any(), gomock.Any()).Return(nil).Times(1) + setupUpdateCRMock(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, expectedVersion, agent.Spec.Version) + }) mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { @@ -232,7 +231,7 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) configurator, mocks := setupConfigurator(ctrl) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + setupCRInK8S(mocks.k8sClient, nil) errFromService := errors.New("some error") mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) @@ -249,12 +248,12 @@ func TestWhenGettingCRFromAPIServerFailsAnErrorIsReturned(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - errFromK8S := errors.New("some error") - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(nil, errFromK8S) + errFromService := errors.New("some error") + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) returnedErr := configurator.RunIteration(context.Background()) assert.Error(t, returnedErr) - assert.ErrorIs(t, returnedErr, errFromK8S, "expected returned error to match or wrap error from service") + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") } func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { @@ -266,12 +265,12 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configChange := remote_configuration.RandomNonNilChange() mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) + setupCRInK8S(mocks.k8sClient, nil) mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) errFromService := errors.New("some error") - mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), configChange, gomock.Any(), gomock.Any()).Return(errFromService) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { @@ -297,11 +296,11 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { configChange := remote_configuration.RandomNonNilChange() + setupCRInK8S(mocks.k8sClient, nil) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(&cbcontainersv1.CBContainersAgent{}, nil) mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) - mocks.syncer.EXPECT().ApplyChangeToCR(gomock.Any(), configChange, gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) @@ -316,7 +315,10 @@ func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - mocks.syncer.EXPECT().GetCR(gomock.Any()).Return(nil, nil) + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{} + }) assert.NoError(t, configurator.RunIteration(context.Background())) } @@ -339,7 +341,7 @@ func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAg return item } -func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcontainersv1.CBContainersAgent)) { +func setupUpdateCRMock(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcontainersv1.CBContainersAgent)) { mock.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, item any, _ ...any) error { asCb, ok := item.(*cbcontainersv1.CBContainersAgent) @@ -349,3 +351,18 @@ func assertUpdateCR(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcont return nil }) } + +func setupValidCompatibilityData(mockGateway *mocksConfigurator.MockApiGateway, sensorVersion, operatorVersion string) { + mockGateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{{ + Version: sensorVersion, + SupportsRuntime: true, + SupportsClusterScanning: true, + SupportsCndr: true, + }}, nil) + + mockGateway.EXPECT().GetCompatibilityMatrixEntryFor(operatorVersion).Return(&models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + }, nil) + +} diff --git a/remote_configuration/mocks/generated.go b/remote_configuration/mocks/generated.go index 1766d559..d0aef348 100644 --- a/remote_configuration/mocks/generated.go +++ b/remote_configuration/mocks/generated.go @@ -2,4 +2,3 @@ package mocks //go:generate mockgen -destination mock_api_gateway.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ApiGateway //go:generate mockgen -destination mock_access_token_provider.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration AccessTokenProvider -//go:generate mockgen -destination mock_resource_syncer.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration CustomResourceSyncer diff --git a/remote_configuration/mocks/mock_resource_syncer.go b/remote_configuration/mocks/mock_resource_syncer.go deleted file mode 100644 index 7e935dd7..00000000 --- a/remote_configuration/mocks/mock_resource_syncer.go +++ /dev/null @@ -1,67 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: CustomResourceSyncer) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - models "github.com/vmware/cbcontainers-operator/cbcontainers/models" - remote_configuration "github.com/vmware/cbcontainers-operator/remote_configuration" -) - -// MockCustomResourceSyncer is a mock of CustomResourceSyncer interface. -type MockCustomResourceSyncer struct { - ctrl *gomock.Controller - recorder *MockCustomResourceSyncerMockRecorder -} - -// MockCustomResourceSyncerMockRecorder is the mock recorder for MockCustomResourceSyncer. -type MockCustomResourceSyncerMockRecorder struct { - mock *MockCustomResourceSyncer -} - -// NewMockCustomResourceSyncer creates a new mock instance. -func NewMockCustomResourceSyncer(ctrl *gomock.Controller) *MockCustomResourceSyncer { - mock := &MockCustomResourceSyncer{ctrl: ctrl} - mock.recorder = &MockCustomResourceSyncerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockCustomResourceSyncer) EXPECT() *MockCustomResourceSyncerMockRecorder { - return m.recorder -} - -// ApplyChangeToCR mocks base method. -func (m *MockCustomResourceSyncer) ApplyChangeToCR(arg0 context.Context, arg1 models.ConfigurationChange, arg2 *v1.CBContainersAgent, arg3 remote_configuration.ChangeValidator) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyChangeToCR", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// ApplyChangeToCR indicates an expected call of ApplyChangeToCR. -func (mr *MockCustomResourceSyncerMockRecorder) ApplyChangeToCR(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyChangeToCR", reflect.TypeOf((*MockCustomResourceSyncer)(nil).ApplyChangeToCR), arg0, arg1, arg2, arg3) -} - -// GetCR mocks base method. -func (m *MockCustomResourceSyncer) GetCR(arg0 context.Context) (*v1.CBContainersAgent, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCR", arg0) - ret0, _ := ret[0].(*v1.CBContainersAgent) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetCR indicates an expected call of GetCR. -func (mr *MockCustomResourceSyncerMockRecorder) GetCR(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCR", reflect.TypeOf((*MockCustomResourceSyncer)(nil).GetCR), arg0) -} diff --git a/remote_configuration/custom_resource_changer.go b/remote_configuration/validation.go similarity index 70% rename from remote_configuration/custom_resource_changer.go rename to remote_configuration/validation.go index bcc16467..e3c6933a 100644 --- a/remote_configuration/custom_resource_changer.go +++ b/remote_configuration/validation.go @@ -6,41 +6,6 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/models" ) -func ApplyChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { - resetVersion := func(ptrToField *string) { - if ptrToField != nil && *ptrToField != "" { - *ptrToField = "" - } - } - - if change.AgentVersion != nil { - cr.Spec.Version = *change.AgentVersion - - resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) - if cr.Spec.Components.Cndr != nil { - resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) - } - } - if change.EnableClusterScanning != nil { - cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning - } - if change.EnableRuntime != nil { - cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime - } - if change.EnableCNDR != nil { - if cr.Spec.Components.Cndr == nil { - cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} - } - cr.Spec.Components.Cndr.Enabled = change.EnableCNDR - } -} - // TODO: Move type invalidChangeError struct { diff --git a/remote_configuration/validation_test.go b/remote_configuration/validation_test.go new file mode 100644 index 00000000..cfd2ddfa --- /dev/null +++ b/remote_configuration/validation_test.go @@ -0,0 +1,239 @@ +package remote_configuration_test + +import ( + "fmt" + "github.com/stretchr/testify/assert" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/remote_configuration" + "testing" +) + +// TODO: add secret detection + +var ( + trueV = true + truePtr = &trueV + falseV = false + falsePtr = &falseV +) + +func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change models.ConfigurationChange + sensorMeta models.SensorMetadata + }{ + { + name: "cluster scanning", + change: models.ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanning: false, + }, + }, + { + name: "runtime protection", + change: models.ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsRuntime: false, + }, + }, + { + name: "CNDR", + change: models.ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsCndr: false, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + err := target.ValidateChange(tC.change, cr) + + assert.Error(t, err) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s not supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + err := target.ValidateChange(tC.change, cr) + + assert.Error(t, err) + }) + } +} + +func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change models.ConfigurationChange + sensorMeta models.SensorMetadata + }{ + { + name: "cluster scanning", + change: models.ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanning: true, + }, + }, + { + name: "runtime protection", + change: models.ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsRuntime: true, + }, + }, + { + name: "CNDR", + change: models.ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsCndr: true, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + err := target.ValidateChange(tC.change, cr) + + assert.NoError(t, err) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s is supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + err := target.ValidateChange(tC.change, cr) + + assert.NoError(t, err) + }) + } +} + +func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility + }{ + { + name: "sensor version is too high", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: "4.0.0", + }, + }, + { + name: "sensor version is too low", + versionToApply: "0.9", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatibility, + } + + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} + + err := target.ValidateChange(change, cr) + assert.Error(t, err) + }) + } +} + +func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility + }{ + { + name: "sensor version is at lower end", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "5.0.0", + MaxAgent: "6.0.0", + }, + }, + { + name: "sensor version is at upper end", + versionToApply: "0.9", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "0.1.0", + MaxAgent: "0.9.0", + }, + }, + { + name: "sensor version is within range", + versionToApply: "2.3.4", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: "2.4", + }, + }, + { + name: "operator supports 'infinite' versions", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatibility, + } + + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} + + err := target.ValidateChange(change, cr) + assert.NoError(t, err) + }) + } +} From 2c582fbeec655152fbb2ba8034c7f20fc50180b4 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 12:45:51 +0300 Subject: [PATCH 38/65] Fixing some tests and main.go --- main.go | 12 ++++++------ remote_configuration/configurator_test.go | 8 +++----- remote_configuration/temp.go | 7 ++++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index eb997194..d88fcf09 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "errors" "flag" "fmt" "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" @@ -182,15 +183,14 @@ func main() { signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") - syncer := remote_configuration.NewChangeSyncerImpl(k8sClient) - versionReader := operator.NewEnvVersionProvider() + versionReader := operator.NewEnvVersionProvider() // TODO: reuse from above operatorVersion, err := versionReader.GetOperatorVersion() - if err != nil { - // TODO - panic(err) + if err != nil && !errors.Is(err, operator.ErrNotSemVer) { + setupLog.Error(err, "unable to read the running operator's version from environment variable") + os.Exit(1) } - applier := remote_configuration.NewConfigurator(remote_configuration.CBGatewayCreator, log, operator.NewSecretAccessTokenProvider(k8sClient), syncer, operatorVersion, operatorNamespace) + applier := remote_configuration.NewConfigurator(k8sClient, remote_configuration.CBGatewayCreator, log, operator.NewSecretAccessTokenProvider(k8sClient), operatorVersion, operatorNamespace) applierController := remote_configuration.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index f2772437..1feca645 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -266,8 +266,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) setupCRInK8S(mocks.k8sClient, nil) - mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) - mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) + setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) errFromService := errors.New("some error") mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) @@ -297,10 +296,9 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { configChange := remote_configuration.RandomNonNilChange() setupCRInK8S(mocks.k8sClient, nil) - mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) - mocks.apiGateway.EXPECT().GetSensorMetadata().Return(nil, nil) - mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{}, nil) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) diff --git a/remote_configuration/temp.go b/remote_configuration/temp.go index 681d7050..5a384861 100644 --- a/remote_configuration/temp.go +++ b/remote_configuration/temp.go @@ -30,6 +30,8 @@ func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update mo return nil } +// TODO: non-nil and with version set + func RandomNonNilChange() models.ConfigurationChange { for { c := RandomChange() @@ -40,10 +42,9 @@ func RandomNonNilChange() models.ConfigurationChange { } func RandomChange() *models.ConfigurationChange { - csRand, runtimeRand, cndrRand, versionRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)+1) + csRand, runtimeRand, cndrRand, versionRand, nilRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)), rand.Intn(10) - //csRand, runtimeRand, versionRand = 1, 2, 3 - if versionRand == len(versions) { + if nilRand%5 == 1 { return nil } From 516637a14ffb376c474bd3c0b1da6f832c664683 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 13:41:59 +0300 Subject: [PATCH 39/65] Change the agent_processor tests to match the new func. Remove a test that was not deterministic --- .../processors/agent_processor_test.go | 57 ++++++++++++------- cbcontainers/processors/mocks/generated.go | 1 - .../mocks/mock_api_gateway_creator.go | 51 ----------------- 3 files changed, 37 insertions(+), 72 deletions(-) delete mode 100644 cbcontainers/processors/mocks/mock_api_gateway_creator.go diff --git a/cbcontainers/processors/agent_processor_test.go b/cbcontainers/processors/agent_processor_test.go index cec857a1..b0c541ca 100644 --- a/cbcontainers/processors/agent_processor_test.go +++ b/cbcontainers/processors/agent_processor_test.go @@ -2,9 +2,9 @@ package processors_test import ( "fmt" + "github.com/go-logr/logr/testr" "testing" - logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" @@ -17,8 +17,9 @@ import ( type ClusterProcessorTestMocks struct { gatewayMock *mocks.MockAPIGateway - gatewayCreatorMock *mocks.MockAPIGatewayCreator operatorVersionProviderMock *mocks.MockOperatorVersionProvider + + mockGatewayCreatorFunc processors.APIGatewayCreator } type SetupAndAssertClusterProcessorTest func(*ClusterProcessorTestMocks, *processors.AgentProcessor) @@ -35,16 +36,22 @@ func testClusterProcessor(t *testing.T, setupAndAssert SetupAndAssertClusterProc mocksObjects := &ClusterProcessorTestMocks{ gatewayMock: mocks.NewMockAPIGateway(ctrl), - gatewayCreatorMock: mocks.NewMockAPIGatewayCreator(ctrl), operatorVersionProviderMock: mocks.NewMockOperatorVersionProvider(ctrl), } - processor := processors.NewAgentProcessor(logrTesting.NewTestLogger(t), mocksObjects.gatewayCreatorMock, mocksObjects.operatorVersionProviderMock, mockIdentifier) + // Proxy so tests can replace the actual implementation without creating a full mock + var mockCreator processors.APIGatewayCreator = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return mocksObjects.mockGatewayCreatorFunc(cbContainersCluster, accessToken) + } + + processor := processors.NewAgentProcessor(testr.New(t), mockCreator, mocksObjects.operatorVersionProviderMock, mockIdentifier) setupAndAssert(mocksObjects, processor) } func setupValidMocksCalls(testMocks *ClusterProcessorTestMocks, times int) { - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), AccessToken).Return(testMocks.gatewayMock, nil).Times(times) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().DoAndReturn(func() (*models.RegistrySecretValues, error) { return &models.RegistrySecretValues{Data: map[string][]byte{test_utils.RandomString(): {}}}, nil }).Times(times) @@ -86,10 +93,12 @@ func TestProcessorIsReCreatingComponentsForDifferentCR(t *testing.T) { }) } -func TestProcessorReturnsErrorWhenCanNotGetRegisterySecret(t *testing.T) { +func TestProcessorReturnsErrorWhenCanNotGetRegistrySecret(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(nil, fmt.Errorf("")) _, err := processor.Process(clusterCR, AccessToken) require.Error(t, err) @@ -99,7 +108,9 @@ func TestProcessorReturnsErrorWhenCanNotGetRegisterySecret(t *testing.T) { func TestProcessorReturnsErrorWhenCanNotRegisterCluster(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(fmt.Errorf("")) _, err := processor.Process(clusterCR, AccessToken) @@ -110,7 +121,9 @@ func TestProcessorReturnsErrorWhenCanNotRegisterCluster(t *testing.T) { func TestProcessorReturnsErrorWhenOperatorVersionProviderReturnsUnknownError(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil) testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", fmt.Errorf("intentional unknown error")) @@ -122,7 +135,9 @@ func TestProcessorReturnsErrorWhenOperatorVersionProviderReturnsUnknownError(t * func TestProcessorReturnsErrorWhenCanNotCreateGateway(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("")) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return nil, fmt.Errorf("") + } _, err := processor.Process(clusterCR, AccessToken) require.Error(t, err) }) @@ -139,18 +154,16 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", operator.ErrNotSemVer) }, }, - { - name: "when CreateGateway returns error", - setup: func(testMocks *ClusterProcessorTestMocks) { - testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("intentional error")) - }, - }, { name: "when GetCompatibilityMatrixEntryFor returns error", setup: func(testMocks *ClusterProcessorTestMocks) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return nil, fmt.Errorf("intentional error") + } + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetCompatibilityMatrixEntryFor(gomock.Any()).Return(nil, fmt.Errorf("intentional error")) }, }, @@ -158,7 +171,9 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { name: "when versions are compatible", setup: func(testMocks *ClusterProcessorTestMocks) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("1.0.0", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetCompatibilityMatrixEntryFor(gomock.Any()).Return(&models.OperatorCompatibility{ MinAgent: "0.9.0", MaxAgent: "1.1.0", @@ -171,7 +186,9 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "1.0.0", Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil) testCase.setup(testMocks) diff --git a/cbcontainers/processors/mocks/generated.go b/cbcontainers/processors/mocks/generated.go index 6d926540..dabf05c6 100644 --- a/cbcontainers/processors/mocks/generated.go +++ b/cbcontainers/processors/mocks/generated.go @@ -1,5 +1,4 @@ package mocks //go:generate mockgen -destination mock_api_gateway.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors APIGateway -//go:generate mockgen -destination mock_api_gateway_creator.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors APIGatewayCreator //go:generate mockgen -destination mock_operator_version_provider.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors OperatorVersionProvider diff --git a/cbcontainers/processors/mocks/mock_api_gateway_creator.go b/cbcontainers/processors/mocks/mock_api_gateway_creator.go deleted file mode 100644 index c0b611f6..00000000 --- a/cbcontainers/processors/mocks/mock_api_gateway_creator.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/cbcontainers/processors (interfaces: APIGatewayCreator) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - processors "github.com/vmware/cbcontainers-operator/cbcontainers/processors" -) - -// MockAPIGatewayCreator is a mock of APIGatewayCreator interface. -type MockAPIGatewayCreator struct { - ctrl *gomock.Controller - recorder *MockAPIGatewayCreatorMockRecorder -} - -// MockAPIGatewayCreatorMockRecorder is the mock recorder for MockAPIGatewayCreator. -type MockAPIGatewayCreatorMockRecorder struct { - mock *MockAPIGatewayCreator -} - -// NewMockAPIGatewayCreator creates a new mock instance. -func NewMockAPIGatewayCreator(ctrl *gomock.Controller) *MockAPIGatewayCreator { - mock := &MockAPIGatewayCreator{ctrl: ctrl} - mock.recorder = &MockAPIGatewayCreatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAPIGatewayCreator) EXPECT() *MockAPIGatewayCreatorMockRecorder { - return m.recorder -} - -// CreateGateway mocks base method. -func (m *MockAPIGatewayCreator) CreateGateway(arg0 *v1.CBContainersAgent, arg1 string) (processors.APIGateway, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateGateway", arg0, arg1) - ret0, _ := ret[0].(processors.APIGateway) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateGateway indicates an expected call of CreateGateway. -func (mr *MockAPIGatewayCreatorMockRecorder) CreateGateway(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateGateway", reflect.TypeOf((*MockAPIGatewayCreator)(nil).CreateGateway), arg0, arg1) -} From 2249a54b7c5da007514cf1e085d86ad44db299ba Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 13:50:05 +0300 Subject: [PATCH 40/65] Add clusterIdentifier parameter to GetConfigurationChanges --- .../communication/gateway/api_gateway.go | 4 +++- main.go | 10 +++++++++- remote_configuration/configurator.go | 9 ++++++--- remote_configuration/configurator_test.go | 18 +++++++++++------- remote_configuration/mocks/mock_api_gateway.go | 8 ++++---- remote_configuration/temp.go | 2 +- 6 files changed, 34 insertions(+), 17 deletions(-) diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index 541345b6..ebca9799 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -17,6 +17,8 @@ var ( ErrGettingOperatorCompatibility = errors.New("error while getting the operator compatibility") ) +// TODO: Extract the cluster group + name + ID as separate struct identifying a cluster and used together + type ApiGateway struct { account string cluster string @@ -172,7 +174,7 @@ func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) return nil, nil } -func (gateway *ApiGateway) GetConfigurationChanges(context.Context) ([]models.ConfigurationChange, error) { +func (gateway *ApiGateway) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { return nil, nil } diff --git a/main.go b/main.go index d88fcf09..86f499ca 100644 --- a/main.go +++ b/main.go @@ -190,7 +190,15 @@ func main() { os.Exit(1) } - applier := remote_configuration.NewConfigurator(k8sClient, remote_configuration.CBGatewayCreator, log, operator.NewSecretAccessTokenProvider(k8sClient), operatorVersion, operatorNamespace) + applier := remote_configuration.NewConfigurator( + k8sClient, + remote_configuration.CBGatewayCreator, + log, + operator.NewSecretAccessTokenProvider(k8sClient), + operatorVersion, + operatorNamespace, + clusterIdentifier, + ) applierController := remote_configuration.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index a0383577..8bb31b91 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -24,7 +24,7 @@ type ApiGateway interface { GetSensorMetadata() ([]models.SensorMetadata, error) GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) - GetConfigurationChanges(context.Context) ([]models.ConfigurationChange, error) + GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error } @@ -41,12 +41,13 @@ func CBGatewayCreator(cbContainersCluster *cbcontainersv1.CBContainersAgent, acc type ApiCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) type Configurator struct { + k8sClient client.Client logger logr.Logger accessTokenProvider AccessTokenProvider apiCreator ApiCreator operatorVersion string deployedNamespace string - k8sClient client.Client + clusterIdentifier string } func NewConfigurator( @@ -56,6 +57,7 @@ func NewConfigurator( accessTokenProvider AccessTokenProvider, operatorVersion string, deployedNamespace string, + clusterIdentifier string, ) *Configurator { return &Configurator{ k8sClient: k8sClient, @@ -64,6 +66,7 @@ func NewConfigurator( accessTokenProvider: accessTokenProvider, operatorVersion: operatorVersion, deployedNamespace: deployedNamespace, + clusterIdentifier: clusterIdentifier, } } @@ -138,7 +141,7 @@ func (configurator *Configurator) getCR(ctx context.Context) (*cbcontainersv1.CB } func (configurator *Configurator) getPendingChange(ctx context.Context, apiGateway ApiGateway) (*models.ConfigurationChange, error) { - changes, err := apiGateway.GetConfigurationChanges(ctx) + changes, err := apiGateway.GetConfigurationChanges(ctx, configurator.clusterIdentifier) if err != nil { return nil, err } diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 1feca645..ffc93024 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -34,6 +34,7 @@ type configuratorMocks struct { stubAccessToken string stubOperatorVersion string stubNamespace string + stubClusterID string } // setupConfigurator TODO @@ -52,6 +53,7 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura namespace := "namespace-name" accessToken := "access-token" operatorVersion := "1.2.3" + clusterID := "1234567" accessTokenProvider.EXPECT().GetCBAccessToken(gomock.Any(), gomock.Any(), namespace).Return(accessToken, nil).AnyTimes() configurator := remote_configuration.NewConfigurator( @@ -61,6 +63,7 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura accessTokenProvider, operatorVersion, namespace, + clusterID, ) mocksHolder := configuratorMocks{ @@ -70,6 +73,7 @@ func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configura stubAccessToken: accessToken, stubOperatorVersion: operatorVersion, stubNamespace: namespace, + stubClusterID: clusterID, } return configurator, mocksHolder @@ -90,7 +94,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { setupCRInK8S(mocks.k8sClient, cr) setupValidCompatibilityData(mocks.apiGateway, expectedAgentVersion, mocks.stubOperatorVersion) - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) // Setup mock assertions mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { @@ -128,7 +132,7 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { configChange.AgentVersion = &agentVersion setupCRInK8S(mocks.k8sClient, cr) - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) // Setup invalid compatibility; no need to do full verification here - this is what the validator tests are for // We just want to check that _some_ validation happens @@ -182,7 +186,7 @@ func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) setupCRInK8S(mocks.k8sClient, nil) - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(tC.dataFromService, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return(tC.dataFromService, nil) mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) err := configurator.RunIteration(context.Background()) @@ -208,7 +212,7 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) setupCRInK8S(mocks.k8sClient, nil) - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{newerChange, olderChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{newerChange, olderChange}, nil) setupValidCompatibilityData(mocks.apiGateway, expectedVersion, mocks.stubOperatorVersion) setupUpdateCRMock(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { @@ -234,7 +238,7 @@ func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) setupCRInK8S(mocks.k8sClient, nil) errFromService := errors.New("some error") - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return(nil, errFromService) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return(nil, errFromService) returnedErr := configurator.RunIteration(context.Background()) @@ -264,7 +268,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configChange := remote_configuration.RandomNonNilChange() - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) setupCRInK8S(mocks.k8sClient, nil) setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) @@ -297,7 +301,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { setupCRInK8S(mocks.k8sClient, nil) setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) - mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any()).Return([]models.ConfigurationChange{configChange}, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) errFromService := errors.New("some error") diff --git a/remote_configuration/mocks/mock_api_gateway.go b/remote_configuration/mocks/mock_api_gateway.go index 8e4458d2..1a2c1386 100644 --- a/remote_configuration/mocks/mock_api_gateway.go +++ b/remote_configuration/mocks/mock_api_gateway.go @@ -51,18 +51,18 @@ func (mr *MockApiGatewayMockRecorder) GetCompatibilityMatrixEntryFor(arg0 interf } // GetConfigurationChanges mocks base method. -func (m *MockApiGateway) GetConfigurationChanges(arg0 context.Context) ([]models.ConfigurationChange, error) { +func (m *MockApiGateway) GetConfigurationChanges(arg0 context.Context, arg1 string) ([]models.ConfigurationChange, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0) + ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0, arg1) ret0, _ := ret[0].([]models.ConfigurationChange) ret1, _ := ret[1].(error) return ret0, ret1 } // GetConfigurationChanges indicates an expected call of GetConfigurationChanges. -func (mr *MockApiGatewayMockRecorder) GetConfigurationChanges(arg0 interface{}) *gomock.Call { +func (mr *MockApiGatewayMockRecorder) GetConfigurationChanges(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockApiGateway)(nil).GetConfigurationChanges), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockApiGateway)(nil).GetConfigurationChanges), arg0, arg1) } // GetSensorMetadata mocks base method. diff --git a/remote_configuration/temp.go b/remote_configuration/temp.go index 5a384861..a58bcca3 100644 --- a/remote_configuration/temp.go +++ b/remote_configuration/temp.go @@ -17,7 +17,7 @@ var ( type DummyAPI struct { } -func (d DummyAPI) GetConfigurationChanges(ctx context.Context) ([]models.ConfigurationChange, error) { +func (d DummyAPI) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { c := RandomChange() if c != nil { return []models.ConfigurationChange{*c}, nil From e0c486bd4ef43af32b46371eee9fa94181c47344 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 13:54:08 +0300 Subject: [PATCH 41/65] Add clusterID to the status update model call --- cbcontainers/models/remote_configuration_changes.go | 7 ++++--- remote_configuration/configurator.go | 8 +++++--- remote_configuration/configurator_test.go | 5 +++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go index 22598656..9a98ad6d 100644 --- a/cbcontainers/models/remote_configuration_changes.go +++ b/cbcontainers/models/remote_configuration_changes.go @@ -17,7 +17,8 @@ type ConfigurationChangeStatusUpdate struct { // AppliedGeneration tracks the generation of the Custom resource where the change was applied AppliedGeneration int64 `json:"applied_generation"` // AppliedTimestamp records when the change was applied in RFC3339 format - AppliedTimestamp string `json:"applied_timestamp"` - - // TODO: CLuster and group. Cluster identifier? + AppliedTimestamp string `json:"applied_timestamp"` + ClusterIdentifier string `json:"cluster_identifier"` + ClusterGroup string `json:"cluster_group"` + ClusterName string `json:"cluster_name"` } diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 8bb31b91..e318e55a 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -182,12 +182,14 @@ func (configurator *Configurator) updateChangeStatus( Reason: "", // TODO AppliedGeneration: cr.Generation, AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), + ClusterIdentifier: configurator.clusterIdentifier, } } else { statusUpdate = models.ConfigurationChangeStatusUpdate{ - ID: change.ID, - Status: string(statusFailed), - Reason: encounteredError.Error(), // TODO + ID: change.ID, + Status: string(statusFailed), + Reason: encounteredError.Error(), // TODO + ClusterIdentifier: configurator.clusterIdentifier, } } diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index ffc93024..1a18c013 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -102,6 +102,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { assert.Equal(t, finalGeneration, update.AppliedGeneration) assert.Equal(t, "ACKNOWLEDGED", update.Status) assert.NotEmpty(t, update.AppliedTimestamp, "applied timestamp should be populated") + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) parsedTime, err := time.Parse(time.RFC3339, update.AppliedTimestamp) assert.NoError(t, err) @@ -118,8 +119,6 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { assert.NoError(t, err) } -// TODO: reintroduce - func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) @@ -150,6 +149,7 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { assert.NotEmpty(t, update.Reason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) return nil }) @@ -282,6 +282,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { assert.NotEmpty(t, update.Reason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) return nil }) From 5df87202d60432d03c8b6545514a266524d1d033 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 8 Sep 2023 16:19:54 +0300 Subject: [PATCH 42/65] Add secret detection to the changer --- .../models/remote_configuration_changes.go | 15 ++-- remote_configuration/change_applier.go | 6 ++ remote_configuration/change_applier_test.go | 89 +++++++++++++------ 3 files changed, 74 insertions(+), 36 deletions(-) diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go index 9a98ad6d..ba4f0c39 100644 --- a/cbcontainers/models/remote_configuration_changes.go +++ b/cbcontainers/models/remote_configuration_changes.go @@ -1,13 +1,14 @@ package models type ConfigurationChange struct { - ID string `json:"id"` - Status string `json:"status"` - AgentVersion *string `json:"agent_version"` - EnableClusterScanning *bool `json:"enable_cluster_scanning"` - EnableRuntime *bool `json:"enable_runtime"` - EnableCNDR *bool `json:"enable_cndr"` - Timestamp string `json:"timestamp"` + ID string `json:"id"` + Status string `json:"status"` + AgentVersion *string `json:"agent_version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` + EnableCNDR *bool `json:"enable_cndr"` + EnableClusterScanningSecretDetection *bool `json:"enable_cluster_scanning_secret_detection"` + Timestamp string `json:"timestamp"` } type ConfigurationChangeStatusUpdate struct { diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index d4c5ed62..52bb86b4 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -31,9 +31,15 @@ func (applier ChangeApplier) ApplyConfigChangeToCR(change models.ConfigurationCh if change.EnableClusterScanning != nil { cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning } + + if change.EnableClusterScanningSecretDetection != nil { + cr.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection = *change.EnableClusterScanningSecretDetection + } + if change.EnableRuntime != nil { cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime } + if change.EnableCNDR != nil { if cr.Spec.Components.Cndr == nil { cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 37674d36..71f3007f 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -14,7 +14,7 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { name string change models.ConfigurationChange initialCR cbcontainersv1.CBContainersAgent - assertFinalCR func(*testing.T, *cbcontainersv1.CBContainersAgent) + assertFinalCR func(*testing.T, cbcontainersv1.CBContainersAgent) } crVersion := "1.2.3" @@ -23,42 +23,43 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) generateFeatureToggleTestCases := func(feature string, - changeFieldSelector func(*models.ConfigurationChange) **bool, - crFieldSelector func(agent *cbcontainersv1.CBContainersAgent) **bool) []appliedChangeTest { + changeFieldChanger func(*models.ConfigurationChange, *bool), + crFieldChanger func(*cbcontainersv1.CBContainersAgent, *bool), + crAsserter func(*testing.T, cbcontainersv1.CBContainersAgent, *bool)) []appliedChangeTest { var result []appliedChangeTest for _, crState := range []*bool{truePtr, falsePtr, nil} { - cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} - crFieldPtr := crFieldSelector(&cr) - *crFieldPtr = crState + crState := crState // Avoid closure issues // Validate that each toggle state works (or doesn't do anything when it matches) - for _, changeState := range []*bool{truePtr, falsePtr} { + for _, changeState := range []*bool{falsePtr, truePtr} { + changeState := changeState // Avoid closure issues + + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} + crFieldChanger(&cr, crState) change := models.ConfigurationChange{} - changeFieldPtr := changeFieldSelector(&change) - *changeFieldPtr = changeState + changeFieldChanger(&change, changeState) - expectedState := changeState // avoid closure issues result = append(result, appliedChangeTest{ name: fmt.Sprintf("toggle feature (%s) from (%v) to (%v)", feature, prettyPrintBoolPtr(crState), prettyPrintBoolPtr(changeState)), change: change, initialCR: cr, - assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { - crFieldPostChangePtr := crFieldSelector(agent) - assert.Equal(t, expectedState, *crFieldPostChangePtr) + assertFinalCR: func(t *testing.T, agent cbcontainersv1.CBContainersAgent) { + crAsserter(t, agent, changeState) }, }) } + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} + crFieldChanger(&cr, crState) // Validate that a change with the toggle unset does not modify the CR result = append(result, appliedChangeTest{ name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), change: models.ConfigurationChange{}, initialCR: cr, - assertFinalCR: func(t *testing.T, agent *cbcontainersv1.CBContainersAgent) { - crFieldPostChangePtr := crFieldSelector(agent) - assert.Equal(t, *crFieldPtr, *crFieldPostChangePtr) + assertFinalCR: func(t *testing.T, agent cbcontainersv1.CBContainersAgent) { + crAsserter(t, agent, crState) }, }) } @@ -69,39 +70,69 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { var testCases []appliedChangeTest clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", - func(change *models.ConfigurationChange) **bool { - return &change.EnableClusterScanning - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { - return &agent.Spec.Components.ClusterScanning.Enabled + func(change *models.ConfigurationChange, val *bool) { + change.EnableClusterScanning = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + agent.Spec.Components.ClusterScanning.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.ClusterScanning.Enabled) + }) + + secretDetectionToggleTestCases := generateFeatureToggleTestCases("cluster scanning secret detection", + func(change *models.ConfigurationChange, val *bool) { + change.EnableClusterScanningSecretDetection = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + if val == nil { + // Bail out, this value is not valid for the flag + return + } + agent.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection = *val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + if b == nil { + // Bail out, this value is not valid for the flag + return + } + assert.Equal(t, *b, agent.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection) }) runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", - func(change *models.ConfigurationChange) **bool { - return &change.EnableRuntime - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { - return &agent.Spec.Components.RuntimeProtection.Enabled + func(change *models.ConfigurationChange, val *bool) { + change.EnableRuntime = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + agent.Spec.Components.RuntimeProtection.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.RuntimeProtection.Enabled) }) cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", - func(change *models.ConfigurationChange) **bool { - return &change.EnableCNDR - }, func(agent *cbcontainersv1.CBContainersAgent) **bool { + func(change *models.ConfigurationChange, val *bool) { + change.EnableCNDR = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { if agent.Spec.Components.Cndr == nil { agent.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} } - return &agent.Spec.Components.Cndr.Enabled + agent.Spec.Components.Cndr.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.Cndr.Enabled) }) testCases = append(testCases, clusterScannerToggleTestCases...) + testCases = append(testCases, secretDetectionToggleTestCases...) testCases = append(testCases, runtimeToggleTestCases...) testCases = append(testCases, cndrToggleTestCases...) + t1 := testCases[0] + t2 := testCases[1] + t3 := testCases[2] for _, testCase := range testCases { + testCase := testCase t.Run(testCase.name, func(t *testing.T) { + t.Log(t1, t2, t3) + target := remote_configuration.ChangeApplier{} target.ApplyConfigChangeToCR(testCase.change, &testCase.initialCR) - testCase.assertFinalCR(t, &testCase.initialCR) + testCase.assertFinalCR(t, testCase.initialCR) }) } } From b7fd76924a63dafd84db0cd006f8b307677e018c Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 14:15:28 +0300 Subject: [PATCH 43/65] Add secret detection to the validation --- cbcontainers/models/sensor_metadata.go | 11 ++++++----- remote_configuration/validation.go | 6 ++++++ remote_configuration/validation_test.go | 20 ++++++++++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/cbcontainers/models/sensor_metadata.go b/cbcontainers/models/sensor_metadata.go index f5c249b3..be27f469 100644 --- a/cbcontainers/models/sensor_metadata.go +++ b/cbcontainers/models/sensor_metadata.go @@ -1,9 +1,10 @@ package models type SensorMetadata struct { - Version string `json:"version"` - IsLatest bool `json:"is_latest" ` - SupportsRuntime bool `json:"supports_runtime"` - SupportsClusterScanning bool `json:"supports_cluster_scanning"` - SupportsCndr bool `json:"supports_cndr"` + Version string `json:"version"` + IsLatest bool `json:"is_latest" ` + SupportsRuntime bool `json:"supports_runtime"` + SupportsClusterScanning bool `json:"supports_cluster_scanning"` + SupportsClusterScanningSecrets bool `json:"supports_cluster_scanning_secrets"` + SupportsCndr bool `json:"supports_cndr"` } diff --git a/remote_configuration/validation.go b/remote_configuration/validation.go index e3c6933a..449bae4b 100644 --- a/remote_configuration/validation.go +++ b/remote_configuration/validation.go @@ -86,6 +86,12 @@ func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibi return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support cluster scanning feature", targetVersion)} } + if change.EnableClusterScanningSecretDetection != nil && + *change.EnableClusterScanningSecretDetection == true && + !sensor.SupportsClusterScanningSecrets { + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support secret detection during cluster scanning feature", targetVersion)} + } + if change.EnableRuntime != nil && *change.EnableRuntime == true && !sensor.SupportsRuntime { diff --git a/remote_configuration/validation_test.go b/remote_configuration/validation_test.go index cfd2ddfa..28916286 100644 --- a/remote_configuration/validation_test.go +++ b/remote_configuration/validation_test.go @@ -9,8 +9,6 @@ import ( "testing" ) -// TODO: add secret detection - var ( trueV = true truePtr = &trueV @@ -33,6 +31,15 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { SupportsClusterScanning: false, }, }, + { + name: "cluster scanning secret detection", + change: models.ConfigurationChange{ + EnableClusterScanningSecretDetection: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanningSecrets: false, + }, + }, { name: "runtime protection", change: models.ConfigurationChange{ @@ -95,6 +102,15 @@ func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { SupportsClusterScanning: true, }, }, + { + name: "cluster scanning secret detection", + change: models.ConfigurationChange{ + EnableClusterScanningSecretDetection: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanningSecrets: true, + }, + }, { name: "runtime protection", change: models.ConfigurationChange{ From 06968c4fe6415cc0412a4a2a0ddbf96d165edbe5 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 14:21:34 +0300 Subject: [PATCH 44/65] Minor TODOs --- main.go | 6 +++--- remote_configuration/configurator.go | 3 +-- remote_configuration/configurator_test.go | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 86f499ca..7ef05d02 100644 --- a/main.go +++ b/main.go @@ -146,6 +146,7 @@ func main() { } clusterIdentifier, k8sVersion := extractConfigurationVariables(mgr) + operatorVersionProvider := operator.NewEnvVersionProvider() // TODO: improve var processorGatewayCreator processors.APIGatewayCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { @@ -160,7 +161,7 @@ func main() { K8sVersion: k8sVersion, Namespace: operatorNamespace, AccessTokenProvider: operator.NewSecretAccessTokenProvider(mgr.GetClient()), - ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processorGatewayCreator, operator.NewEnvVersionProvider(), clusterIdentifier), + ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processorGatewayCreator, operatorVersionProvider, clusterIdentifier), StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent") @@ -183,8 +184,7 @@ func main() { signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") - versionReader := operator.NewEnvVersionProvider() // TODO: reuse from above - operatorVersion, err := versionReader.GetOperatorVersion() + operatorVersion, err := operatorVersionProvider.GetOperatorVersion() if err != nil && !errors.Is(err, operator.ErrNotSemVer) { setupLog.Error(err, "unable to read the running operator's version from environment variable") os.Exit(1) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index e318e55a..a4ae660e 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -12,9 +12,8 @@ import ( "time" ) -// TODO: Respect proxy config - // TODO: Split errors into visible and not visible +// TODO: timeout setup const ( timeoutSingleIteration = time.Second * 60 diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 1a18c013..ec225d2e 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -17,11 +17,9 @@ import ( "time" ) -// TODO: Add back the .finish for older mockgens // TODO: What error data to show and what not? // TODO: Reads cluster, etc from CR correctly? -// TODO: Respects proxy // TODO: Review gomock.any usages here // TODO: error on compatiblity calls @@ -121,6 +119,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { ctrl := gomock.NewController(t) + defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) @@ -315,6 +314,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { ctrl := gomock.NewController(t) + defer ctrl.Finish() configurator, mocks := setupConfigurator(ctrl) From 1b0db282dfe5a54598db83799e19c54ea7f2d14b Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 14:46:12 +0300 Subject: [PATCH 45/65] Some missing test cases for validation --- remote_configuration/validation.go | 4 +- remote_configuration/validation_test.go | 77 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/remote_configuration/validation.go b/remote_configuration/validation.go index 449bae4b..dce7fd20 100644 --- a/remote_configuration/validation.go +++ b/remote_configuration/validation.go @@ -21,13 +21,15 @@ func NewConfigurationChangeValidator(operatorVersion string, api ApiGateway) (*C if err != nil { return nil, err } + if compatibilityMatrix == nil { + return nil, fmt.Errorf("compatibility matrix API returned no data but no error as well, cannot continue") + } sensors, err := api.GetSensorMetadata() if err != nil { return nil, err } - // TODO: Dereference return &ConfigurationChangeValidator{ SensorData: sensors, OperatorCompatibilityData: *compatibilityMatrix, diff --git a/remote_configuration/validation_test.go b/remote_configuration/validation_test.go index 28916286..bfce3e46 100644 --- a/remote_configuration/validation_test.go +++ b/remote_configuration/validation_test.go @@ -1,11 +1,14 @@ package remote_configuration_test import ( + "errors" "fmt" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" "github.com/vmware/cbcontainers-operator/remote_configuration" + "github.com/vmware/cbcontainers-operator/remote_configuration/mocks" "testing" ) @@ -16,6 +19,52 @@ var ( falsePtr = &falseV ) +func TestValidatorConstructorReturnsErrOnFailures(t *testing.T) { + expectedOperatorVersion := "5.0.0" + + testCases := []struct { + name string + setupGatewayMock func(gateway *mocks.MockApiGateway) + }{ + { + name: "get compatibility returns err", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(nil, errors.New("some error")).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{}, nil).AnyTimes() + }, + }, + { + name: "get sensor metadata returns err", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(&models.OperatorCompatibility{}, nil).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return(nil, errors.New("some error")).AnyTimes() + }, + }, + { + name: "get compatibility returns nil", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(nil, nil).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{}, nil).AnyTimes() + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockGateway := mocks.NewMockApiGateway(ctrl) + tC.setupGatewayMock(mockGateway) + + validator, err := remote_configuration.NewConfigurationChangeValidator(expectedOperatorVersion, mockGateway) + + assert.Nil(t, validator) + assert.Error(t, err) + }) + } +} + func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { testCases := []struct { name string @@ -87,6 +136,34 @@ func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { } } +func TestValidateFailsIfSensorIsNotInList(t *testing.T) { + sensorMetaWithoutTargetSensor := []models.SensorMetadata{{ + Version: "1.0.0", + IsLatest: false, + SupportsRuntime: true, + SupportsClusterScanning: true, + SupportsClusterScanningSecrets: true, + SupportsCndr: true, + }} + operatorSupportsAll := models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + } + unknownVersion := "1.2.3" + + validator := remote_configuration.ConfigurationChangeValidator{ + SensorData: sensorMetaWithoutTargetSensor, + OperatorCompatibilityData: operatorSupportsAll, + } + + change := models.ConfigurationChange{ + AgentVersion: &unknownVersion, + } + cr := &cbcontainersv1.CBContainersAgent{} + + assert.Error(t, validator.ValidateChange(change, cr)) +} + func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { testCases := []struct { name string From 61874de1526e59cb6cf7a4af837014f9b6432182 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 15:51:22 +0300 Subject: [PATCH 46/65] More minor changes --- remote_configuration/configurator.go | 8 +++++--- remote_configuration/configurator_test.go | 7 +------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index a4ae660e..3ae71196 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -13,7 +13,7 @@ import ( ) // TODO: Split errors into visible and not visible -// TODO: timeout setup +// TODO: Check which type sshould be exposed const ( timeoutSingleIteration = time.Second * 60 @@ -86,7 +86,8 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { apiGateway, err := configurator.createAPIGateway(ctx, cr) if err != nil { - return err // TODO: ! + configurator.logger.Error(err, "Failed to create a valid CB API Gateway, cannot continue") + return err } configurator.logger.Info("Checking for pending remote configuration changes...") @@ -105,7 +106,8 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) if err != nil { - return err // TODO + configurator.logger.Error(err, "Failed to create a configuration change validator") + return err } errApplyingCR := configurator.applyChangeToCR(ctx, apiGateway, *change, cr, validator) diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index ec225d2e..0a2e6039 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -19,11 +19,6 @@ import ( // TODO: What error data to show and what not? -// TODO: Reads cluster, etc from CR correctly? -// TODO: Review gomock.any usages here - -// TODO: error on compatiblity calls - type configuratorMocks struct { k8sClient *k8sMocks.MockClient apiGateway *mocksConfigurator.MockApiGateway @@ -35,7 +30,7 @@ type configuratorMocks struct { stubClusterID string } -// setupConfigurator TODO +// setupConfigurator sets up mocks and creates a Configurator instance with those mocks and some dummy data func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { k8sClient := k8sMocks.NewMockClient(ctrl) apiGateway := mocksConfigurator.NewMockApiGateway(ctrl) From af5e9b03b0e5a1bd75457e17e5c821e63a7a9d5d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 16:48:18 +0300 Subject: [PATCH 47/65] Add sensor metadata implementation. Add dummy config changes implementation. --- .../communication/gateway/api_gateway.go | 31 ++++++++- .../gateway/dummy_configuration_data.go | 63 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 cbcontainers/communication/gateway/dummy_configuration_data.go diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index ebca9799..3d451adf 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -170,14 +170,41 @@ func (gateway *ApiGateway) GetCompatibilityMatrixEntryFor(operatorVersion string } func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) { - // TODO - return nil, nil + type getSensorsResponse struct { + Sensors []models.SensorMetadata `json:"sensors"` + } + + url := gateway.baseUrl("/setup/sensors") + resp, err := gateway.baseRequest(). + SetResult(getSensorsResponse{}). + Get(url) + + if err != nil { + return nil, err + } + if !resp.IsSuccess() { + return nil, fmt.Errorf("failed to get sensor metadata with status code (%d)", resp.StatusCode()) + } + + r, ok := resp.Result().(getSensorsResponse) + if !ok { + return nil, fmt.Errorf("malformed sensor metadata response") + } + return r.Sensors, nil } func (gateway *ApiGateway) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { + // TODO: Real implementation with CNS-2790 + c := randomRemoteConfigChange() + if c != nil { + return []models.ConfigurationChange{*c}, nil + + } return nil, nil } func (gateway *ApiGateway) UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error { + // TODO: Real implementation with CNS-2790 + return nil } diff --git a/cbcontainers/communication/gateway/dummy_configuration_data.go b/cbcontainers/communication/gateway/dummy_configuration_data.go new file mode 100644 index 00000000..62fc391b --- /dev/null +++ b/cbcontainers/communication/gateway/dummy_configuration_data.go @@ -0,0 +1,63 @@ +package gateway + +import ( + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "math/rand" + "strconv" +) + +// TODO: This will be removed once real APIs are implemented for this but it helps try the feature while in development +// API task - CNS-2790 + +var ( + tr = true + fal = false + dummyAgentVersions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} +) + +func randomRemoteConfigChange() *models.ConfigurationChange { + csRand, runtimeRand, cndrRand, versionRand, nilRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(dummyAgentVersions)), rand.Int() + + if nilRand%5 == 1 { + return nil + } + + changeVersion := &dummyAgentVersions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + var changeCNDR *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = &tr + case 2, 4: + changeClusterScanning = &fal + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = &tr + case 2, 4: + changeRuntime = &fal + default: + changeRuntime = nil + } + + if changeVersion != nil && *changeVersion == "3.0.0" && cndrRand%2 == 0 { + changeCNDR = &tr + } else { + changeCNDR = &fal + } + + return &models.ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + EnableCNDR: changeCNDR, + Status: models.ChangeStatusPending, + } +} From 5add191aa73ba1cd1171233bc7f4f8a0e99dc23a Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 16:59:06 +0300 Subject: [PATCH 48/65] Move data classes around --- .../models/remote_configuration_changes.go | 30 +++--- remote_configuration/change_applier_test.go | 52 ++++++++++ remote_configuration/configurator.go | 6 +- remote_configuration/configurator_test.go | 18 ++-- remote_configuration/temp.go | 97 ------------------- 5 files changed, 83 insertions(+), 120 deletions(-) delete mode 100644 remote_configuration/temp.go diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go index ba4f0c39..5e12645b 100644 --- a/cbcontainers/models/remote_configuration_changes.go +++ b/cbcontainers/models/remote_configuration_changes.go @@ -1,20 +1,28 @@ package models +type RemoteChangeStatus string + +var ( + ChangeStatusPending RemoteChangeStatus = "PENDING" + ChangeStatusAcked RemoteChangeStatus = "ACKNOWLEDGED" + ChangeStatusFailed RemoteChangeStatus = "FAILED" +) + type ConfigurationChange struct { - ID string `json:"id"` - Status string `json:"status"` - AgentVersion *string `json:"agent_version"` - EnableClusterScanning *bool `json:"enable_cluster_scanning"` - EnableRuntime *bool `json:"enable_runtime"` - EnableCNDR *bool `json:"enable_cndr"` - EnableClusterScanningSecretDetection *bool `json:"enable_cluster_scanning_secret_detection"` - Timestamp string `json:"timestamp"` + ID string `json:"id"` + Status RemoteChangeStatus `json:"status"` + AgentVersion *string `json:"agent_version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` + EnableCNDR *bool `json:"enable_cndr"` + EnableClusterScanningSecretDetection *bool `json:"enable_cluster_scanning_secret_detection"` + Timestamp string `json:"timestamp"` } type ConfigurationChangeStatusUpdate struct { - ID string `json:"id"` - Status string `json:"status"` - Reason string `json:"reason"` + ID string `json:"id"` + Status RemoteChangeStatus `json:"status"` + Reason string `json:"reason"` // AppliedGeneration tracks the generation of the Custom resource where the change was applied AppliedGeneration int64 `json:"applied_generation"` // AppliedTimestamp records when the change was applied in RFC3339 format diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 71f3007f..1e74bbda 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -6,6 +6,8 @@ import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" "github.com/vmware/cbcontainers-operator/remote_configuration" + "math/rand" + "strconv" "testing" ) @@ -242,3 +244,53 @@ func prettyPrintBoolPtr(v *bool) string { } return fmt.Sprintf("%t", *v) } + +// randomPendingConfigChange creates a non-empty configuration change with randomly populated fields in pending state +// the change is not guaranteed to be 100% valid +func randomPendingConfigChange() models.ConfigurationChange { + var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} + + csRand, runtimeRand, cndrRand, versionRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)) + + changeVersion := &versions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + var changeCNDR *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = truePtr + case 2, 4: + changeClusterScanning = falsePtr + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = truePtr + case 2, 4: + changeRuntime = falsePtr + default: + changeRuntime = nil + } + + switch cndrRand % 5 { + case 1, 3: + changeCNDR = truePtr + case 2, 4: + changeCNDR = falsePtr + default: + changeCNDR = nil + } + + return models.ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + EnableCNDR: changeCNDR, + Status: models.ChangeStatusPending, + } +} diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 3ae71196..b6f823b4 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -152,7 +152,7 @@ func (configurator *Configurator) getPendingChange(ctx context.Context, apiGatew }) for _, change := range changes { - if change.Status == string(statusPending) { + if change.Status == models.ChangeStatusPending { return &change, nil } } @@ -179,7 +179,7 @@ func (configurator *Configurator) updateChangeStatus( if encounteredError == nil { statusUpdate = models.ConfigurationChangeStatusUpdate{ ID: change.ID, - Status: string(statusAcknowledged), + Status: models.ChangeStatusAcked, Reason: "", // TODO AppliedGeneration: cr.Generation, AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), @@ -188,7 +188,7 @@ func (configurator *Configurator) updateChangeStatus( } else { statusUpdate = models.ConfigurationChangeStatusUpdate{ ID: change.ID, - Status: string(statusFailed), + Status: models.ChangeStatusFailed, Reason: encounteredError.Error(), // TODO ClusterIdentifier: configurator.clusterIdentifier, } diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 0a2e6039..e0bbdca0 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -82,7 +82,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { var initialGeneration, finalGeneration int64 = 1, 2 expectedAgentVersion := "3.0.0" cr := &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}} - configChange := remote_configuration.RandomNonNilChange() + configChange := randomPendingConfigChange() configChange.AgentVersion = &expectedAgentVersion setupCRInK8S(mocks.k8sClient, cr) @@ -93,7 +93,7 @@ func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, finalGeneration, update.AppliedGeneration) - assert.Equal(t, "ACKNOWLEDGED", update.Status) + assert.Equal(t, models.ChangeStatusAcked, update.Status) assert.NotEmpty(t, update.AppliedTimestamp, "applied timestamp should be populated") assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) @@ -121,7 +121,7 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { cr := &cbcontainersv1.CBContainersAgent{} maxAgentVersionForOperator := "4.0.0" agentVersion := "5.0.0" - configChange := remote_configuration.RandomNonNilChange() + configChange := randomPendingConfigChange() configChange.AgentVersion = &agentVersion setupCRInK8S(mocks.k8sClient, cr) @@ -139,7 +139,7 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) + assert.Equal(t, models.ChangeStatusFailed, update.Status) assert.NotEmpty(t, update.Reason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) @@ -195,8 +195,8 @@ func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - olderChange := remote_configuration.RandomNonNilChange() - newerChange := remote_configuration.RandomNonNilChange() + olderChange := randomPendingConfigChange() + newerChange := randomPendingConfigChange() expectedVersion := "version-for-older-change" versionThatShouldNotBe := "version-for-newer-change" @@ -260,7 +260,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - configChange := remote_configuration.RandomNonNilChange() + configChange := randomPendingConfigChange() mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) setupCRInK8S(mocks.k8sClient, nil) @@ -272,7 +272,7 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) - assert.Equal(t, "FAILED", update.Status) + assert.Equal(t, models.ChangeStatusFailed, update.Status) assert.NotEmpty(t, update.Reason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) @@ -292,7 +292,7 @@ func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { configurator, mocks := setupConfigurator(ctrl) - configChange := remote_configuration.RandomNonNilChange() + configChange := randomPendingConfigChange() setupCRInK8S(mocks.k8sClient, nil) setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) diff --git a/remote_configuration/temp.go b/remote_configuration/temp.go deleted file mode 100644 index a58bcca3..00000000 --- a/remote_configuration/temp.go +++ /dev/null @@ -1,97 +0,0 @@ -package remote_configuration - -import ( - "context" - "github.com/vmware/cbcontainers-operator/cbcontainers/models" - "math/rand" - "strconv" -) - -var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} - -var ( - tr = true - fal = false -) - -type DummyAPI struct { -} - -func (d DummyAPI) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { - c := RandomChange() - if c != nil { - return []models.ConfigurationChange{*c}, nil - - } - return nil, nil -} - -func (d DummyAPI) UpdateConfigurationChangeStatus(ctx context.Context, update models.ConfigurationChangeStatusUpdate) error { - return nil -} - -// TODO: non-nil and with version set - -func RandomNonNilChange() models.ConfigurationChange { - for { - c := RandomChange() - if c != nil { - return *c - } - } -} - -func RandomChange() *models.ConfigurationChange { - csRand, runtimeRand, cndrRand, versionRand, nilRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)), rand.Intn(10) - - if nilRand%5 == 1 { - return nil - } - - changeVersion := &versions[versionRand] - - var changeClusterScanning *bool - var changeRuntime *bool - var changeCNDR *bool - - switch csRand % 5 { - case 1, 3: - changeClusterScanning = &tr - case 2, 4: - changeClusterScanning = &fal - default: - changeClusterScanning = nil - } - - switch runtimeRand % 5 { - case 1, 3: - changeRuntime = &tr - case 2, 4: - changeRuntime = &fal - default: - changeRuntime = nil - } - - if changeVersion != nil && *changeVersion == "3.0.0" && cndrRand%2 == 0 { - changeCNDR = &tr - } else { - changeCNDR = &fal - } - - return &models.ConfigurationChange{ - ID: strconv.Itoa(rand.Int()), - AgentVersion: changeVersion, - EnableClusterScanning: changeClusterScanning, - EnableRuntime: changeRuntime, - EnableCNDR: changeCNDR, - Status: string(statusPending), - } -} - -type changeStatus string - -var ( - statusPending changeStatus = "PENDING" - statusAcknowledged changeStatus = "ACKNOWLEDGED" // TODO: Acknowledged or applied? - statusFailed changeStatus = "FAILED" -) From c40ca39c0a48f6a4c37451dabbf1d6bbdce15b44 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Mon, 11 Sep 2023 17:06:00 +0300 Subject: [PATCH 49/65] Removing TODOs --- main.go | 13 ++++++------- remote_configuration/configurator.go | 7 ------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index 7ef05d02..9b71c2b2 100644 --- a/main.go +++ b/main.go @@ -147,13 +147,11 @@ func main() { clusterIdentifier, k8sVersion := extractConfigurationVariables(mgr) operatorVersionProvider := operator.NewEnvVersionProvider() - - // TODO: improve var processorGatewayCreator processors.APIGatewayCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { return gateway.NewDefaultGatewayCreator().CreateGateway(cbContainersCluster, accessToken) } - cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") + if err = (&controllers.CBContainersAgentController{ Client: mgr.GetClient(), Log: cbContainersAgentLogger, @@ -179,9 +177,6 @@ func main() { os.Exit(1) } - // TODO: Prettify - - signalsContext := ctrl.SetupSignalHandler() k8sClient := mgr.GetClient() log := ctrl.Log.WithName("configurator") operatorVersion, err := operatorVersionProvider.GetOperatorVersion() @@ -189,10 +184,13 @@ func main() { setupLog.Error(err, "unable to read the running operator's version from environment variable") os.Exit(1) } + var configuratorGatewayCreator remote_configuration.ApiCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (remote_configuration.ApiGateway, error) { + return gateway.NewDefaultGatewayCreator().CreateGateway(cbContainersCluster, accessToken) + } applier := remote_configuration.NewConfigurator( k8sClient, - remote_configuration.CBGatewayCreator, + configuratorGatewayCreator, log, operator.NewSecretAccessTokenProvider(k8sClient), operatorVersion, @@ -204,6 +202,7 @@ func main() { var wg sync.WaitGroup wg.Add(2) + signalsContext := ctrl.SetupSignalHandler() go func() { defer wg.Done() setupLog.Info("starting manager") diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index b6f823b4..408ecfc6 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" "github.com/vmware/cbcontainers-operator/cbcontainers/models" "sigs.k8s.io/controller-runtime/pkg/client" "sort" @@ -31,12 +30,6 @@ type AccessTokenProvider interface { GetCBAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, deployedNamespace string) (string, error) } -// CBGatewayCreator creates an implementation of ApiGateway that talks to the real backend -func CBGatewayCreator(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) { - creator := gateway.DefaultGatewayCreator{} - return creator.CreateGateway(cbContainersCluster, accessToken) -} - type ApiCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) type Configurator struct { From ca55c8dfdb53b74709fe5a07bc5f93b8760054e4 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 10:29:31 +0300 Subject: [PATCH 50/65] Change the feature toggle to be part of the CR --- api/v1/cbcontainersagent_types.go | 11 +++++++++ .../templates/operator.yaml | 12 ++++++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 12 ++++++++++ ...ers.carbonblack.io_cbcontainersagents.yaml | 10 ++++++++ docs/crds.md | 7 +++--- main.go | 21 +++++++---------- remote_configuration/configurator.go | 7 ++++++ remote_configuration/configurator_test.go | 23 +++++++++++++++++++ 8 files changed, 87 insertions(+), 16 deletions(-) diff --git a/api/v1/cbcontainersagent_types.go b/api/v1/cbcontainersagent_types.go index 2ac3d9a8..24727d3c 100644 --- a/api/v1/cbcontainersagent_types.go +++ b/api/v1/cbcontainersagent_types.go @@ -83,6 +83,9 @@ type CBContainersComponentsSettings struct { // care of determining the necessary `NO_PROXY` settings. // Proxy *CBContainersProxySettings `json:"proxy,omitempty"` + + // RemoteConfiguration holds settings for the operator/agent's feature to apply configuration changes via the Carbon black console + RemoteConfiguration *CBContainersRemoteConfigurationSettings `json:"remoteConfiguration,omitempty"` } func (s CBContainersComponentsSettings) ShouldCreateDefaultImagePullSecrets() bool { @@ -126,6 +129,14 @@ type CBContainersProxySettings struct { NoProxySuffix *string `json:"noProxySuffix,omitempty"` } +// CBContainersRemoteConfigurationSettings holds settings for the operator/agent's feature to apply configuration changes via the Carbon black console +type CBContainersRemoteConfigurationSettings struct { + // EnabledForAgent turns the feature to change agent configuration remotely (as opposed to operator configuration) + // + // +kubebuilder:default:=true + EnabledForAgent *bool `json:"enabledForAgent,omitempty"` +} + // CBContainersAgentStatus defines the observed state of CBContainersAgent type CBContainersAgentStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 4a0add7e..27652a5d 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -6322,6 +6322,18 @@ spec: defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + default: true + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator + configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 6109fb6e..9c409c03 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -6312,6 +6312,18 @@ spec: defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + default: true + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator + configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 792a2e84..27d1607b 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -5830,6 +5830,16 @@ spec: exposed more as a means by which to control the defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/docs/crds.md b/docs/crds.md index 5d15e836..0dd4eb99 100644 --- a/docs/crds.md +++ b/docs/crds.md @@ -110,6 +110,7 @@ This is the CR you'll need to deploy in order to trigger the operator to deploy ### Other Components Optional parameters -| Parameter | Description | Default | -|--------------------------------------------------|----------------------------------------------|-------------| -| `spec.components.settings.daemonSetsTolerations` | Carbon Black DaemonSet Component Tolerations | Empty array | +| Parameter | Description | Default | +|----------------------------------------------------------------|--------------------------------------------------------------------------------|-------------| +| `spec.components.settings.daemonSetsTolerations` | Carbon Black DaemonSet Component Tolerations | Empty array | +| `spec.components.settings.remoteConfiguration.enabledForAgent` | Enables applying custom resource changes remotely via the Carbon Black Console | True | diff --git a/main.go b/main.go index 9b71c2b2..11237bfd 100644 --- a/main.go +++ b/main.go @@ -60,12 +60,11 @@ var ( ) const ( - NamespaceIdentifier = "default" - httpProxyEnv = "HTTP_PROXY" - httpsProxyEnv = "HTTPS_PROXY" - noProxyEnv = "NO_PROXY" - namespaceEnv = "OPERATOR_NAMESPACE" - enableRemoteConfiguratorEnv = "ENABLE_REMOTE_CONFIGURATOR" + NamespaceIdentifier = "default" + httpProxyEnv = "HTTP_PROXY" + httpsProxyEnv = "HTTPS_PROXY" + noProxyEnv = "NO_PROXY" + namespaceEnv = "OPERATOR_NAMESPACE" ) func init() { @@ -205,6 +204,7 @@ func main() { signalsContext := ctrl.SetupSignalHandler() go func() { defer wg.Done() + setupLog.Info("starting manager") if err := mgr.Start(signalsContext); err != nil { setupLog.Error(err, "problem running manager") @@ -214,13 +214,8 @@ func main() { go func() { defer wg.Done() - enableConfigurator := os.Getenv(enableRemoteConfiguratorEnv) - if enableConfigurator == "true" { - setupLog.Info("Starting remote configurator") - applierController.RunLoop(signalsContext) - } else { - setupLog.Info(fmt.Sprintf("Environment variable %s is not set to true, remote configuration feature will be disabled", enableRemoteConfiguratorEnv)) - } + setupLog.Info("Starting remote configurator") + applierController.RunLoop(signalsContext) }() wg.Wait() diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 408ecfc6..e4aa5a8d 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -77,6 +77,13 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return nil } + if remoteConfigSettings := cr.Spec.Components.Settings.RemoteConfiguration; remoteConfigSettings != nil && + remoteConfigSettings.EnabledForAgent != nil && + *remoteConfigSettings.EnabledForAgent == false { + configurator.logger.Info("Remote configuration feature is disabled, no changes will be made") + return nil + + } apiGateway, err := configurator.createAPIGateway(ctx, cr) if err != nil { configurator.logger.Error(err, "Failed to create a valid CB API Gateway, cannot continue") diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index e0bbdca0..97718a05 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -318,6 +318,29 @@ func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { list.Items = []cbcontainersv1.CBContainersAgent{} }) + // No other mock calls should happen without a CR + assert.NoError(t, configurator.RunIteration(context.Background())) +} + +func TestWhenFeatureIsDisabledInCRNothingHappens(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + cr := &cbcontainersv1.CBContainersAgent{ + Spec: cbcontainersv1.CBContainersAgentSpec{ + Components: cbcontainersv1.CBContainersComponentsSpec{ + Settings: cbcontainersv1.CBContainersComponentsSettings{ + RemoteConfiguration: &cbcontainersv1.CBContainersRemoteConfigurationSettings{ + EnabledForAgent: falsePtr, + }, + }}, + }, + } + setupCRInK8S(mocks.k8sClient, cr) + + // No other mock calls should happen once we know the feature is disabled assert.NoError(t, configurator.RunIteration(context.Background())) } From 4b891dd401420287f614378e1c5e5dcb873ae5e8 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 11:06:51 +0300 Subject: [PATCH 51/65] Change errors to have system info and user info for debugging vs user context in the UI --- .../models/remote_configuration_changes.go | 21 +++++++++----- remote_configuration/configurator.go | 29 +++++++++---------- remote_configuration/configurator_test.go | 8 ++--- remote_configuration/validation.go | 2 -- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go index 5e12645b..54603b6d 100644 --- a/cbcontainers/models/remote_configuration_changes.go +++ b/cbcontainers/models/remote_configuration_changes.go @@ -20,14 +20,21 @@ type ConfigurationChange struct { } type ConfigurationChangeStatusUpdate struct { - ID string `json:"id"` - Status RemoteChangeStatus `json:"status"` - Reason string `json:"reason"` + ID string `json:"id"` + ClusterIdentifier string `json:"cluster_identifier"` + ClusterGroup string `json:"cluster_group"` + ClusterName string `json:"cluster_name"` + Status RemoteChangeStatus `json:"status"` + // AppliedGeneration tracks the generation of the Custom resource where the change was applied AppliedGeneration int64 `json:"applied_generation"` // AppliedTimestamp records when the change was applied in RFC3339 format - AppliedTimestamp string `json:"applied_timestamp"` - ClusterIdentifier string `json:"cluster_identifier"` - ClusterGroup string `json:"cluster_group"` - ClusterName string `json:"cluster_name"` + AppliedTimestamp string `json:"applied_timestamp"` + + // Error should hold information about encountered errors when the change application failed. + // For system usage only, not meant for end-users. + Error string `json:"encountered_error"` + // ErrorReason should be populated if some additional information can be shown to the user (e.g. why a change was invalid) + // It should not be used to store system information + ErrorReason string `json:"error_reason"` } diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index e4aa5a8d..02a3475b 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -2,6 +2,7 @@ package remote_configuration import ( "context" + "errors" "fmt" "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" @@ -11,7 +12,6 @@ import ( "time" ) -// TODO: Split errors into visible and not visible // TODO: Check which type sshould be exposed const ( @@ -175,22 +175,21 @@ func (configurator *Configurator) updateChangeStatus( cr *cbcontainersv1.CBContainersAgent, encounteredError error, ) error { - var statusUpdate models.ConfigurationChangeStatusUpdate + statusUpdate := models.ConfigurationChangeStatusUpdate{ + ID: change.ID, + ClusterIdentifier: configurator.clusterIdentifier, + } + if encounteredError == nil { - statusUpdate = models.ConfigurationChangeStatusUpdate{ - ID: change.ID, - Status: models.ChangeStatusAcked, - Reason: "", // TODO - AppliedGeneration: cr.Generation, - AppliedTimestamp: time.Now().UTC().Format(time.RFC3339), - ClusterIdentifier: configurator.clusterIdentifier, - } + statusUpdate.Status = models.ChangeStatusAcked + statusUpdate.AppliedGeneration = cr.Generation + statusUpdate.AppliedTimestamp = time.Now().UTC().Format(time.RFC3339) } else { - statusUpdate = models.ConfigurationChangeStatusUpdate{ - ID: change.ID, - Status: models.ChangeStatusFailed, - Reason: encounteredError.Error(), // TODO - ClusterIdentifier: configurator.clusterIdentifier, + statusUpdate.Status = models.ChangeStatusFailed + statusUpdate.Error = encounteredError.Error() + // Validation change is the only thing we can safely give information to the user about + if errors.As(encounteredError, &invalidChangeError{}) { + statusUpdate.ErrorReason = encounteredError.Error() } } diff --git a/remote_configuration/configurator_test.go b/remote_configuration/configurator_test.go index 97718a05..b336bb7e 100644 --- a/remote_configuration/configurator_test.go +++ b/remote_configuration/configurator_test.go @@ -17,8 +17,6 @@ import ( "time" ) -// TODO: What error data to show and what not? - type configuratorMocks struct { k8sClient *k8sMocks.MockClient apiGateway *mocksConfigurator.MockApiGateway @@ -140,7 +138,8 @@ func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, models.ChangeStatusFailed, update.Status) - assert.NotEmpty(t, update.Reason) + assert.NotEmpty(t, update.Error) + assert.NotEmpty(t, update.ErrorReason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) @@ -273,7 +272,8 @@ func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { assert.Equal(t, configChange.ID, update.ID) assert.Equal(t, models.ChangeStatusFailed, update.Status) - assert.NotEmpty(t, update.Reason) + assert.NotEmpty(t, update.Error) + assert.Empty(t, update.ErrorReason) assert.Equal(t, int64(0), update.AppliedGeneration) assert.Empty(t, update.AppliedTimestamp) assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) diff --git a/remote_configuration/validation.go b/remote_configuration/validation.go index dce7fd20..09276d1c 100644 --- a/remote_configuration/validation.go +++ b/remote_configuration/validation.go @@ -6,8 +6,6 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/models" ) -// TODO: Move - type invalidChangeError struct { msg string } From 375cf9980e2c08ece8ce654febf80d463c21c6c2 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 12:26:25 +0300 Subject: [PATCH 52/65] Small refactor --- remote_configuration/configurator.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 02a3475b..633e57a8 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -102,23 +102,21 @@ func (configurator *Configurator) RunIteration(ctx context.Context) error { return nil } - // TODO: This is ugly... configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) - validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) - if err != nil { - configurator.logger.Error(err, "Failed to create a configuration change validator") - return err + errApplyingCR := configurator.applyChangeToCR(ctx, apiGateway, *change, cr) + if errApplyingCR != nil { + configurator.logger.Error(errApplyingCR, "Failed to apply configuration changes to CBContainerAGent resource") + // Intentional fallthrough as we want to report the change application as failed to the backend + } else { + configurator.logger.Info("Successfully applied configuration changes to CBContainerAgent resource") } - errApplyingCR := configurator.applyChangeToCR(ctx, apiGateway, *change, cr, validator) - // TODO: Explain - if err := configurator.updateChangeStatus(ctx, apiGateway, *change, cr, errApplyingCR); err != nil { configurator.logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") return err } - // If we failed to apply the CR, we still report this to the backend but want to return the apply error here to propagate properly + // If we failed to apply the CR, we report this to the backend but want to return the apply error here to indicate a failure return errApplyingCR } @@ -159,7 +157,11 @@ func (configurator *Configurator) getPendingChange(ctx context.Context, apiGatew return nil, nil } -func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGateway ApiGateway, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent, validator *ConfigurationChangeValidator) error { +func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGateway ApiGateway, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { + validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) + if err != nil { + return fmt.Errorf("failed to create configuration change validator") + } if err := validator.ValidateChange(change, cr); err != nil { return err } From d533eea56900cff34f4b335a2d7bae0db4e7936d Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 12:30:47 +0300 Subject: [PATCH 53/65] Change dummy struct to just a func --- remote_configuration/change_applier.go | 5 ++--- remote_configuration/change_applier_test.go | 20 ++++---------------- remote_configuration/configurator.go | 5 +---- remote_configuration/controller.go | 4 ++-- 4 files changed, 9 insertions(+), 25 deletions(-) diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index 52bb86b4..c8c7c020 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -5,9 +5,8 @@ import ( "github.com/vmware/cbcontainers-operator/cbcontainers/models" ) -type ChangeApplier struct{} - -func (applier ChangeApplier) ApplyConfigChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { +// ApplyConfigChangeToCR will modify CR according to the values in the configuration change provided +func ApplyConfigChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { resetVersion := func(ptrToField *string) { if ptrToField != nil && *ptrToField != "" { *ptrToField = "" diff --git a/remote_configuration/change_applier_test.go b/remote_configuration/change_applier_test.go index 1e74bbda..52bf5d14 100644 --- a/remote_configuration/change_applier_test.go +++ b/remote_configuration/change_applier_test.go @@ -123,17 +123,10 @@ func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { testCases = append(testCases, runtimeToggleTestCases...) testCases = append(testCases, cndrToggleTestCases...) - t1 := testCases[0] - t2 := testCases[1] - t3 := testCases[2] for _, testCase := range testCases { testCase := testCase t.Run(testCase.name, func(t *testing.T) { - t.Log(t1, t2, t3) - - target := remote_configuration.ChangeApplier{} - - target.ApplyConfigChangeToCR(testCase.change, &testCase.initialCR) + remote_configuration.ApplyConfigChangeToCR(testCase.change, &testCase.initialCR) testCase.assertFinalCR(t, testCase.initialCR) }) } @@ -145,9 +138,7 @@ func TestVersionIsAppliedCorrectly(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := models.ConfigurationChange{AgentVersion: &newVersion} - target := remote_configuration.ChangeApplier{} - - target.ApplyConfigChangeToCR(change, &cr) + remote_configuration.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) } @@ -156,8 +147,7 @@ func TestMissingVersionDoesNotModifyCR(t *testing.T) { cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} change := models.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} - target := remote_configuration.ChangeApplier{} - target.ApplyConfigChangeToCR(change, &cr) + remote_configuration.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, originalVersion, cr.Spec.Version) } @@ -222,9 +212,7 @@ func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { newVersion := "new-version" change := models.ConfigurationChange{AgentVersion: &newVersion} - target := remote_configuration.ChangeApplier{} - - target.ApplyConfigChangeToCR(change, &cr) + remote_configuration.ApplyConfigChangeToCR(change, &cr) assert.Equal(t, newVersion, cr.Spec.Version) // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields // => the operator will use the common version instead diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 633e57a8..9d2e975c 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -12,8 +12,6 @@ import ( "time" ) -// TODO: Check which type sshould be exposed - const ( timeoutSingleIteration = time.Second * 60 ) @@ -165,8 +163,7 @@ func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGatewa if err := validator.ValidateChange(change, cr); err != nil { return err } - c := ChangeApplier{} - c.ApplyConfigChangeToCR(change, cr) + ApplyConfigChangeToCR(change, cr) return configurator.k8sClient.Update(ctx, cr) } diff --git a/remote_configuration/controller.go b/remote_configuration/controller.go index a2c35aee..10630395 100644 --- a/remote_configuration/controller.go +++ b/remote_configuration/controller.go @@ -8,8 +8,8 @@ import ( ) const ( - sleepDuration = 20 * time.Second - maxRetries = 10 // 1024s or ~17 minutes at peak + sleepDuration = 20 * time.Second // TODO: Increase + maxRetries = 10 // 1024s or ~17 minutes at peak ) type configurationApplier interface { From ef9ea8ff7fc7c442a6ee243c742eb05f1760d702 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 12:35:14 +0300 Subject: [PATCH 54/65] Bump timeout a bit --- remote_configuration/configurator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 9d2e975c..772502e8 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -13,7 +13,7 @@ import ( ) const ( - timeoutSingleIteration = time.Second * 60 + timeoutSingleIteration = time.Second * 120 ) type ApiGateway interface { From 4ca10a7e630beacfb7fb0449a1db8a5a04ed64b1 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 12:36:25 +0300 Subject: [PATCH 55/65] Remove controller_test.go --- remote_configuration/controller_test.go | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 remote_configuration/controller_test.go diff --git a/remote_configuration/controller_test.go b/remote_configuration/controller_test.go deleted file mode 100644 index 08735e67..00000000 --- a/remote_configuration/controller_test.go +++ /dev/null @@ -1,5 +0,0 @@ -package remote_configuration_test - -// No error -> normal waiting time -// Error encountered -> with backoff -// Respects cancellation From e4e143594aa05a376d8c272f61e67f4d2910bbfd Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 12:47:22 +0300 Subject: [PATCH 56/65] Add back the env var during development until the feature is completed (so it is not ON by default when testing) --- main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 11237bfd..5a5bb834 100644 --- a/main.go +++ b/main.go @@ -214,8 +214,12 @@ func main() { go func() { defer wg.Done() - setupLog.Info("Starting remote configurator") - applierController.RunLoop(signalsContext) + // TODO: Remove once the feature is ready for go-live + enableConfigurator := os.Getenv("ENABLE_REMOTE_CONFIGURATOR") + if enableConfigurator == "true" { + setupLog.Info("Starting remote configurator") + applierController.RunLoop(signalsContext) + } }() wg.Wait() From 061805da482e68620bf51617105cc6272094ab42 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 13:53:18 +0300 Subject: [PATCH 57/65] Add remote_configuration to docker build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 63bd8523..0f62efb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ COPY main.go main.go COPY api/ api/ COPY controllers/ controllers/ COPY cbcontainers/ cbcontainers/ +COPY remote_configuration remote_configuration/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go From 2f90e095aee305f52bdc39a4297c8b22b90e2a47 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 15:19:44 +0300 Subject: [PATCH 58/65] Fix not wrapped error --- remote_configuration/configurator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote_configuration/configurator.go b/remote_configuration/configurator.go index 772502e8..1cb9272b 100644 --- a/remote_configuration/configurator.go +++ b/remote_configuration/configurator.go @@ -158,7 +158,7 @@ func (configurator *Configurator) getPendingChange(ctx context.Context, apiGatew func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGateway ApiGateway, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) if err != nil { - return fmt.Errorf("failed to create configuration change validator") + return fmt.Errorf("failed to create configuration change validator; %w", err) } if err := validator.ValidateChange(change, cr); err != nil { return err From 064570dec7919bd9c9ed2c92ad1f2a7daecc4bb2 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 15:19:56 +0300 Subject: [PATCH 59/65] Fix missing generate run --- api/v1/zz_generated.deepcopy.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index fc4b3c86..4a7d3fff 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -333,6 +333,11 @@ func (in *CBContainersComponentsSettings) DeepCopyInto(out *CBContainersComponen *out = new(CBContainersProxySettings) (*in).DeepCopyInto(*out) } + if in.RemoteConfiguration != nil { + in, out := &in.RemoteConfiguration, &out.RemoteConfiguration + *out = new(CBContainersRemoteConfigurationSettings) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersComponentsSettings. @@ -732,6 +737,26 @@ func (in *CBContainersProxySettings) DeepCopy() *CBContainersProxySettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersRemoteConfigurationSettings) DeepCopyInto(out *CBContainersRemoteConfigurationSettings) { + *out = *in + if in.EnabledForAgent != nil { + in, out := &in.EnabledForAgent, &out.EnabledForAgent + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRemoteConfigurationSettings. +func (in *CBContainersRemoteConfigurationSettings) DeepCopy() *CBContainersRemoteConfigurationSettings { + if in == nil { + return nil + } + out := new(CBContainersRemoteConfigurationSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CBContainersRuntimeProtectionSpec) DeepCopyInto(out *CBContainersRuntimeProtectionSpec) { *out = *in From bffcf912e47604b6c70955a90c040eeee83f9b78 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 15:45:07 +0300 Subject: [PATCH 60/65] Fix sensors path and response --- cbcontainers/communication/gateway/api_gateway.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index 3d451adf..cf17dea2 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -174,7 +174,7 @@ func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) Sensors []models.SensorMetadata `json:"sensors"` } - url := gateway.baseUrl("/setup/sensors") + url := gateway.baseUrl("setup/sensors") resp, err := gateway.baseRequest(). SetResult(getSensorsResponse{}). Get(url) @@ -186,8 +186,8 @@ func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) return nil, fmt.Errorf("failed to get sensor metadata with status code (%d)", resp.StatusCode()) } - r, ok := resp.Result().(getSensorsResponse) - if !ok { + r, ok := resp.Result().(*getSensorsResponse) + if !ok || r == nil { return nil, fmt.Errorf("malformed sensor metadata response") } return r.Sensors, nil From 4a52a715260044fd49870fcaaa51f992390b161a Mon Sep 17 00:00:00 2001 From: ltsonov Date: Tue, 12 Sep 2023 15:46:50 +0300 Subject: [PATCH 61/65] Increase iteration sleep time --- remote_configuration/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remote_configuration/controller.go b/remote_configuration/controller.go index 10630395..15b70f2c 100644 --- a/remote_configuration/controller.go +++ b/remote_configuration/controller.go @@ -8,8 +8,8 @@ import ( ) const ( - sleepDuration = 20 * time.Second // TODO: Increase - maxRetries = 10 // 1024s or ~17 minutes at peak + sleepDuration = 2 * time.Minute + maxRetries = 10 // 1024s or ~17 minutes at peak ) type configurationApplier interface { From b9740fc462b04be981971a31c7a2b1a213dd5f41 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Thu, 14 Sep 2023 12:19:01 +0300 Subject: [PATCH 62/65] Fix helm indentation --- .../templates/operator.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 27652a5d..bc052549 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -6322,18 +6322,18 @@ spec: defaults. type: string type: object - remoteConfiguration: - description: RemoteConfiguration holds settings for the operator/agent's - feature to apply configuration changes via the Carbon black - console - properties: - enabledForAgent: - default: true - description: EnabledForAgent turns the feature to change - agent configuration remotely (as opposed to operator - configuration) - type: boolean - type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + default: true + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator + configuration) + type: boolean + type: object type: object type: object gateways: From ff1a33ba78c04af2964dd707d9d7f9e812102864 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Fri, 15 Sep 2023 08:38:35 +0300 Subject: [PATCH 63/65] Change the version reset to be more readable --- remote_configuration/change_applier.go | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/remote_configuration/change_applier.go b/remote_configuration/change_applier.go index c8c7c020..84c33064 100644 --- a/remote_configuration/change_applier.go +++ b/remote_configuration/change_applier.go @@ -7,24 +7,26 @@ import ( // ApplyConfigChangeToCR will modify CR according to the values in the configuration change provided func ApplyConfigChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { - resetVersion := func(ptrToField *string) { - if ptrToField != nil && *ptrToField != "" { - *ptrToField = "" - } - } - if change.AgentVersion != nil { cr.Spec.Version = *change.AgentVersion - resetVersion(&cr.Spec.Components.Basic.Monitor.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.Enforcer.Image.Tag) - resetVersion(&cr.Spec.Components.Basic.StateReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) - resetVersion(&cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) - resetVersion(&cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + // We do not set the tag to the version as that would make it harder to upgrade manually + // Instead, we reset any "custom" tags, which will fall back to the default (spec.Version) + images := []*cbcontainersv1.CBContainersImageSpec{ + &cr.Spec.Components.Basic.Monitor.Image, + &cr.Spec.Components.Basic.Enforcer.Image, + &cr.Spec.Components.Basic.StateReporter.Image, + &cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image, + &cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image, + &cr.Spec.Components.RuntimeProtection.Sensor.Image, + &cr.Spec.Components.RuntimeProtection.Resolver.Image, + } if cr.Spec.Components.Cndr != nil { - resetVersion(&cr.Spec.Components.Cndr.Sensor.Image.Tag) + images = append(images, &cr.Spec.Components.Cndr.Sensor.Image) + } + + for _, i := range images { + i.Tag = "" } } if change.EnableClusterScanning != nil { From b07e326e03c2fbefc674ebac31aec5062655cebf Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 20 Sep 2023 11:04:31 +0300 Subject: [PATCH 64/65] Move the remote_configuration under cbcontainers and controllers --- Dockerfile | 1 - .../remote_configuration}/change_applier.go | 0 .../remote_configuration}/change_applier_test.go | 2 +- .../remote_configuration}/configurator.go | 0 .../remote_configuration}/configurator_test.go | 14 +++++++------- .../remote_configuration}/mocks/generated.go | 0 .../mocks/mock_access_token_provider.go | 0 .../mocks/mock_api_gateway.go | 0 .../remote_configuration}/validation.go | 0 .../remote_configuration}/validation_test.go | 4 ++-- .../remote_configuration_controller.go | 2 +- main.go | 4 ++-- 12 files changed, 13 insertions(+), 14 deletions(-) rename {remote_configuration => cbcontainers/remote_configuration}/change_applier.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/change_applier_test.go (99%) rename {remote_configuration => cbcontainers/remote_configuration}/configurator.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/configurator_test.go (96%) rename {remote_configuration => cbcontainers/remote_configuration}/mocks/generated.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/mocks/mock_access_token_provider.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/mocks/mock_api_gateway.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/validation.go (100%) rename {remote_configuration => cbcontainers/remote_configuration}/validation_test.go (98%) rename remote_configuration/controller.go => controllers/remote_configuration_controller.go (98%) diff --git a/Dockerfile b/Dockerfile index 0f62efb0..63bd8523 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ COPY main.go main.go COPY api/ api/ COPY controllers/ controllers/ COPY cbcontainers/ cbcontainers/ -COPY remote_configuration remote_configuration/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go diff --git a/remote_configuration/change_applier.go b/cbcontainers/remote_configuration/change_applier.go similarity index 100% rename from remote_configuration/change_applier.go rename to cbcontainers/remote_configuration/change_applier.go diff --git a/remote_configuration/change_applier_test.go b/cbcontainers/remote_configuration/change_applier_test.go similarity index 99% rename from remote_configuration/change_applier_test.go rename to cbcontainers/remote_configuration/change_applier_test.go index 52bf5d14..25a7e150 100644 --- a/remote_configuration/change_applier_test.go +++ b/cbcontainers/remote_configuration/change_applier_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" - "github.com/vmware/cbcontainers-operator/remote_configuration" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" "math/rand" "strconv" "testing" diff --git a/remote_configuration/configurator.go b/cbcontainers/remote_configuration/configurator.go similarity index 100% rename from remote_configuration/configurator.go rename to cbcontainers/remote_configuration/configurator.go diff --git a/remote_configuration/configurator_test.go b/cbcontainers/remote_configuration/configurator_test.go similarity index 96% rename from remote_configuration/configurator_test.go rename to cbcontainers/remote_configuration/configurator_test.go index b336bb7e..b4127102 100644 --- a/remote_configuration/configurator_test.go +++ b/cbcontainers/remote_configuration/configurator_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration/mocks" k8sMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" - "github.com/vmware/cbcontainers-operator/remote_configuration" - mocksConfigurator "github.com/vmware/cbcontainers-operator/remote_configuration/mocks" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" "time" @@ -19,8 +19,8 @@ import ( type configuratorMocks struct { k8sClient *k8sMocks.MockClient - apiGateway *mocksConfigurator.MockApiGateway - accessTokenProvider *mocksConfigurator.MockAccessTokenProvider + apiGateway *mocks.MockApiGateway + accessTokenProvider *mocks.MockAccessTokenProvider stubAccessToken string stubOperatorVersion string @@ -31,8 +31,8 @@ type configuratorMocks struct { // setupConfigurator sets up mocks and creates a Configurator instance with those mocks and some dummy data func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { k8sClient := k8sMocks.NewMockClient(ctrl) - apiGateway := mocksConfigurator.NewMockApiGateway(ctrl) - accessTokenProvider := mocksConfigurator.NewMockAccessTokenProvider(ctrl) + apiGateway := mocks.NewMockApiGateway(ctrl) + accessTokenProvider := mocks.NewMockAccessTokenProvider(ctrl) var mockAPIProvider remote_configuration.ApiCreator = func( cbContainersCluster *cbcontainersv1.CBContainersAgent, @@ -373,7 +373,7 @@ func setupUpdateCRMock(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbc }) } -func setupValidCompatibilityData(mockGateway *mocksConfigurator.MockApiGateway, sensorVersion, operatorVersion string) { +func setupValidCompatibilityData(mockGateway *mocks.MockApiGateway, sensorVersion, operatorVersion string) { mockGateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{{ Version: sensorVersion, SupportsRuntime: true, diff --git a/remote_configuration/mocks/generated.go b/cbcontainers/remote_configuration/mocks/generated.go similarity index 100% rename from remote_configuration/mocks/generated.go rename to cbcontainers/remote_configuration/mocks/generated.go diff --git a/remote_configuration/mocks/mock_access_token_provider.go b/cbcontainers/remote_configuration/mocks/mock_access_token_provider.go similarity index 100% rename from remote_configuration/mocks/mock_access_token_provider.go rename to cbcontainers/remote_configuration/mocks/mock_access_token_provider.go diff --git a/remote_configuration/mocks/mock_api_gateway.go b/cbcontainers/remote_configuration/mocks/mock_api_gateway.go similarity index 100% rename from remote_configuration/mocks/mock_api_gateway.go rename to cbcontainers/remote_configuration/mocks/mock_api_gateway.go diff --git a/remote_configuration/validation.go b/cbcontainers/remote_configuration/validation.go similarity index 100% rename from remote_configuration/validation.go rename to cbcontainers/remote_configuration/validation.go diff --git a/remote_configuration/validation_test.go b/cbcontainers/remote_configuration/validation_test.go similarity index 98% rename from remote_configuration/validation_test.go rename to cbcontainers/remote_configuration/validation_test.go index bfce3e46..fbed9294 100644 --- a/remote_configuration/validation_test.go +++ b/cbcontainers/remote_configuration/validation_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" - "github.com/vmware/cbcontainers-operator/remote_configuration" - "github.com/vmware/cbcontainers-operator/remote_configuration/mocks" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration/mocks" "testing" ) diff --git a/remote_configuration/controller.go b/controllers/remote_configuration_controller.go similarity index 98% rename from remote_configuration/controller.go rename to controllers/remote_configuration_controller.go index 15b70f2c..4a9d70bc 100644 --- a/remote_configuration/controller.go +++ b/controllers/remote_configuration_controller.go @@ -1,4 +1,4 @@ -package remote_configuration +package controllers import ( "context" diff --git a/main.go b/main.go index 5a5bb834..4a722555 100644 --- a/main.go +++ b/main.go @@ -22,12 +22,12 @@ import ( "flag" "fmt" "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" "github.com/vmware/cbcontainers-operator/cbcontainers/state" "github.com/vmware/cbcontainers-operator/cbcontainers/state/agent_applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/state/operator" - "github.com/vmware/cbcontainers-operator/remote_configuration" "go.uber.org/zap/zapcore" coreV1 "k8s.io/api/core/v1" "os" @@ -196,7 +196,7 @@ func main() { operatorNamespace, clusterIdentifier, ) - applierController := remote_configuration.NewRemoteConfigurationController(applier, log) + applierController := controllers.NewRemoteConfigurationController(applier, log) var wg sync.WaitGroup wg.Add(2) From 804772210aaea0043a797d958d99a4e7ffb3d2d6 Mon Sep 17 00:00:00 2001 From: ltsonov Date: Wed, 20 Sep 2023 11:05:47 +0300 Subject: [PATCH 65/65] Added godoc --- cbcontainers/communication/gateway/api_gateway.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index cf17dea2..cd4527c0 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -193,6 +193,7 @@ func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) return r.Sensors, nil } +// GetConfigurationChanges returns a list of configuration changes for the cluster func (gateway *ApiGateway) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { // TODO: Real implementation with CNS-2790 c := randomRemoteConfigChange() @@ -203,6 +204,7 @@ func (gateway *ApiGateway) GetConfigurationChanges(ctx context.Context, clusterI return nil, nil } +// UpdateConfigurationChangeStatus either acknowledges a remote configuration change applied to the cluster or marks the attempt as a failure func (gateway *ApiGateway) UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error { // TODO: Real implementation with CNS-2790