From 82c2bffc13603c22f7ac1cb5c381c846c8d4752d Mon Sep 17 00:00:00 2001 From: Lachezar Tsonov <80523253+ltsonov-cb@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:33:23 +0300 Subject: [PATCH] CNS-2791: Add a remote configurator that pulls changes from CB Console and applies them to the CR (#184) * Update README.md * Update README.md * Fix charts for main to be idetical to latest release v6.0.1 * Update README.md * Adding labels field to charts * Adding company code secret template to charts * Adding company code secret template to charts fix * Add documentation for secrets with helm * config applier skeleton * More work on the configuration applier loop. * Happy path test for config_applier * Pass context to API functions as they'll do IO * More tests * Send Failed status to backend when failing to list or update CR * Treat missing CR when a change is pending as error * Move "scheduling" logic to a separate struct for config applier * Use a DummyAPI implementation * Small refactoring * Add test for feature toggles * Add tests for the version field. Split the modification logic in a separate helper function to make testing easier * Sort the change slice in case there are multiple pending changes * Clearing TODOs, small fixes * Removing more TODOs * Turn off configurator by default and use env var to enable * Renaming things * Refactor tests a bit and remove more TODOs * Change the configurator to read the CR first , so it can extract the cluster name * Validate change against sensor capabilities * Validate operator and agent version compatibility * Tests for positive validation path * Bring the Validate and Apply change funcs under 1 struct and validate before applying * Adding change validation to the configuration * Change validator interface and add fetcher to download metadata from the API * Add auth_provider and adjust test to match * Create AccessTokenProvider in main.go * Extract the ApiGateway from processors.* and use it in configurator as well. * Go back to the previous version as it looked simpler * Fixing some tests and main.go * Change the agent_processor tests to match the new func. Remove a test that was not deterministic * Add clusterIdentifier parameter to GetConfigurationChanges * Add clusterID to the status update model call * Add secret detection to the changer * Add secret detection to the validation * Minor TODOs * Some missing test cases for validation * More minor changes * Add sensor metadata implementation. Add dummy config changes implementation. * Move data classes around * Removing TODOs * Change the feature toggle to be part of the CR * Change errors to have system info and user info for debugging vs user context in the UI * Small refactor * Change dummy struct to just a func * Bump timeout a bit * Remove controller_test.go * Add back the env var during development until the feature is completed (so it is not ON by default when testing) * Add remote_configuration to docker build * Fix not wrapped error * Fix missing generate run * Fix sensors path and response * Increase iteration sleep time * Fix helm indentation * Change the version reset to be more readable * Move the remote_configuration under cbcontainers and controllers * Added godoc --------- Co-authored-by: BenRub Co-authored-by: benrub --- api/v1/cbcontainersagent_types.go | 11 + api/v1/zz_generated.deepcopy.go | 25 ++ .../communication/gateway/api_gateway.go | 45 ++ .../gateway}/default_gateway_creator.go | 7 +- .../gateway/dummy_configuration_data.go | 63 +++ cbcontainers/models/operator_compatibility.go | 2 +- .../models/remote_configuration_changes.go | 40 ++ cbcontainers/models/sensor_metadata.go | 10 + cbcontainers/processors/agent_processor.go | 8 +- .../processors/agent_processor_test.go | 57 ++- cbcontainers/processors/mocks/generated.go | 1 - .../mocks/mock_api_gateway_creator.go | 51 --- .../remote_configuration/change_applier.go | 50 +++ .../change_applier_test.go | 284 +++++++++++++ .../remote_configuration/configurator.go | 204 +++++++++ .../remote_configuration/configurator_test.go | 389 ++++++++++++++++++ .../remote_configuration/mocks/generated.go | 4 + .../mocks/mock_access_token_provider.go | 51 +++ .../mocks/mock_api_gateway.go | 95 +++++ .../remote_configuration/validation.go | 108 +++++ .../remote_configuration/validation_test.go | 332 +++++++++++++++ cbcontainers/state/operator/auth_provider.go | 43 ++ .../templates/operator.yaml | 12 + ...ers.carbonblack.io_cbcontainersagents.yaml | 12 + ...ers.carbonblack.io_cbcontainersagents.yaml | 10 + controllers/cbcontainersagent_controller.go | 29 +- .../cbcontainersagent_controller_test.go | 96 ++--- controllers/mocks/generated.go | 3 +- .../mocks/mock_access_token_provider.go | 51 +++ controllers/mocks/mock_agent_processor.go | 51 +++ controllers/mocks/mock_cluster_processor.go | 51 --- .../remote_configuration_controller.go | 79 ++++ docs/crds.md | 7 +- main.go | 79 +++- 34 files changed, 2142 insertions(+), 218 deletions(-) rename cbcontainers/{processors => communication/gateway}/default_gateway_creator.go (78%) create mode 100644 cbcontainers/communication/gateway/dummy_configuration_data.go create mode 100644 cbcontainers/models/remote_configuration_changes.go create mode 100644 cbcontainers/models/sensor_metadata.go delete mode 100644 cbcontainers/processors/mocks/mock_api_gateway_creator.go create mode 100644 cbcontainers/remote_configuration/change_applier.go create mode 100644 cbcontainers/remote_configuration/change_applier_test.go create mode 100644 cbcontainers/remote_configuration/configurator.go create mode 100644 cbcontainers/remote_configuration/configurator_test.go create mode 100644 cbcontainers/remote_configuration/mocks/generated.go create mode 100644 cbcontainers/remote_configuration/mocks/mock_access_token_provider.go create mode 100644 cbcontainers/remote_configuration/mocks/mock_api_gateway.go create mode 100644 cbcontainers/remote_configuration/validation.go create mode 100644 cbcontainers/remote_configuration/validation_test.go create mode 100644 cbcontainers/state/operator/auth_provider.go create mode 100644 controllers/mocks/mock_access_token_provider.go create mode 100644 controllers/mocks/mock_agent_processor.go delete mode 100644 controllers/mocks/mock_cluster_processor.go create mode 100644 controllers/remote_configuration_controller.go diff --git a/api/v1/cbcontainersagent_types.go b/api/v1/cbcontainersagent_types.go index 2ac3d9a8..24727d3c 100644 --- a/api/v1/cbcontainersagent_types.go +++ b/api/v1/cbcontainersagent_types.go @@ -83,6 +83,9 @@ type CBContainersComponentsSettings struct { // care of determining the necessary `NO_PROXY` settings. // Proxy *CBContainersProxySettings `json:"proxy,omitempty"` + + // RemoteConfiguration holds settings for the operator/agent's feature to apply configuration changes via the Carbon black console + RemoteConfiguration *CBContainersRemoteConfigurationSettings `json:"remoteConfiguration,omitempty"` } func (s CBContainersComponentsSettings) ShouldCreateDefaultImagePullSecrets() bool { @@ -126,6 +129,14 @@ type CBContainersProxySettings struct { NoProxySuffix *string `json:"noProxySuffix,omitempty"` } +// CBContainersRemoteConfigurationSettings holds settings for the operator/agent's feature to apply configuration changes via the Carbon black console +type CBContainersRemoteConfigurationSettings struct { + // EnabledForAgent turns the feature to change agent configuration remotely (as opposed to operator configuration) + // + // +kubebuilder:default:=true + EnabledForAgent *bool `json:"enabledForAgent,omitempty"` +} + // CBContainersAgentStatus defines the observed state of CBContainersAgent type CBContainersAgentStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index fc4b3c86..4a7d3fff 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -333,6 +333,11 @@ func (in *CBContainersComponentsSettings) DeepCopyInto(out *CBContainersComponen *out = new(CBContainersProxySettings) (*in).DeepCopyInto(*out) } + if in.RemoteConfiguration != nil { + in, out := &in.RemoteConfiguration, &out.RemoteConfiguration + *out = new(CBContainersRemoteConfigurationSettings) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersComponentsSettings. @@ -732,6 +737,26 @@ func (in *CBContainersProxySettings) DeepCopy() *CBContainersProxySettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersRemoteConfigurationSettings) DeepCopyInto(out *CBContainersRemoteConfigurationSettings) { + *out = *in + if in.EnabledForAgent != nil { + in, out := &in.EnabledForAgent, &out.EnabledForAgent + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRemoteConfigurationSettings. +func (in *CBContainersRemoteConfigurationSettings) DeepCopy() *CBContainersRemoteConfigurationSettings { + if in == nil { + return nil + } + out := new(CBContainersRemoteConfigurationSettings) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CBContainersRuntimeProtectionSpec) DeepCopyInto(out *CBContainersRuntimeProtectionSpec) { *out = *in diff --git a/cbcontainers/communication/gateway/api_gateway.go b/cbcontainers/communication/gateway/api_gateway.go index 2eff48c1..cd4527c0 100644 --- a/cbcontainers/communication/gateway/api_gateway.go +++ b/cbcontainers/communication/gateway/api_gateway.go @@ -1,6 +1,7 @@ package gateway import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -16,6 +17,8 @@ var ( ErrGettingOperatorCompatibility = errors.New("error while getting the operator compatibility") ) +// TODO: Extract the cluster group + name + ID as separate struct identifying a cluster and used together + type ApiGateway struct { account string cluster string @@ -165,3 +168,45 @@ func (gateway *ApiGateway) GetCompatibilityMatrixEntryFor(operatorVersion string return r, nil } + +func (gateway *ApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) { + type getSensorsResponse struct { + Sensors []models.SensorMetadata `json:"sensors"` + } + + url := gateway.baseUrl("setup/sensors") + resp, err := gateway.baseRequest(). + SetResult(getSensorsResponse{}). + Get(url) + + if err != nil { + return nil, err + } + if !resp.IsSuccess() { + return nil, fmt.Errorf("failed to get sensor metadata with status code (%d)", resp.StatusCode()) + } + + r, ok := resp.Result().(*getSensorsResponse) + if !ok || r == nil { + return nil, fmt.Errorf("malformed sensor metadata response") + } + return r.Sensors, nil +} + +// GetConfigurationChanges returns a list of configuration changes for the cluster +func (gateway *ApiGateway) GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) { + // TODO: Real implementation with CNS-2790 + c := randomRemoteConfigChange() + if c != nil { + return []models.ConfigurationChange{*c}, nil + + } + return nil, nil +} + +// UpdateConfigurationChangeStatus either acknowledges a remote configuration change applied to the cluster or marks the attempt as a failure +func (gateway *ApiGateway) UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error { + // TODO: Real implementation with CNS-2790 + + return nil +} diff --git a/cbcontainers/processors/default_gateway_creator.go b/cbcontainers/communication/gateway/default_gateway_creator.go similarity index 78% rename from cbcontainers/processors/default_gateway_creator.go rename to cbcontainers/communication/gateway/default_gateway_creator.go index daa1e718..6e7c30e3 100644 --- a/cbcontainers/processors/default_gateway_creator.go +++ b/cbcontainers/communication/gateway/default_gateway_creator.go @@ -1,8 +1,7 @@ -package processors +package gateway import ( cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" ) type DefaultGatewayCreator struct { @@ -12,9 +11,9 @@ func NewDefaultGatewayCreator() *DefaultGatewayCreator { return &DefaultGatewayCreator{} } -func (creator *DefaultGatewayCreator) CreateGateway(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) { +func (creator *DefaultGatewayCreator) CreateGateway(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (*ApiGateway, error) { spec := cbContainersAgent.Spec - builder := gateway.NewBuilder(spec.Account, spec.ClusterName, accessToken, spec.Gateways.ApiGateway.Host, cbContainersAgent.ObjectMeta.Labels). + builder := NewBuilder(spec.Account, spec.ClusterName, accessToken, spec.Gateways.ApiGateway.Host, cbContainersAgent.ObjectMeta.Labels). SetURLComponents(spec.Gateways.ApiGateway.Scheme, spec.Gateways.ApiGateway.Port, spec.Gateways.ApiGateway.Adapter). SetTLSInsecureSkipVerify(spec.Gateways.GatewayTLS.InsecureSkipVerify). SetTLSRootCAsBundle(spec.Gateways.GatewayTLS.RootCAsBundle) diff --git a/cbcontainers/communication/gateway/dummy_configuration_data.go b/cbcontainers/communication/gateway/dummy_configuration_data.go new file mode 100644 index 00000000..62fc391b --- /dev/null +++ b/cbcontainers/communication/gateway/dummy_configuration_data.go @@ -0,0 +1,63 @@ +package gateway + +import ( + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "math/rand" + "strconv" +) + +// TODO: This will be removed once real APIs are implemented for this but it helps try the feature while in development +// API task - CNS-2790 + +var ( + tr = true + fal = false + dummyAgentVersions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} +) + +func randomRemoteConfigChange() *models.ConfigurationChange { + csRand, runtimeRand, cndrRand, versionRand, nilRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(dummyAgentVersions)), rand.Int() + + if nilRand%5 == 1 { + return nil + } + + changeVersion := &dummyAgentVersions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + var changeCNDR *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = &tr + case 2, 4: + changeClusterScanning = &fal + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = &tr + case 2, 4: + changeRuntime = &fal + default: + changeRuntime = nil + } + + if changeVersion != nil && *changeVersion == "3.0.0" && cndrRand%2 == 0 { + changeCNDR = &tr + } else { + changeCNDR = &fal + } + + return &models.ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + EnableCNDR: changeCNDR, + Status: models.ChangeStatusPending, + } +} diff --git a/cbcontainers/models/operator_compatibility.go b/cbcontainers/models/operator_compatibility.go index d0fcc52d..1509d929 100644 --- a/cbcontainers/models/operator_compatibility.go +++ b/cbcontainers/models/operator_compatibility.go @@ -21,6 +21,6 @@ func (c OperatorCompatibility) CheckCompatibility(agentVersion string) error { return fmt.Errorf("agent version too low, downgrade the operator to use that agent version: min is [%s], desired is [%s]", c.MinAgent, agentVersion) } - // if we are here it means the operator and the agent version are compatibile + // if we are here it means the operator and the agent version are compatible return nil } diff --git a/cbcontainers/models/remote_configuration_changes.go b/cbcontainers/models/remote_configuration_changes.go new file mode 100644 index 00000000..54603b6d --- /dev/null +++ b/cbcontainers/models/remote_configuration_changes.go @@ -0,0 +1,40 @@ +package models + +type RemoteChangeStatus string + +var ( + ChangeStatusPending RemoteChangeStatus = "PENDING" + ChangeStatusAcked RemoteChangeStatus = "ACKNOWLEDGED" + ChangeStatusFailed RemoteChangeStatus = "FAILED" +) + +type ConfigurationChange struct { + ID string `json:"id"` + Status RemoteChangeStatus `json:"status"` + AgentVersion *string `json:"agent_version"` + EnableClusterScanning *bool `json:"enable_cluster_scanning"` + EnableRuntime *bool `json:"enable_runtime"` + EnableCNDR *bool `json:"enable_cndr"` + EnableClusterScanningSecretDetection *bool `json:"enable_cluster_scanning_secret_detection"` + Timestamp string `json:"timestamp"` +} + +type ConfigurationChangeStatusUpdate struct { + ID string `json:"id"` + ClusterIdentifier string `json:"cluster_identifier"` + ClusterGroup string `json:"cluster_group"` + ClusterName string `json:"cluster_name"` + Status RemoteChangeStatus `json:"status"` + + // AppliedGeneration tracks the generation of the Custom resource where the change was applied + AppliedGeneration int64 `json:"applied_generation"` + // AppliedTimestamp records when the change was applied in RFC3339 format + AppliedTimestamp string `json:"applied_timestamp"` + + // Error should hold information about encountered errors when the change application failed. + // For system usage only, not meant for end-users. + Error string `json:"encountered_error"` + // ErrorReason should be populated if some additional information can be shown to the user (e.g. why a change was invalid) + // It should not be used to store system information + ErrorReason string `json:"error_reason"` +} diff --git a/cbcontainers/models/sensor_metadata.go b/cbcontainers/models/sensor_metadata.go new file mode 100644 index 00000000..be27f469 --- /dev/null +++ b/cbcontainers/models/sensor_metadata.go @@ -0,0 +1,10 @@ +package models + +type SensorMetadata struct { + Version string `json:"version"` + IsLatest bool `json:"is_latest" ` + SupportsRuntime bool `json:"supports_runtime"` + SupportsClusterScanning bool `json:"supports_cluster_scanning"` + SupportsClusterScanningSecrets bool `json:"supports_cluster_scanning_secrets"` + SupportsCndr bool `json:"supports_cndr"` +} diff --git a/cbcontainers/processors/agent_processor.go b/cbcontainers/processors/agent_processor.go index 5e8deaba..07fc6717 100644 --- a/cbcontainers/processors/agent_processor.go +++ b/cbcontainers/processors/agent_processor.go @@ -16,9 +16,7 @@ type APIGateway interface { GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) } -type APIGatewayCreator interface { - CreateGateway(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) -} +type APIGatewayCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (APIGateway, error) type OperatorVersionProvider interface { GetOperatorVersion() (string, error) @@ -79,7 +77,7 @@ func (processor *AgentProcessor) initializeIfNeeded(cbContainersCluster *cbconta } processor.log.Info("Initializing AgentProcessor components") - gateway, err := processor.gatewayCreator.CreateGateway(cbContainersCluster, accessToken) + gateway, err := processor.gatewayCreator(cbContainersCluster, accessToken) if err != nil { return err } @@ -118,7 +116,7 @@ func (processor *AgentProcessor) checkCompatibility(cbContainersAgent *cbcontain } return err } - gateway, err := processor.gatewayCreator.CreateGateway(cbContainersAgent, accessToken) + gateway, err := processor.gatewayCreator(cbContainersAgent, accessToken) if err != nil { processor.log.Error(err, "skipping compatibility check, error while building API gateway") // if there is an error while building the gateway log it and skip the check diff --git a/cbcontainers/processors/agent_processor_test.go b/cbcontainers/processors/agent_processor_test.go index cec857a1..b0c541ca 100644 --- a/cbcontainers/processors/agent_processor_test.go +++ b/cbcontainers/processors/agent_processor_test.go @@ -2,9 +2,9 @@ package processors_test import ( "fmt" + "github.com/go-logr/logr/testr" "testing" - logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" @@ -17,8 +17,9 @@ import ( type ClusterProcessorTestMocks struct { gatewayMock *mocks.MockAPIGateway - gatewayCreatorMock *mocks.MockAPIGatewayCreator operatorVersionProviderMock *mocks.MockOperatorVersionProvider + + mockGatewayCreatorFunc processors.APIGatewayCreator } type SetupAndAssertClusterProcessorTest func(*ClusterProcessorTestMocks, *processors.AgentProcessor) @@ -35,16 +36,22 @@ func testClusterProcessor(t *testing.T, setupAndAssert SetupAndAssertClusterProc mocksObjects := &ClusterProcessorTestMocks{ gatewayMock: mocks.NewMockAPIGateway(ctrl), - gatewayCreatorMock: mocks.NewMockAPIGatewayCreator(ctrl), operatorVersionProviderMock: mocks.NewMockOperatorVersionProvider(ctrl), } - processor := processors.NewAgentProcessor(logrTesting.NewTestLogger(t), mocksObjects.gatewayCreatorMock, mocksObjects.operatorVersionProviderMock, mockIdentifier) + // Proxy so tests can replace the actual implementation without creating a full mock + var mockCreator processors.APIGatewayCreator = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return mocksObjects.mockGatewayCreatorFunc(cbContainersCluster, accessToken) + } + + processor := processors.NewAgentProcessor(testr.New(t), mockCreator, mocksObjects.operatorVersionProviderMock, mockIdentifier) setupAndAssert(mocksObjects, processor) } func setupValidMocksCalls(testMocks *ClusterProcessorTestMocks, times int) { - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), AccessToken).Return(testMocks.gatewayMock, nil).Times(times) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().DoAndReturn(func() (*models.RegistrySecretValues, error) { return &models.RegistrySecretValues{Data: map[string][]byte{test_utils.RandomString(): {}}}, nil }).Times(times) @@ -86,10 +93,12 @@ func TestProcessorIsReCreatingComponentsForDifferentCR(t *testing.T) { }) } -func TestProcessorReturnsErrorWhenCanNotGetRegisterySecret(t *testing.T) { +func TestProcessorReturnsErrorWhenCanNotGetRegistrySecret(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(nil, fmt.Errorf("")) _, err := processor.Process(clusterCR, AccessToken) require.Error(t, err) @@ -99,7 +108,9 @@ func TestProcessorReturnsErrorWhenCanNotGetRegisterySecret(t *testing.T) { func TestProcessorReturnsErrorWhenCanNotRegisterCluster(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(fmt.Errorf("")) _, err := processor.Process(clusterCR, AccessToken) @@ -110,7 +121,9 @@ func TestProcessorReturnsErrorWhenCanNotRegisterCluster(t *testing.T) { func TestProcessorReturnsErrorWhenOperatorVersionProviderReturnsUnknownError(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil) testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", fmt.Errorf("intentional unknown error")) @@ -122,7 +135,9 @@ func TestProcessorReturnsErrorWhenOperatorVersionProviderReturnsUnknownError(t * func TestProcessorReturnsErrorWhenCanNotCreateGateway(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("")) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return nil, fmt.Errorf("") + } _, err := processor.Process(clusterCR, AccessToken) require.Error(t, err) }) @@ -139,18 +154,16 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", operator.ErrNotSemVer) }, }, - { - name: "when CreateGateway returns error", - setup: func(testMocks *ClusterProcessorTestMocks) { - testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("intentional error")) - }, - }, { name: "when GetCompatibilityMatrixEntryFor returns error", setup: func(testMocks *ClusterProcessorTestMocks) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return nil, fmt.Errorf("intentional error") + } + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetCompatibilityMatrixEntryFor(gomock.Any()).Return(nil, fmt.Errorf("intentional error")) }, }, @@ -158,7 +171,9 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { name: "when versions are compatible", setup: func(testMocks *ClusterProcessorTestMocks) { testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("1.0.0", nil) - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetCompatibilityMatrixEntryFor(gomock.Any()).Return(&models.OperatorCompatibility{ MinAgent: "0.9.0", MaxAgent: "1.1.0", @@ -171,7 +186,9 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { testClusterProcessor(t, func(testMocks *ClusterProcessorTestMocks, processor *processors.AgentProcessor) { clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "1.0.0", Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}} - testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil) + testMocks.mockGatewayCreatorFunc = func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return testMocks.gatewayMock, nil + } testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil) testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil) testCase.setup(testMocks) diff --git a/cbcontainers/processors/mocks/generated.go b/cbcontainers/processors/mocks/generated.go index 6d926540..dabf05c6 100644 --- a/cbcontainers/processors/mocks/generated.go +++ b/cbcontainers/processors/mocks/generated.go @@ -1,5 +1,4 @@ package mocks //go:generate mockgen -destination mock_api_gateway.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors APIGateway -//go:generate mockgen -destination mock_api_gateway_creator.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors APIGatewayCreator //go:generate mockgen -destination mock_operator_version_provider.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/processors OperatorVersionProvider diff --git a/cbcontainers/processors/mocks/mock_api_gateway_creator.go b/cbcontainers/processors/mocks/mock_api_gateway_creator.go deleted file mode 100644 index c0b611f6..00000000 --- a/cbcontainers/processors/mocks/mock_api_gateway_creator.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/cbcontainers/processors (interfaces: APIGatewayCreator) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - processors "github.com/vmware/cbcontainers-operator/cbcontainers/processors" -) - -// MockAPIGatewayCreator is a mock of APIGatewayCreator interface. -type MockAPIGatewayCreator struct { - ctrl *gomock.Controller - recorder *MockAPIGatewayCreatorMockRecorder -} - -// MockAPIGatewayCreatorMockRecorder is the mock recorder for MockAPIGatewayCreator. -type MockAPIGatewayCreatorMockRecorder struct { - mock *MockAPIGatewayCreator -} - -// NewMockAPIGatewayCreator creates a new mock instance. -func NewMockAPIGatewayCreator(ctrl *gomock.Controller) *MockAPIGatewayCreator { - mock := &MockAPIGatewayCreator{ctrl: ctrl} - mock.recorder = &MockAPIGatewayCreatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAPIGatewayCreator) EXPECT() *MockAPIGatewayCreatorMockRecorder { - return m.recorder -} - -// CreateGateway mocks base method. -func (m *MockAPIGatewayCreator) CreateGateway(arg0 *v1.CBContainersAgent, arg1 string) (processors.APIGateway, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateGateway", arg0, arg1) - ret0, _ := ret[0].(processors.APIGateway) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateGateway indicates an expected call of CreateGateway. -func (mr *MockAPIGatewayCreatorMockRecorder) CreateGateway(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateGateway", reflect.TypeOf((*MockAPIGatewayCreator)(nil).CreateGateway), arg0, arg1) -} diff --git a/cbcontainers/remote_configuration/change_applier.go b/cbcontainers/remote_configuration/change_applier.go new file mode 100644 index 00000000..84c33064 --- /dev/null +++ b/cbcontainers/remote_configuration/change_applier.go @@ -0,0 +1,50 @@ +package remote_configuration + +import ( + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// ApplyConfigChangeToCR will modify CR according to the values in the configuration change provided +func ApplyConfigChangeToCR(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) { + if change.AgentVersion != nil { + cr.Spec.Version = *change.AgentVersion + + // We do not set the tag to the version as that would make it harder to upgrade manually + // Instead, we reset any "custom" tags, which will fall back to the default (spec.Version) + images := []*cbcontainersv1.CBContainersImageSpec{ + &cr.Spec.Components.Basic.Monitor.Image, + &cr.Spec.Components.Basic.Enforcer.Image, + &cr.Spec.Components.Basic.StateReporter.Image, + &cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image, + &cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image, + &cr.Spec.Components.RuntimeProtection.Sensor.Image, + &cr.Spec.Components.RuntimeProtection.Resolver.Image, + } + if cr.Spec.Components.Cndr != nil { + images = append(images, &cr.Spec.Components.Cndr.Sensor.Image) + } + + for _, i := range images { + i.Tag = "" + } + } + if change.EnableClusterScanning != nil { + cr.Spec.Components.ClusterScanning.Enabled = change.EnableClusterScanning + } + + if change.EnableClusterScanningSecretDetection != nil { + cr.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection = *change.EnableClusterScanningSecretDetection + } + + if change.EnableRuntime != nil { + cr.Spec.Components.RuntimeProtection.Enabled = change.EnableRuntime + } + + if change.EnableCNDR != nil { + if cr.Spec.Components.Cndr == nil { + cr.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + cr.Spec.Components.Cndr.Enabled = change.EnableCNDR + } +} diff --git a/cbcontainers/remote_configuration/change_applier_test.go b/cbcontainers/remote_configuration/change_applier_test.go new file mode 100644 index 00000000..25a7e150 --- /dev/null +++ b/cbcontainers/remote_configuration/change_applier_test.go @@ -0,0 +1,284 @@ +package remote_configuration_test + +import ( + "fmt" + "github.com/stretchr/testify/assert" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" + "math/rand" + "strconv" + "testing" +) + +func TestFeatureTogglesAreAppliedCorrectly(t *testing.T) { + type appliedChangeTest struct { + name string + change models.ConfigurationChange + initialCR cbcontainersv1.CBContainersAgent + assertFinalCR func(*testing.T, cbcontainersv1.CBContainersAgent) + } + + crVersion := "1.2.3" + + // generateFeatureToggleTestCases produces a set of tests for a single feature toggle in the requested change + // The tests validate if each toggle state (true, false, nil) is applied correctly or ignored when it's not needed against the CR's state (true, false, nil) + generateFeatureToggleTestCases := + func(feature string, + changeFieldChanger func(*models.ConfigurationChange, *bool), + crFieldChanger func(*cbcontainersv1.CBContainersAgent, *bool), + crAsserter func(*testing.T, cbcontainersv1.CBContainersAgent, *bool)) []appliedChangeTest { + + var result []appliedChangeTest + + for _, crState := range []*bool{truePtr, falsePtr, nil} { + crState := crState // Avoid closure issues + + // Validate that each toggle state works (or doesn't do anything when it matches) + for _, changeState := range []*bool{falsePtr, truePtr} { + changeState := changeState // Avoid closure issues + + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} + crFieldChanger(&cr, crState) + change := models.ConfigurationChange{} + changeFieldChanger(&change, changeState) + + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("toggle feature (%s) from (%v) to (%v)", feature, prettyPrintBoolPtr(crState), prettyPrintBoolPtr(changeState)), + change: change, + initialCR: cr, + assertFinalCR: func(t *testing.T, agent cbcontainersv1.CBContainersAgent) { + crAsserter(t, agent, changeState) + }, + }) + } + + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: crVersion}} + crFieldChanger(&cr, crState) + // Validate that a change with the toggle unset does not modify the CR + result = append(result, appliedChangeTest{ + name: fmt.Sprintf("missing toggle feature (%s) with CR state (%v)", feature, prettyPrintBoolPtr(crState)), + change: models.ConfigurationChange{}, + initialCR: cr, + assertFinalCR: func(t *testing.T, agent cbcontainersv1.CBContainersAgent) { + crAsserter(t, agent, crState) + }, + }) + } + + return result + } + + var testCases []appliedChangeTest + + clusterScannerToggleTestCases := generateFeatureToggleTestCases("cluster scanning", + func(change *models.ConfigurationChange, val *bool) { + change.EnableClusterScanning = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + agent.Spec.Components.ClusterScanning.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.ClusterScanning.Enabled) + }) + + secretDetectionToggleTestCases := generateFeatureToggleTestCases("cluster scanning secret detection", + func(change *models.ConfigurationChange, val *bool) { + change.EnableClusterScanningSecretDetection = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + if val == nil { + // Bail out, this value is not valid for the flag + return + } + agent.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection = *val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + if b == nil { + // Bail out, this value is not valid for the flag + return + } + assert.Equal(t, *b, agent.Spec.Components.ClusterScanning.ClusterScannerAgent.CLIFlags.EnableSecretDetection) + }) + + runtimeToggleTestCases := generateFeatureToggleTestCases("runtime protection", + func(change *models.ConfigurationChange, val *bool) { + change.EnableRuntime = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + agent.Spec.Components.RuntimeProtection.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.RuntimeProtection.Enabled) + }) + + cndrToggleTestCases := generateFeatureToggleTestCases("CNDR", + func(change *models.ConfigurationChange, val *bool) { + change.EnableCNDR = val + }, func(agent *cbcontainersv1.CBContainersAgent, val *bool) { + if agent.Spec.Components.Cndr == nil { + agent.Spec.Components.Cndr = &cbcontainersv1.CBContainersCndrSpec{} + } + agent.Spec.Components.Cndr.Enabled = val + }, func(t *testing.T, agent cbcontainersv1.CBContainersAgent, b *bool) { + assert.Equal(t, b, agent.Spec.Components.Cndr.Enabled) + }) + + testCases = append(testCases, clusterScannerToggleTestCases...) + testCases = append(testCases, secretDetectionToggleTestCases...) + testCases = append(testCases, runtimeToggleTestCases...) + testCases = append(testCases, cndrToggleTestCases...) + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + remote_configuration.ApplyConfigChangeToCR(testCase.change, &testCase.initialCR) + testCase.assertFinalCR(t, testCase.initialCR) + }) + } +} + +func TestVersionIsAppliedCorrectly(t *testing.T) { + originalVersion := "my-version-42" + newVersion := "new-version" + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} + change := models.ConfigurationChange{AgentVersion: &newVersion} + + remote_configuration.ApplyConfigChangeToCR(change, &cr) + assert.Equal(t, newVersion, cr.Spec.Version) +} + +func TestMissingVersionDoesNotModifyCR(t *testing.T) { + originalVersion := "my-version-42" + cr := cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: originalVersion}} + change := models.ConfigurationChange{AgentVersion: nil, EnableRuntime: truePtr} + + remote_configuration.ApplyConfigChangeToCR(change, &cr) + assert.Equal(t, originalVersion, cr.Spec.Version) + +} + +func TestVersionOverwritesCustomTagsByRemovingThem(t *testing.T) { + cr := cbcontainersv1.CBContainersAgent{ + Spec: cbcontainersv1.CBContainersAgentSpec{ + Version: "some-version", + Components: cbcontainersv1.CBContainersComponentsSpec{ + Basic: cbcontainersv1.CBContainersBasicSpec{ + Enforcer: cbcontainersv1.CBContainersEnforcerSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-enforcer", + }, + }, + StateReporter: cbcontainersv1.CBContainersStateReporterSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-state-repoter", + }, + }, + Monitor: cbcontainersv1.CBContainersMonitorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-monitor", + }, + }, + }, + RuntimeProtection: cbcontainersv1.CBContainersRuntimeProtectionSpec{ + Resolver: cbcontainersv1.CBContainersRuntimeResolverSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-runtime-resolver", + }, + }, + Sensor: cbcontainersv1.CBContainersRuntimeSensorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-runtime-sensor", + }, + }, + }, + Cndr: &cbcontainersv1.CBContainersCndrSpec{ + Sensor: cbcontainersv1.CBContainersCndrSensorSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-cndr-sensor", + }, + }, + }, + ClusterScanning: cbcontainersv1.CBContainersClusterScanningSpec{ + ClusterScannerAgent: cbcontainersv1.CBContainersClusterScannerAgentSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-cluster-scanning-agent", + }, + }, + ImageScanningReporter: cbcontainersv1.CBContainersImageScanningReporterSpec{ + Image: cbcontainersv1.CBContainersImageSpec{ + Tag: "custom-image-scanning-reporter", + }, + }, + }, + }, + }, + } + + newVersion := "new-version" + change := models.ConfigurationChange{AgentVersion: &newVersion} + + remote_configuration.ApplyConfigChangeToCR(change, &cr) + assert.Equal(t, newVersion, cr.Spec.Version) + // To avoid keeping "custom" tags forever, the apply change should instead reset all such fields + // => the operator will use the common version instead + assert.Empty(t, cr.Spec.Components.Basic.Monitor.Image.Tag) + assert.Empty(t, cr.Spec.Components.Basic.Enforcer.Image.Tag) + assert.Empty(t, cr.Spec.Components.Basic.StateReporter.Image.Tag) + assert.Empty(t, cr.Spec.Components.ClusterScanning.ImageScanningReporter.Image.Tag) + assert.Empty(t, cr.Spec.Components.ClusterScanning.ClusterScannerAgent.Image.Tag) + assert.Empty(t, cr.Spec.Components.RuntimeProtection.Sensor.Image.Tag) + assert.Empty(t, cr.Spec.Components.RuntimeProtection.Resolver.Image.Tag) + assert.Empty(t, cr.Spec.Components.Cndr.Sensor.Image.Tag) +} + +func prettyPrintBoolPtr(v *bool) string { + if v == nil { + return "" + } + return fmt.Sprintf("%t", *v) +} + +// randomPendingConfigChange creates a non-empty configuration change with randomly populated fields in pending state +// the change is not guaranteed to be 100% valid +func randomPendingConfigChange() models.ConfigurationChange { + var versions = []string{"2.12.1", "2.10.0", "2.12.0", "2.11.0", "3.0.0"} + + csRand, runtimeRand, cndrRand, versionRand := rand.Int(), rand.Int(), rand.Int(), rand.Intn(len(versions)) + + changeVersion := &versions[versionRand] + + var changeClusterScanning *bool + var changeRuntime *bool + var changeCNDR *bool + + switch csRand % 5 { + case 1, 3: + changeClusterScanning = truePtr + case 2, 4: + changeClusterScanning = falsePtr + default: + changeClusterScanning = nil + } + + switch runtimeRand % 5 { + case 1, 3: + changeRuntime = truePtr + case 2, 4: + changeRuntime = falsePtr + default: + changeRuntime = nil + } + + switch cndrRand % 5 { + case 1, 3: + changeCNDR = truePtr + case 2, 4: + changeCNDR = falsePtr + default: + changeCNDR = nil + } + + return models.ConfigurationChange{ + ID: strconv.Itoa(rand.Int()), + AgentVersion: changeVersion, + EnableClusterScanning: changeClusterScanning, + EnableRuntime: changeRuntime, + EnableCNDR: changeCNDR, + Status: models.ChangeStatusPending, + } +} diff --git a/cbcontainers/remote_configuration/configurator.go b/cbcontainers/remote_configuration/configurator.go new file mode 100644 index 00000000..1cb9272b --- /dev/null +++ b/cbcontainers/remote_configuration/configurator.go @@ -0,0 +1,204 @@ +package remote_configuration + +import ( + "context" + "errors" + "fmt" + "github.com/go-logr/logr" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "sigs.k8s.io/controller-runtime/pkg/client" + "sort" + "time" +) + +const ( + timeoutSingleIteration = time.Second * 120 +) + +type ApiGateway interface { + GetSensorMetadata() ([]models.SensorMetadata, error) + GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error) + + GetConfigurationChanges(ctx context.Context, clusterIdentifier string) ([]models.ConfigurationChange, error) + UpdateConfigurationChangeStatus(context.Context, models.ConfigurationChangeStatusUpdate) error +} + +type AccessTokenProvider interface { + GetCBAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, deployedNamespace string) (string, error) +} + +type ApiCreator func(cbContainersCluster *cbcontainersv1.CBContainersAgent, accessToken string) (ApiGateway, error) + +type Configurator struct { + k8sClient client.Client + logger logr.Logger + accessTokenProvider AccessTokenProvider + apiCreator ApiCreator + operatorVersion string + deployedNamespace string + clusterIdentifier string +} + +func NewConfigurator( + k8sClient client.Client, + gatewayCreator ApiCreator, + logger logr.Logger, + accessTokenProvider AccessTokenProvider, + operatorVersion string, + deployedNamespace string, + clusterIdentifier string, +) *Configurator { + return &Configurator{ + k8sClient: k8sClient, + logger: logger, + apiCreator: gatewayCreator, + accessTokenProvider: accessTokenProvider, + operatorVersion: operatorVersion, + deployedNamespace: deployedNamespace, + clusterIdentifier: clusterIdentifier, + } +} + +func (configurator *Configurator) RunIteration(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, timeoutSingleIteration) + defer cancel() + + configurator.logger.Info("Checking for installed agent...") + cr, err := configurator.getCR(ctx) + if err != nil { + configurator.logger.Error(err, "Failed to get CBContainerAgent resource, cannot continue") + return err + } + if cr == nil { + configurator.logger.Info("No CBContainerAgent installed, there is nothing to configure") + return nil + } + + if remoteConfigSettings := cr.Spec.Components.Settings.RemoteConfiguration; remoteConfigSettings != nil && + remoteConfigSettings.EnabledForAgent != nil && + *remoteConfigSettings.EnabledForAgent == false { + configurator.logger.Info("Remote configuration feature is disabled, no changes will be made") + return nil + + } + apiGateway, err := configurator.createAPIGateway(ctx, cr) + if err != nil { + configurator.logger.Error(err, "Failed to create a valid CB API Gateway, cannot continue") + return err + } + + configurator.logger.Info("Checking for pending remote configuration changes...") + change, errGettingChanges := configurator.getPendingChange(ctx, apiGateway) + if errGettingChanges != nil { + configurator.logger.Error(errGettingChanges, "Failed to get pending configuration changes") + return errGettingChanges + } + + if change == nil { + configurator.logger.Info("No pending remote configuration changes found") + return nil + } + + configurator.logger.Info("Applying remote configuration change to CBContainerAgent resource", "change", change) + errApplyingCR := configurator.applyChangeToCR(ctx, apiGateway, *change, cr) + if errApplyingCR != nil { + configurator.logger.Error(errApplyingCR, "Failed to apply configuration changes to CBContainerAGent resource") + // Intentional fallthrough as we want to report the change application as failed to the backend + } else { + configurator.logger.Info("Successfully applied configuration changes to CBContainerAgent resource") + } + + if err := configurator.updateChangeStatus(ctx, apiGateway, *change, cr, errApplyingCR); err != nil { + configurator.logger.Error(err, "Failed to update the status of a configuration change; it might be re-applied again in the future") + return err + } + + // If we failed to apply the CR, we report this to the backend but want to return the apply error here to indicate a failure + return errApplyingCR +} + +// getCR loads exactly 0 or 1 CBContainersAgent definitions +// if no resource is defined, nil is returned +// in case more than 1 resource is defined (which is not generally supported), only the first one is returned +func (configurator *Configurator) getCR(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { + // keep implementation in-sync with CBContainersAgentController.getContainersAgentObject() to ensure both operate on the same agent instance + + cbContainersAgentsList := &cbcontainersv1.CBContainersAgentList{} + if err := configurator.k8sClient.List(ctx, cbContainersAgentsList); err != nil { + return nil, fmt.Errorf("couldn't list CBContainersAgent k8s objects: %w", err) + } + + if len(cbContainersAgentsList.Items) == 0 { + return nil, nil + } + + // We don't log a warning if len >=2 as the controller already warns users about that + return &cbContainersAgentsList.Items[0], nil +} + +func (configurator *Configurator) getPendingChange(ctx context.Context, apiGateway ApiGateway) (*models.ConfigurationChange, error) { + changes, err := apiGateway.GetConfigurationChanges(ctx, configurator.clusterIdentifier) + if err != nil { + return nil, err + } + + sort.SliceStable(changes, func(i, j int) bool { + return changes[i].Timestamp < changes[j].Timestamp + }) + + for _, change := range changes { + if change.Status == models.ChangeStatusPending { + return &change, nil + } + } + return nil, nil +} + +func (configurator *Configurator) applyChangeToCR(ctx context.Context, apiGateway ApiGateway, change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { + validator, err := NewConfigurationChangeValidator(configurator.operatorVersion, apiGateway) + if err != nil { + return fmt.Errorf("failed to create configuration change validator; %w", err) + } + if err := validator.ValidateChange(change, cr); err != nil { + return err + } + ApplyConfigChangeToCR(change, cr) + return configurator.k8sClient.Update(ctx, cr) +} + +func (configurator *Configurator) updateChangeStatus( + ctx context.Context, + apiGateway ApiGateway, + change models.ConfigurationChange, + cr *cbcontainersv1.CBContainersAgent, + encounteredError error, +) error { + statusUpdate := models.ConfigurationChangeStatusUpdate{ + ID: change.ID, + ClusterIdentifier: configurator.clusterIdentifier, + } + + if encounteredError == nil { + statusUpdate.Status = models.ChangeStatusAcked + statusUpdate.AppliedGeneration = cr.Generation + statusUpdate.AppliedTimestamp = time.Now().UTC().Format(time.RFC3339) + } else { + statusUpdate.Status = models.ChangeStatusFailed + statusUpdate.Error = encounteredError.Error() + // Validation change is the only thing we can safely give information to the user about + if errors.As(encounteredError, &invalidChangeError{}) { + statusUpdate.ErrorReason = encounteredError.Error() + } + } + + return apiGateway.UpdateConfigurationChangeStatus(ctx, statusUpdate) +} + +func (configurator *Configurator) createAPIGateway(ctx context.Context, cr *cbcontainersv1.CBContainersAgent) (ApiGateway, error) { + accessToken, err := configurator.accessTokenProvider.GetCBAccessToken(ctx, cr, configurator.deployedNamespace) + if err != nil { + return nil, err + } + return configurator.apiCreator(cr, accessToken) +} diff --git a/cbcontainers/remote_configuration/configurator_test.go b/cbcontainers/remote_configuration/configurator_test.go new file mode 100644 index 00000000..b4127102 --- /dev/null +++ b/cbcontainers/remote_configuration/configurator_test.go @@ -0,0 +1,389 @@ +package remote_configuration_test + +import ( + "context" + "errors" + "github.com/go-logr/logr" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration/mocks" + k8sMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "time" +) + +type configuratorMocks struct { + k8sClient *k8sMocks.MockClient + apiGateway *mocks.MockApiGateway + accessTokenProvider *mocks.MockAccessTokenProvider + + stubAccessToken string + stubOperatorVersion string + stubNamespace string + stubClusterID string +} + +// setupConfigurator sets up mocks and creates a Configurator instance with those mocks and some dummy data +func setupConfigurator(ctrl *gomock.Controller) (*remote_configuration.Configurator, configuratorMocks) { + k8sClient := k8sMocks.NewMockClient(ctrl) + apiGateway := mocks.NewMockApiGateway(ctrl) + accessTokenProvider := mocks.NewMockAccessTokenProvider(ctrl) + + var mockAPIProvider remote_configuration.ApiCreator = func( + cbContainersCluster *cbcontainersv1.CBContainersAgent, + accessToken string, + ) (remote_configuration.ApiGateway, error) { + return apiGateway, nil + } + + namespace := "namespace-name" + accessToken := "access-token" + operatorVersion := "1.2.3" + clusterID := "1234567" + accessTokenProvider.EXPECT().GetCBAccessToken(gomock.Any(), gomock.Any(), namespace).Return(accessToken, nil).AnyTimes() + + configurator := remote_configuration.NewConfigurator( + k8sClient, + mockAPIProvider, + logr.Discard(), + accessTokenProvider, + operatorVersion, + namespace, + clusterID, + ) + + mocksHolder := configuratorMocks{ + k8sClient: k8sClient, + apiGateway: apiGateway, + accessTokenProvider: accessTokenProvider, + stubAccessToken: accessToken, + stubOperatorVersion: operatorVersion, + stubNamespace: namespace, + stubClusterID: clusterID, + } + + return configurator, mocksHolder +} + +func TestConfigChangeIsAppliedAndAcknowledgedCorrectly(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + // Setup stub data + var initialGeneration, finalGeneration int64 = 1, 2 + expectedAgentVersion := "3.0.0" + cr := &cbcontainersv1.CBContainersAgent{ObjectMeta: metav1.ObjectMeta{Generation: initialGeneration}} + configChange := randomPendingConfigChange() + configChange.AgentVersion = &expectedAgentVersion + + setupCRInK8S(mocks.k8sClient, cr) + setupValidCompatibilityData(mocks.apiGateway, expectedAgentVersion, mocks.stubOperatorVersion) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) + + // Setup mock assertions + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, finalGeneration, update.AppliedGeneration) + assert.Equal(t, models.ChangeStatusAcked, update.Status) + assert.NotEmpty(t, update.AppliedTimestamp, "applied timestamp should be populated") + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) + + parsedTime, err := time.Parse(time.RFC3339, update.AppliedTimestamp) + assert.NoError(t, err) + assert.True(t, time.Now().After(parsedTime)) + return nil + }) + + setupUpdateCRMock(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, expectedAgentVersion, agent.Spec.Version) + agent.ObjectMeta.Generation = finalGeneration + }) + + err := configurator.RunIteration(context.Background()) + assert.NoError(t, err) +} + +func TestWhenChangeIsNotApplicableShouldReturnError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + cr := &cbcontainersv1.CBContainersAgent{} + maxAgentVersionForOperator := "4.0.0" + agentVersion := "5.0.0" + configChange := randomPendingConfigChange() + configChange.AgentVersion = &agentVersion + + setupCRInK8S(mocks.k8sClient, cr) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) + + // Setup invalid compatibility; no need to do full verification here - this is what the validator tests are for + // We just want to check that _some_ validation happens + mocks.apiGateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{{Version: agentVersion}}, nil) + mocks.apiGateway.EXPECT().GetCompatibilityMatrixEntryFor(mocks.stubOperatorVersion).Return(&models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentVersion(maxAgentVersionForOperator), + }, nil) + + // Setup mock assertions + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, models.ChangeStatusFailed, update.Status) + assert.NotEmpty(t, update.Error) + assert.NotEmpty(t, update.ErrorReason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) + + return nil + }) + + err := configurator.RunIteration(context.Background()) + assert.Error(t, err) +} + +func TestWhenThereAreNoPendingChangesNothingHappens(t *testing.T) { + testCases := []struct { + name string + dataFromService []models.ConfigurationChange + }{ + { + name: "empty list", + dataFromService: []models.ConfigurationChange{}, + }, + { + name: "list is not empty but there are no PENDING changes", + dataFromService: []models.ConfigurationChange{ + {ID: "123", Status: "non-existent"}, + {ID: "234", Status: "FAILED"}, + {ID: "345", Status: "ACKNOWLEDGED"}, + {ID: "456", Status: "SUCCEEDED"}, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + setupCRInK8S(mocks.k8sClient, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return(tC.dataFromService, nil) + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Times(0) + + err := configurator.RunIteration(context.Background()) + assert.NoError(t, err) + }) + } +} + +func TestWhenThereAreMultiplePendingChangesTheOldestIsSelected(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + olderChange := randomPendingConfigChange() + newerChange := randomPendingConfigChange() + + expectedVersion := "version-for-older-change" + versionThatShouldNotBe := "version-for-newer-change" + olderChange.AgentVersion = &expectedVersion + newerChange.AgentVersion = &versionThatShouldNotBe + olderChange.Timestamp = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) + newerChange.Timestamp = time.Now().UTC().Add(-1 * time.Hour).Format(time.RFC3339) + + setupCRInK8S(mocks.k8sClient, nil) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{newerChange, olderChange}, nil) + setupValidCompatibilityData(mocks.apiGateway, expectedVersion, mocks.stubOperatorVersion) + + setupUpdateCRMock(t, mocks.k8sClient, func(agent *cbcontainersv1.CBContainersAgent) { + assert.Equal(t, expectedVersion, agent.Spec.Version) + }) + + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, olderChange.ID, update.ID) + return nil + }) + + err := configurator.RunIteration(context.Background()) + assert.NoError(t, err) +} + +func TestWhenConfigurationAPIReturnsErrorForListShouldPropagateErr(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + setupCRInK8S(mocks.k8sClient, nil) + + errFromService := errors.New("some error") + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return(nil, errFromService) + + returnedErr := configurator.RunIteration(context.Background()) + + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} + +func TestWhenGettingCRFromAPIServerFailsAnErrorIsReturned(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + errFromService := errors.New("some error") + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}).Return(errFromService) + + returnedErr := configurator.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} + +func TestWhenUpdatingCRFailsChangeIsUpdatedAsFailed(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + configChange := randomPendingConfigChange() + + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) + setupCRInK8S(mocks.k8sClient, nil) + setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) + + errFromService := errors.New("some error") + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errFromService) + + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, update models.ConfigurationChangeStatusUpdate) error { + assert.Equal(t, configChange.ID, update.ID) + assert.Equal(t, models.ChangeStatusFailed, update.Status) + assert.NotEmpty(t, update.Error) + assert.Empty(t, update.ErrorReason) + assert.Equal(t, int64(0), update.AppliedGeneration) + assert.Empty(t, update.AppliedTimestamp) + assert.Equal(t, mocks.stubClusterID, update.ClusterIdentifier) + + return nil + }) + + returnedErr := configurator.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, returnedErr, errFromService, "expected returned error to match or wrap error from service") +} + +func TestWhenUpdatingStatusToBackendFailsShouldReturnError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + configChange := randomPendingConfigChange() + + setupCRInK8S(mocks.k8sClient, nil) + setupValidCompatibilityData(mocks.apiGateway, *configChange.AgentVersion, mocks.stubOperatorVersion) + mocks.apiGateway.EXPECT().GetConfigurationChanges(gomock.Any(), mocks.stubClusterID).Return([]models.ConfigurationChange{configChange}, nil) + mocks.k8sClient.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + + errFromService := errors.New("some error") + mocks.apiGateway.EXPECT().UpdateConfigurationChangeStatus(gomock.Any(), gomock.Any()).Return(errFromService) + + returnedErr := configurator.RunIteration(context.Background()) + assert.Error(t, returnedErr) + assert.ErrorIs(t, errFromService, returnedErr, "expected returned error to match or wrap error from service") +} + +func TestWhenThereIsNoCRInstalledNothingHappens(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + mocks.k8sClient.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{} + }) + + // No other mock calls should happen without a CR + assert.NoError(t, configurator.RunIteration(context.Background())) +} + +func TestWhenFeatureIsDisabledInCRNothingHappens(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + configurator, mocks := setupConfigurator(ctrl) + + cr := &cbcontainersv1.CBContainersAgent{ + Spec: cbcontainersv1.CBContainersAgentSpec{ + Components: cbcontainersv1.CBContainersComponentsSpec{ + Settings: cbcontainersv1.CBContainersComponentsSettings{ + RemoteConfiguration: &cbcontainersv1.CBContainersRemoteConfigurationSettings{ + EnabledForAgent: falsePtr, + }, + }}, + }, + } + setupCRInK8S(mocks.k8sClient, cr) + + // No other mock calls should happen once we know the feature is disabled + assert.NoError(t, configurator.RunIteration(context.Background())) +} + +// setupCRInK8S ensures the mock client will return 1 agent item for List calls - either the provided one or an empty CR otherwise +func setupCRInK8S(mock *k8sMocks.MockClient, item *cbcontainersv1.CBContainersAgent) *cbcontainersv1.CBContainersAgent { + if item == nil { + item = &cbcontainersv1.CBContainersAgent{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: cbcontainersv1.CBContainersAgentSpec{}, + Status: cbcontainersv1.CBContainersAgentStatus{}, + } + } + mock.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersAgentList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersAgentList, _ ...any) { + list.Items = []cbcontainersv1.CBContainersAgent{*item} + }) + + return item +} + +func setupUpdateCRMock(t *testing.T, mock *k8sMocks.MockClient, assert func(*cbcontainersv1.CBContainersAgent)) { + mock.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, item any, _ ...any) error { + asCb, ok := item.(*cbcontainersv1.CBContainersAgent) + require.True(t, ok) + + assert(asCb) + return nil + }) +} + +func setupValidCompatibilityData(mockGateway *mocks.MockApiGateway, sensorVersion, operatorVersion string) { + mockGateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{{ + Version: sensorVersion, + SupportsRuntime: true, + SupportsClusterScanning: true, + SupportsCndr: true, + }}, nil) + + mockGateway.EXPECT().GetCompatibilityMatrixEntryFor(operatorVersion).Return(&models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + }, nil) + +} diff --git a/cbcontainers/remote_configuration/mocks/generated.go b/cbcontainers/remote_configuration/mocks/generated.go new file mode 100644 index 00000000..d0aef348 --- /dev/null +++ b/cbcontainers/remote_configuration/mocks/generated.go @@ -0,0 +1,4 @@ +package mocks + +//go:generate mockgen -destination mock_api_gateway.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration ApiGateway +//go:generate mockgen -destination mock_access_token_provider.go -package mocks github.com/vmware/cbcontainers-operator/remote_configuration AccessTokenProvider diff --git a/cbcontainers/remote_configuration/mocks/mock_access_token_provider.go b/cbcontainers/remote_configuration/mocks/mock_access_token_provider.go new file mode 100644 index 00000000..2c3af28a --- /dev/null +++ b/cbcontainers/remote_configuration/mocks/mock_access_token_provider.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: AccessTokenProvider) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" +) + +// MockAccessTokenProvider is a mock of AccessTokenProvider interface. +type MockAccessTokenProvider struct { + ctrl *gomock.Controller + recorder *MockAccessTokenProviderMockRecorder +} + +// MockAccessTokenProviderMockRecorder is the mock recorder for MockAccessTokenProvider. +type MockAccessTokenProviderMockRecorder struct { + mock *MockAccessTokenProvider +} + +// NewMockAccessTokenProvider creates a new mock instance. +func NewMockAccessTokenProvider(ctrl *gomock.Controller) *MockAccessTokenProvider { + mock := &MockAccessTokenProvider{ctrl: ctrl} + mock.recorder = &MockAccessTokenProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessTokenProvider) EXPECT() *MockAccessTokenProviderMockRecorder { + return m.recorder +} + +// GetCBAccessToken mocks base method. +func (m *MockAccessTokenProvider) GetCBAccessToken(arg0 context.Context, arg1 *v1.CBContainersAgent, arg2 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCBAccessToken", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCBAccessToken indicates an expected call of GetCBAccessToken. +func (mr *MockAccessTokenProviderMockRecorder) GetCBAccessToken(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCBAccessToken", reflect.TypeOf((*MockAccessTokenProvider)(nil).GetCBAccessToken), arg0, arg1, arg2) +} diff --git a/cbcontainers/remote_configuration/mocks/mock_api_gateway.go b/cbcontainers/remote_configuration/mocks/mock_api_gateway.go new file mode 100644 index 00000000..1a2c1386 --- /dev/null +++ b/cbcontainers/remote_configuration/mocks/mock_api_gateway.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/remote_configuration (interfaces: ApiGateway) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + models "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// MockApiGateway is a mock of ApiGateway interface. +type MockApiGateway struct { + ctrl *gomock.Controller + recorder *MockApiGatewayMockRecorder +} + +// MockApiGatewayMockRecorder is the mock recorder for MockApiGateway. +type MockApiGatewayMockRecorder struct { + mock *MockApiGateway +} + +// NewMockApiGateway creates a new mock instance. +func NewMockApiGateway(ctrl *gomock.Controller) *MockApiGateway { + mock := &MockApiGateway{ctrl: ctrl} + mock.recorder = &MockApiGatewayMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockApiGateway) EXPECT() *MockApiGatewayMockRecorder { + return m.recorder +} + +// GetCompatibilityMatrixEntryFor mocks base method. +func (m *MockApiGateway) GetCompatibilityMatrixEntryFor(arg0 string) (*models.OperatorCompatibility, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCompatibilityMatrixEntryFor", arg0) + ret0, _ := ret[0].(*models.OperatorCompatibility) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCompatibilityMatrixEntryFor indicates an expected call of GetCompatibilityMatrixEntryFor. +func (mr *MockApiGatewayMockRecorder) GetCompatibilityMatrixEntryFor(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCompatibilityMatrixEntryFor", reflect.TypeOf((*MockApiGateway)(nil).GetCompatibilityMatrixEntryFor), arg0) +} + +// GetConfigurationChanges mocks base method. +func (m *MockApiGateway) GetConfigurationChanges(arg0 context.Context, arg1 string) ([]models.ConfigurationChange, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConfigurationChanges", arg0, arg1) + ret0, _ := ret[0].([]models.ConfigurationChange) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConfigurationChanges indicates an expected call of GetConfigurationChanges. +func (mr *MockApiGatewayMockRecorder) GetConfigurationChanges(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigurationChanges", reflect.TypeOf((*MockApiGateway)(nil).GetConfigurationChanges), arg0, arg1) +} + +// GetSensorMetadata mocks base method. +func (m *MockApiGateway) GetSensorMetadata() ([]models.SensorMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSensorMetadata") + ret0, _ := ret[0].([]models.SensorMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSensorMetadata indicates an expected call of GetSensorMetadata. +func (mr *MockApiGatewayMockRecorder) GetSensorMetadata() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSensorMetadata", reflect.TypeOf((*MockApiGateway)(nil).GetSensorMetadata)) +} + +// UpdateConfigurationChangeStatus mocks base method. +func (m *MockApiGateway) UpdateConfigurationChangeStatus(arg0 context.Context, arg1 models.ConfigurationChangeStatusUpdate) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateConfigurationChangeStatus", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateConfigurationChangeStatus indicates an expected call of UpdateConfigurationChangeStatus. +func (mr *MockApiGatewayMockRecorder) UpdateConfigurationChangeStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigurationChangeStatus", reflect.TypeOf((*MockApiGateway)(nil).UpdateConfigurationChangeStatus), arg0, arg1) +} diff --git a/cbcontainers/remote_configuration/validation.go b/cbcontainers/remote_configuration/validation.go new file mode 100644 index 00000000..09276d1c --- /dev/null +++ b/cbcontainers/remote_configuration/validation.go @@ -0,0 +1,108 @@ +package remote_configuration + +import ( + "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +type invalidChangeError struct { + msg string +} + +func (i invalidChangeError) Error() string { + return i.msg +} + +func NewConfigurationChangeValidator(operatorVersion string, api ApiGateway) (*ConfigurationChangeValidator, error) { + compatibilityMatrix, err := api.GetCompatibilityMatrixEntryFor(operatorVersion) + if err != nil { + return nil, err + } + if compatibilityMatrix == nil { + return nil, fmt.Errorf("compatibility matrix API returned no data but no error as well, cannot continue") + } + + sensors, err := api.GetSensorMetadata() + if err != nil { + return nil, err + } + + return &ConfigurationChangeValidator{ + SensorData: sensors, + OperatorCompatibilityData: *compatibilityMatrix, + }, nil +} + +type ConfigurationChangeValidator struct { + SensorData []models.SensorMetadata + OperatorCompatibilityData models.OperatorCompatibility +} + +func (validator *ConfigurationChangeValidator) ValidateChange(change models.ConfigurationChange, cr *cbcontainersv1.CBContainersAgent) error { + var versionToValidate string + + // If the change will be modifying the agent version as well, we need to check what the _new_ version supports + if change.AgentVersion != nil { + versionToValidate = *change.AgentVersion + } else { + // Otherwise the current agent must actually work with the requested features + versionToValidate = cr.Spec.Version + } + + if err := validator.validateOperatorAndSensorVersionCompatibility(versionToValidate); err != nil { + return err + } + + return validator.validateSensorAndFeatureCompatibility(versionToValidate, change) +} + +func (validator *ConfigurationChangeValidator) findMatchingSensor(sensorVersion string) *models.SensorMetadata { + for _, sensor := range validator.SensorData { + if sensor.Version == sensorVersion { + return &sensor + } + } + + return nil +} + +func (validator *ConfigurationChangeValidator) validateOperatorAndSensorVersionCompatibility(sensorVersion string) error { + if err := validator.OperatorCompatibilityData.CheckCompatibility(sensorVersion); err != nil { + return invalidChangeError{msg: err.Error()} + } + return nil +} + +func (validator *ConfigurationChangeValidator) validateSensorAndFeatureCompatibility(targetVersion string, change models.ConfigurationChange) error { + sensor := validator.findMatchingSensor(targetVersion) + if sensor == nil { + return fmt.Errorf("could not find sensor metadata for version %s", targetVersion) + } + + if change.EnableClusterScanning != nil && + *change.EnableClusterScanning == true && + !sensor.SupportsClusterScanning { + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support cluster scanning feature", targetVersion)} + } + + if change.EnableClusterScanningSecretDetection != nil && + *change.EnableClusterScanningSecretDetection == true && + !sensor.SupportsClusterScanningSecrets { + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support secret detection during cluster scanning feature", targetVersion)} + } + + if change.EnableRuntime != nil && + *change.EnableRuntime == true && + !sensor.SupportsRuntime { + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support runtime protection feature", targetVersion)} + } + + if change.EnableCNDR != nil && + *change.EnableCNDR == true && + !sensor.SupportsCndr { + return invalidChangeError{msg: fmt.Sprintf("sensor version %s does not support cloud-native detect and response feature", targetVersion)} + } + + return nil +} diff --git a/cbcontainers/remote_configuration/validation_test.go b/cbcontainers/remote_configuration/validation_test.go new file mode 100644 index 00000000..fbed9294 --- /dev/null +++ b/cbcontainers/remote_configuration/validation_test.go @@ -0,0 +1,332 @@ +package remote_configuration_test + +import ( + "errors" + "fmt" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/models" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration/mocks" + "testing" +) + +var ( + trueV = true + truePtr = &trueV + falseV = false + falsePtr = &falseV +) + +func TestValidatorConstructorReturnsErrOnFailures(t *testing.T) { + expectedOperatorVersion := "5.0.0" + + testCases := []struct { + name string + setupGatewayMock func(gateway *mocks.MockApiGateway) + }{ + { + name: "get compatibility returns err", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(nil, errors.New("some error")).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{}, nil).AnyTimes() + }, + }, + { + name: "get sensor metadata returns err", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(&models.OperatorCompatibility{}, nil).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return(nil, errors.New("some error")).AnyTimes() + }, + }, + { + name: "get compatibility returns nil", + setupGatewayMock: func(gateway *mocks.MockApiGateway) { + gateway.EXPECT().GetCompatibilityMatrixEntryFor(expectedOperatorVersion).Return(nil, nil).AnyTimes() + gateway.EXPECT().GetSensorMetadata().Return([]models.SensorMetadata{}, nil).AnyTimes() + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockGateway := mocks.NewMockApiGateway(ctrl) + tC.setupGatewayMock(mockGateway) + + validator, err := remote_configuration.NewConfigurationChangeValidator(expectedOperatorVersion, mockGateway) + + assert.Nil(t, validator) + assert.Error(t, err) + }) + } +} + +func TestValidateFailsIfSensorDoesNotSupportRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change models.ConfigurationChange + sensorMeta models.SensorMetadata + }{ + { + name: "cluster scanning", + change: models.ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanning: false, + }, + }, + { + name: "cluster scanning secret detection", + change: models.ConfigurationChange{ + EnableClusterScanningSecretDetection: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanningSecrets: false, + }, + }, + { + name: "runtime protection", + change: models.ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsRuntime: false, + }, + }, + { + name: "CNDR", + change: models.ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsCndr: false, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s not supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + err := target.ValidateChange(tC.change, cr) + + assert.Error(t, err) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s not supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + err := target.ValidateChange(tC.change, cr) + + assert.Error(t, err) + }) + } +} + +func TestValidateFailsIfSensorIsNotInList(t *testing.T) { + sensorMetaWithoutTargetSensor := []models.SensorMetadata{{ + Version: "1.0.0", + IsLatest: false, + SupportsRuntime: true, + SupportsClusterScanning: true, + SupportsClusterScanningSecrets: true, + SupportsCndr: true, + }} + operatorSupportsAll := models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + } + unknownVersion := "1.2.3" + + validator := remote_configuration.ConfigurationChangeValidator{ + SensorData: sensorMetaWithoutTargetSensor, + OperatorCompatibilityData: operatorSupportsAll, + } + + change := models.ConfigurationChange{ + AgentVersion: &unknownVersion, + } + cr := &cbcontainersv1.CBContainersAgent{} + + assert.Error(t, validator.ValidateChange(change, cr)) +} + +func TestValidateSucceedsIfSensorSupportsRequestedFeature(t *testing.T) { + testCases := []struct { + name string + change models.ConfigurationChange + sensorMeta models.SensorMetadata + }{ + { + name: "cluster scanning", + change: models.ConfigurationChange{ + EnableClusterScanning: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanning: true, + }, + }, + { + name: "cluster scanning secret detection", + change: models.ConfigurationChange{ + EnableClusterScanningSecretDetection: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsClusterScanningSecrets: true, + }, + }, + { + name: "runtime protection", + change: models.ConfigurationChange{ + EnableRuntime: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsRuntime: true, + }, + }, + { + name: "CNDR", + change: models.ConfigurationChange{ + EnableCNDR: truePtr, + }, + sensorMeta: models.SensorMetadata{ + SupportsCndr: true, + }, + }, + } + + for _, tC := range testCases { + version := "dummy-version" + tC.sensorMeta.Version = version + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{tC.sensorMeta}, + } + + t.Run(fmt.Sprintf("no version in change, %s is supported by current agent", tC.name), func(t *testing.T) { + tC.change.AgentVersion = nil + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: version}} + + err := target.ValidateChange(tC.change, cr) + + assert.NoError(t, err) + }) + + t.Run(fmt.Sprintf("change also applies agent version, %s is supported by that version", tC.name), func(t *testing.T) { + tC.change.AgentVersion = &version + cr := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "some-other-verson"}} + + err := target.ValidateChange(tC.change, cr) + + assert.NoError(t, err) + }) + } +} + +func TestValidateFailsIfSensorAndOperatorAreNotCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility + }{ + { + name: "sensor version is too high", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: "4.0.0", + }, + }, + { + name: "sensor version is too low", + versionToApply: "0.9", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatibility, + } + + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} + + err := target.ValidateChange(change, cr) + assert.Error(t, err) + }) + } +} + +func TestValidateSucceedsIfSensorAndOperatorAreCompatible(t *testing.T) { + testCases := []struct { + name string + versionToApply string + operatorCompatibility models.OperatorCompatibility + }{ + { + name: "sensor version is at lower end", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "5.0.0", + MaxAgent: "6.0.0", + }, + }, + { + name: "sensor version is at upper end", + versionToApply: "0.9", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "0.1.0", + MaxAgent: "0.9.0", + }, + }, + { + name: "sensor version is within range", + versionToApply: "2.3.4", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: "1.0.0", + MaxAgent: "2.4", + }, + }, + { + name: "operator supports 'infinite' versions", + versionToApply: "5.0.0", + operatorCompatibility: models.OperatorCompatibility{ + MinAgent: models.AgentMinVersionNone, + MaxAgent: models.AgentMaxVersionLatest, + }, + }, + } + + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + target := remote_configuration.ConfigurationChangeValidator{ + SensorData: []models.SensorMetadata{{Version: tC.versionToApply}}, + OperatorCompatibilityData: tC.operatorCompatibility, + } + + change := models.ConfigurationChange{AgentVersion: &tC.versionToApply} + cr := &cbcontainersv1.CBContainersAgent{} + + err := target.ValidateChange(change, cr) + assert.NoError(t, err) + }) + } +} diff --git a/cbcontainers/state/operator/auth_provider.go b/cbcontainers/state/operator/auth_provider.go new file mode 100644 index 00000000..2e26a117 --- /dev/null +++ b/cbcontainers/state/operator/auth_provider.go @@ -0,0 +1,43 @@ +package operator + +import ( + "context" + "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type SecretAccessTokenProvider struct { + k8sClient client.Client +} + +func NewSecretAccessTokenProvider(k8sClient client.Client) *SecretAccessTokenProvider { + return &SecretAccessTokenProvider{k8sClient: k8sClient} +} + +// GetCBAccessToken will attempt to read the access token value from a secret in the deployed namespace +// The secret should be defined in the provided Custom resource +func (provider *SecretAccessTokenProvider) GetCBAccessToken( + ctx context.Context, + cbContainersCluster *cbcontainersv1.CBContainersAgent, + deployedNamespace string, +) (string, error) { + accessTokenSecretNamespacedName := types.NamespacedName{ + Name: cbContainersCluster.Spec.AccessTokenSecretName, + Namespace: deployedNamespace, + } + accessTokenSecret := &corev1.Secret{} + if err := provider.k8sClient.Get(ctx, accessTokenSecretNamespacedName, accessTokenSecret); err != nil { + return "", fmt.Errorf("couldn't find access token secret k8s object: %v", err) + } + + accessToken := string(accessTokenSecret.Data[commonState.AccessTokenSecretKeyName]) + if accessToken == "" { + return "", fmt.Errorf("the k8s secret %v is missing the key %v", accessTokenSecretNamespacedName, commonState.AccessTokenSecretKeyName) + } + + return accessToken, nil +} diff --git a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml index 4a0add7e..bc052549 100644 --- a/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml +++ b/charts/cbcontainers-operator/cbcontainers-operator-chart/templates/operator.yaml @@ -6322,6 +6322,18 @@ spec: defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + default: true + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator + configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 6109fb6e..9c409c03 100644 --- a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -6312,6 +6312,18 @@ spec: defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + default: true + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator + configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml index 792a2e84..27d1607b 100644 --- a/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml +++ b/config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml @@ -5830,6 +5830,16 @@ spec: exposed more as a means by which to control the defaults. type: string type: object + remoteConfiguration: + description: RemoteConfiguration holds settings for the operator/agent's + feature to apply configuration changes via the Carbon black + console + properties: + enabledForAgent: + description: EnabledForAgent turns the feature to change + agent configuration remotely (as opposed to operator configuration) + type: boolean + type: object type: object type: object gateways: diff --git a/controllers/cbcontainersagent_controller.go b/controllers/cbcontainersagent_controller.go index 682ac582..ce43d563 100644 --- a/controllers/cbcontainersagent_controller.go +++ b/controllers/cbcontainersagent_controller.go @@ -28,11 +28,9 @@ import ( "github.com/go-logr/logr" "github.com/vmware/cbcontainers-operator/cbcontainers/models" applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,6 +52,10 @@ type AgentProcessor interface { Process(cbContainersAgent *cbcontainersv1.CBContainersAgent, accessToken string) (*models.RegistrySecretValues, error) } +type AccessTokenProvider interface { + GetCBAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, namespace string) (string, error) +} + type CBContainersAgentController struct { client.Client Log logr.Logger @@ -62,7 +64,8 @@ type CBContainersAgentController struct { StateApplier StateApplier K8sVersion string // Namespace is the kubernetes namespace for all agent components - Namespace string + Namespace string + AccessTokenProvider AccessTokenProvider } func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Context) (*cbcontainersv1.CBContainersAgent, error) { @@ -124,10 +127,13 @@ func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Re return ctrl.SetControllerReference(cbContainersAgent, controlledResource, r.Scheme) } - accessToken, err := r.getAccessToken(context.Background(), cbContainersAgent) + accessToken, err := r.AccessTokenProvider.GetCBAccessToken(ctx, cbContainersAgent, r.Namespace) if err != nil { return ctrl.Result{}, err } + if accessToken == "" { + return ctrl.Result{}, fmt.Errorf("CB access token has empty value, cannot continue") + } var registrySecret *models.RegistrySecretValues if cbContainersAgent.Spec.Components.Settings.ShouldCreateDefaultImagePullSecrets() { @@ -166,21 +172,6 @@ func (r *CBContainersAgentController) getRegistrySecretValues(ctx context.Contex return r.ClusterProcessor.Process(cbContainersCluster, accessToken) } -func (r *CBContainersAgentController) getAccessToken(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent) (string, error) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: cbContainersCluster.Spec.AccessTokenSecretName, Namespace: r.Namespace} - accessTokenSecret := &corev1.Secret{} - if err := r.Get(ctx, accessTokenSecretNamespacedName, accessTokenSecret); err != nil { - return "", fmt.Errorf("couldn't find access token secret k8s object: %v", err) - } - - accessToken := string(accessTokenSecret.Data[commonState.AccessTokenSecretKeyName]) - if accessToken == "" { - return "", fmt.Errorf("the k8s secret %v is missing the key %v", accessTokenSecretNamespacedName, commonState.AccessTokenSecretKeyName) - } - - return accessToken, nil -} - func (r *CBContainersAgentController) updateCRStatus(ctx context.Context, cbContainersCluster *cbcontainersv1.CBContainersAgent, agentStateWasChanged bool) error { // If we don't expect more changes (i.e. nothing changed in reality) and we haven't updated the status, we do so now. if !agentStateWasChanged && cbContainersCluster.Status.ObservedGeneration < cbContainersCluster.ObjectMeta.Generation { diff --git a/controllers/cbcontainersagent_controller_test.go b/controllers/cbcontainersagent_controller_test.go index 1eec9aab..06ff6a60 100644 --- a/controllers/cbcontainersagent_controller_test.go +++ b/controllers/cbcontainersagent_controller_test.go @@ -9,30 +9,28 @@ import ( "testing" "time" - logrTesting "github.com/go-logr/logr/testing" + logrTesting "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils" testUtilsMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" "github.com/vmware/cbcontainers-operator/controllers" "github.com/vmware/cbcontainers-operator/controllers/mocks" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrlRuntime "sigs.k8s.io/controller-runtime" ) type SetupClusterControllerTest func(*ClusterControllerTestMocks) type ClusterControllerTestMocks struct { - client *testUtilsMocks.MockClient - statusWriter *testUtilsMocks.MockStatusWriter - clusterProcessor *mocks.MockClusterProcessor - stateApplier *mocks.MockStateApplier - ctx context.Context + client *testUtilsMocks.MockClient + statusWriter *testUtilsMocks.MockStatusWriter + accessTokenProvider *mocks.MockAccessTokenProvider + mockAgentProcessor *mocks.MockAgentProcessor + stateApplier *mocks.MockStateApplier + ctx context.Context } const ( @@ -70,11 +68,12 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr mockK8SClient.EXPECT().Status().Return(mockStatusWriter).AnyTimes() mocksObjects := &ClusterControllerTestMocks{ - ctx: context.TODO(), - client: mockK8SClient, - statusWriter: mockStatusWriter, - clusterProcessor: mocks.NewMockClusterProcessor(ctrl), - stateApplier: mocks.NewMockStateApplier(ctrl), + ctx: context.TODO(), + client: mockK8SClient, + statusWriter: mockStatusWriter, + accessTokenProvider: mocks.NewMockAccessTokenProvider(ctrl), + mockAgentProcessor: mocks.NewMockAgentProcessor(ctrl), + stateApplier: mocks.NewMockStateApplier(ctrl), } for _, setup := range setups { @@ -83,12 +82,13 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr controller := &controllers.CBContainersAgentController{ Client: mocksObjects.client, - Log: logrTesting.NewTestLogger(t), + Log: logrTesting.New(t), Scheme: &runtime.Scheme{}, Namespace: agentNamespace, - ClusterProcessor: mocksObjects.clusterProcessor, - StateApplier: mocksObjects.stateApplier, + AccessTokenProvider: mocksObjects.accessTokenProvider, + ClusterProcessor: mocksObjects.mockAgentProcessor, + StateApplier: mocksObjects.stateApplier, } return controller.Reconcile(mocksObjects.ctx, ctrlRuntime.Request{}) @@ -109,15 +109,11 @@ func setupClusterCustomResource(items ...cbcontainersv1.CBContainersAgent) Setup } } -func setUpTokenSecretValues(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}). - Do(func(ctx context.Context, namespacedName types.NamespacedName, secret *corev1.Secret, _ ...interface{}) { - secret.Data = map[string][]byte{ - commonState.AccessTokenSecretKeyName: []byte(MyClusterTokenValue), - } - }). - Return(nil) +func setUpAccessToken(testMocks *ClusterControllerTestMocks) { + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return(MyClusterTokenValue, nil) } func TestListClusterResourcesErrorShouldReturnError(t *testing.T) { @@ -152,8 +148,10 @@ func TestFindingMoreThanOneClusterResourceShouldReturnError(t *testing.T) { func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource(), func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(fmt.Errorf("")) + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return("", fmt.Errorf("some error")) }) require.Error(t, err) @@ -161,8 +159,10 @@ func TestGetTokenSecretErrorShouldReturnError(t *testing.T) { func TestTokenSecretWithoutTokenValueShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, setupClusterCustomResource(), func(testMocks *ClusterControllerTestMocks) { - accessTokenSecretNamespacedName := types.NamespacedName{Name: ClusterAccessTokenSecretName, Namespace: agentNamespace} - testMocks.client.EXPECT().Get(testMocks.ctx, accessTokenSecretNamespacedName, &corev1.Secret{}).Return(nil) + testMocks.accessTokenProvider. + EXPECT(). + GetCBAccessToken(testMocks.ctx, gomock.AssignableToTypeOf(&cbcontainersv1.CBContainersAgent{}), agentNamespace). + Return("", nil) }) require.Error(t, err) @@ -172,16 +172,16 @@ func TestClusterReconcile(t *testing.T) { secretValues := &models.RegistrySecretValues{Data: map[string][]byte{test_utils.RandomString(): {}}} t.Run("When processor returns error, reconcile should return error", func(t *testing.T) { - _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(nil, fmt.Errorf("")) + _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(nil, fmt.Errorf("")) }) require.Error(t, err) }) t.Run("When state applier returns error, reconcile should return error", func(t *testing.T) { - _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + _, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(false, fmt.Errorf("")) }) @@ -189,8 +189,8 @@ func TestClusterReconcile(t *testing.T) { }) t.Run("When state applier returns state was changed, reconcile should return Requeue true", func(t *testing.T) { - result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(true, nil) }) @@ -199,8 +199,8 @@ func TestClusterReconcile(t *testing.T) { }) t.Run("When state applier returns state was not changed, reconcile should return default Requeue", func(t *testing.T) { - result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&ClusterCustomResourceItems[0]), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&ClusterCustomResourceItems[0].Spec), secretValues, gomock.Any()).Return(false, nil) }) @@ -217,8 +217,8 @@ func TestStatusUpdates(t *testing.T) { resourceWithStatus.ObjectMeta.Generation = 2 resourceWithStatus.Status.ObservedGeneration = 1 - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceWithStatus.Spec), secretValues, gomock.Any()).Return(true, nil) testMocks.statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).MaxTimes(0) }) @@ -232,8 +232,8 @@ func TestStatusUpdates(t *testing.T) { resourceWithStatus.ObjectMeta.Generation = 1 resourceWithStatus.Status.ObservedGeneration = 1 - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceWithStatus), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceWithStatus), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceWithStatus.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(gomock.Any(), gomock.Any()).MaxTimes(0) }) @@ -250,8 +250,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Times(1).Return(nil) }) @@ -268,8 +268,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Return(k8sErrors.NewConflict(schema.GroupResource{}, "conflict", nil)) }) @@ -286,8 +286,8 @@ func TestStatusUpdates(t *testing.T) { expectedResourceWithUpdatedStatus := resourceBeforeReconcile expectedResourceWithUpdatedStatus.Status.ObservedGeneration = expectedResourceWithUpdatedStatus.Generation - result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpTokenSecretValues, func(testMocks *ClusterControllerTestMocks) { - testMocks.clusterProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) + result, err := testCBContainersClusterController(t, setupClusterCustomResource(resourceBeforeReconcile), setUpAccessToken, func(testMocks *ClusterControllerTestMocks) { + testMocks.mockAgentProcessor.EXPECT().Process(MatchAgentResource(&resourceBeforeReconcile), MyClusterTokenValue).Return(secretValues, nil) testMocks.stateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, MatchAgentSpec(&resourceBeforeReconcile.Spec), secretValues, gomock.Any()).Return(false, nil) testMocks.statusWriter.EXPECT().Update(testMocks.ctx, MatchAgentResource(&expectedResourceWithUpdatedStatus), gomock.Any()).Return(fmt.Errorf("some error")) }) diff --git a/controllers/mocks/generated.go b/controllers/mocks/generated.go index c343b9b6..6a139b21 100644 --- a/controllers/mocks/generated.go +++ b/controllers/mocks/generated.go @@ -1,4 +1,5 @@ package mocks //go:generate mockgen -destination mock_state_applier.go -package mocks github.com/vmware/cbcontainers-operator/controllers StateApplier -//go:generate mockgen -destination mock_cluster_processor.go -package mocks github.com/vmware/cbcontainers-operator/controllers ClusterProcessor +//go:generate mockgen -destination mock_agent_processor.go -package mocks github.com/vmware/cbcontainers-operator/controllers AgentProcessor +//go:generate mockgen -destination mock_access_token_provider.go -package mocks github.com/vmware/cbcontainers-operator/controllers AccessTokenProvider diff --git a/controllers/mocks/mock_access_token_provider.go b/controllers/mocks/mock_access_token_provider.go new file mode 100644 index 00000000..68cca361 --- /dev/null +++ b/controllers/mocks/mock_access_token_provider.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: AccessTokenProvider) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" +) + +// MockAccessTokenProvider is a mock of AccessTokenProvider interface. +type MockAccessTokenProvider struct { + ctrl *gomock.Controller + recorder *MockAccessTokenProviderMockRecorder +} + +// MockAccessTokenProviderMockRecorder is the mock recorder for MockAccessTokenProvider. +type MockAccessTokenProviderMockRecorder struct { + mock *MockAccessTokenProvider +} + +// NewMockAccessTokenProvider creates a new mock instance. +func NewMockAccessTokenProvider(ctrl *gomock.Controller) *MockAccessTokenProvider { + mock := &MockAccessTokenProvider{ctrl: ctrl} + mock.recorder = &MockAccessTokenProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessTokenProvider) EXPECT() *MockAccessTokenProviderMockRecorder { + return m.recorder +} + +// GetCBAccessToken mocks base method. +func (m *MockAccessTokenProvider) GetCBAccessToken(arg0 context.Context, arg1 *v1.CBContainersAgent, arg2 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCBAccessToken", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCBAccessToken indicates an expected call of GetCBAccessToken. +func (mr *MockAccessTokenProviderMockRecorder) GetCBAccessToken(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCBAccessToken", reflect.TypeOf((*MockAccessTokenProvider)(nil).GetCBAccessToken), arg0, arg1, arg2) +} diff --git a/controllers/mocks/mock_agent_processor.go b/controllers/mocks/mock_agent_processor.go new file mode 100644 index 00000000..ba7d7aa7 --- /dev/null +++ b/controllers/mocks/mock_agent_processor.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: AgentProcessor) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + models "github.com/vmware/cbcontainers-operator/cbcontainers/models" +) + +// MockAgentProcessor is a mock of AgentProcessor interface. +type MockAgentProcessor struct { + ctrl *gomock.Controller + recorder *MockAgentProcessorMockRecorder +} + +// MockAgentProcessorMockRecorder is the mock recorder for MockAgentProcessor. +type MockAgentProcessorMockRecorder struct { + mock *MockAgentProcessor +} + +// NewMockAgentProcessor creates a new mock instance. +func NewMockAgentProcessor(ctrl *gomock.Controller) *MockAgentProcessor { + mock := &MockAgentProcessor{ctrl: ctrl} + mock.recorder = &MockAgentProcessorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAgentProcessor) EXPECT() *MockAgentProcessorMockRecorder { + return m.recorder +} + +// Process mocks base method. +func (m *MockAgentProcessor) Process(arg0 *v1.CBContainersAgent, arg1 string) (*models.RegistrySecretValues, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Process", arg0, arg1) + ret0, _ := ret[0].(*models.RegistrySecretValues) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Process indicates an expected call of Process. +func (mr *MockAgentProcessorMockRecorder) Process(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Process", reflect.TypeOf((*MockAgentProcessor)(nil).Process), arg0, arg1) +} diff --git a/controllers/mocks/mock_cluster_processor.go b/controllers/mocks/mock_cluster_processor.go deleted file mode 100644 index a3f416c3..00000000 --- a/controllers/mocks/mock_cluster_processor.go +++ /dev/null @@ -1,51 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: ClusterProcessor) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/vmware/cbcontainers-operator/api/v1" - models "github.com/vmware/cbcontainers-operator/cbcontainers/models" -) - -// MockClusterProcessor is a mock of ClusterProcessor interface. -type MockClusterProcessor struct { - ctrl *gomock.Controller - recorder *MockClusterProcessorMockRecorder -} - -// MockClusterProcessorMockRecorder is the mock recorder for MockClusterProcessor. -type MockClusterProcessorMockRecorder struct { - mock *MockClusterProcessor -} - -// NewMockClusterProcessor creates a new mock instance. -func NewMockClusterProcessor(ctrl *gomock.Controller) *MockClusterProcessor { - mock := &MockClusterProcessor{ctrl: ctrl} - mock.recorder = &MockClusterProcessorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClusterProcessor) EXPECT() *MockClusterProcessorMockRecorder { - return m.recorder -} - -// Process mocks base method. -func (m *MockClusterProcessor) Process(arg0 *v1.CBContainersAgent, arg1 string) (*models.RegistrySecretValues, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Process", arg0, arg1) - ret0, _ := ret[0].(*models.RegistrySecretValues) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Process indicates an expected call of Process. -func (mr *MockClusterProcessorMockRecorder) Process(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Process", reflect.TypeOf((*MockClusterProcessor)(nil).Process), arg0, arg1) -} diff --git a/controllers/remote_configuration_controller.go b/controllers/remote_configuration_controller.go new file mode 100644 index 00000000..4a9d70bc --- /dev/null +++ b/controllers/remote_configuration_controller.go @@ -0,0 +1,79 @@ +package controllers + +import ( + "context" + "github.com/go-logr/logr" + "math" + "time" +) + +const ( + sleepDuration = 2 * time.Minute + maxRetries = 10 // 1024s or ~17 minutes at peak +) + +type configurationApplier interface { + RunIteration(ctx context.Context) error +} + +type RemoteConfigurationController struct { + applier configurationApplier + logger logr.Logger +} + +func NewRemoteConfigurationController(applier configurationApplier, logger logr.Logger) *RemoteConfigurationController { + return &RemoteConfigurationController{applier: applier, logger: logger} +} + +func (controller *RemoteConfigurationController) RunLoop(signalsContext context.Context) { + pollingTimer := backoffTicker{ + Ticker: time.NewTicker(sleepDuration), + sleepDuration: sleepDuration, + maxRetries: maxRetries, + } + defer pollingTimer.Stop() + + for { + select { + case <-signalsContext.Done(): + controller.logger.Info("Received cancel signal, turning off configuration applier") + return + case <-pollingTimer.C: + // Nothing to do; this is the polling sleep case + } + err := controller.applier.RunIteration(signalsContext) + + if err != nil { + controller.logger.Error(err, "Configuration applier iteration failed, it will be retried on next iteration period") + pollingTimer.resetErr() + } else { + controller.logger.Info("Completed configuration applier iteration, sleeping") + pollingTimer.resetSuccess() + } + } +} + +// backoffTicker is a ticker with exponential backoff for errors and static backoff for success cases +// Note: When calling resetErr or resetSuccess, the ticker will wait the full sleep duration again +type backoffTicker struct { + *time.Ticker + + sleepDuration time.Duration + maxRetries int + + currentRetries int +} + +func (b *backoffTicker) resetErr() { + if b.currentRetries < b.maxRetries { + b.currentRetries++ + } + + nextSleepDuration := time.Duration(math.Pow(2.0, float64(b.currentRetries)))*time.Second + b.sleepDuration + b.Reset(nextSleepDuration) +} + +func (b *backoffTicker) resetSuccess() { + b.currentRetries = 0 + b.Reset(b.sleepDuration) +} diff --git a/docs/crds.md b/docs/crds.md index 5d15e836..0dd4eb99 100644 --- a/docs/crds.md +++ b/docs/crds.md @@ -110,6 +110,7 @@ This is the CR you'll need to deploy in order to trigger the operator to deploy ### Other Components Optional parameters -| Parameter | Description | Default | -|--------------------------------------------------|----------------------------------------------|-------------| -| `spec.components.settings.daemonSetsTolerations` | Carbon Black DaemonSet Component Tolerations | Empty array | +| Parameter | Description | Default | +|----------------------------------------------------------------|--------------------------------------------------------------------------------|-------------| +| `spec.components.settings.daemonSetsTolerations` | Carbon Black DaemonSet Component Tolerations | Empty array | +| `spec.components.settings.remoteConfiguration.enabledForAgent` | Enables applying custom resource changes remotely via the Carbon Black Console | True | diff --git a/main.go b/main.go index fc23d85c..4a722555 100644 --- a/main.go +++ b/main.go @@ -18,21 +18,24 @@ package main import ( "context" + "errors" "flag" "fmt" + "github.com/vmware/cbcontainers-operator/cbcontainers/communication/gateway" + "github.com/vmware/cbcontainers-operator/cbcontainers/remote_configuration" "github.com/vmware/cbcontainers-operator/cbcontainers/state" "github.com/vmware/cbcontainers-operator/cbcontainers/state/agent_applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" "github.com/vmware/cbcontainers-operator/cbcontainers/state/operator" "go.uber.org/zap/zapcore" + coreV1 "k8s.io/api/core/v1" "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "strings" - - coreV1 "k8s.io/api/core/v1" + "sync" "github.com/vmware/cbcontainers-operator/cbcontainers/processors" @@ -61,6 +64,7 @@ const ( httpProxyEnv = "HTTP_PROXY" httpsProxyEnv = "HTTPS_PROXY" noProxyEnv = "NO_PROXY" + namespaceEnv = "OPERATOR_NAMESPACE" ) func init() { @@ -113,7 +117,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog.Info("Getting the namespace where operator is running and which should host the agent") - operatorNamespace := os.Getenv("OPERATOR_NAMESPACE") + operatorNamespace := os.Getenv(namespaceEnv) if operatorNamespace == "" { setupLog.Info(fmt.Sprintf("Operator namespace variable was not found. Falling back to default %s", common.DataPlaneNamespaceName)) operatorNamespace = common.DataPlaneNamespaceName @@ -141,16 +145,21 @@ func main() { } clusterIdentifier, k8sVersion := extractConfigurationVariables(mgr) - + operatorVersionProvider := operator.NewEnvVersionProvider() + var processorGatewayCreator processors.APIGatewayCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (processors.APIGateway, error) { + return gateway.NewDefaultGatewayCreator().CreateGateway(cbContainersCluster, accessToken) + } cbContainersAgentLogger := ctrl.Log.WithName("controllers").WithName("CBContainersAgent") + if err = (&controllers.CBContainersAgentController{ - Client: mgr.GetClient(), - Log: cbContainersAgentLogger, - Scheme: mgr.GetScheme(), - K8sVersion: k8sVersion, - Namespace: operatorNamespace, - ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier), - StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), + Client: mgr.GetClient(), + Log: cbContainersAgentLogger, + Scheme: mgr.GetScheme(), + K8sVersion: k8sVersion, + Namespace: operatorNamespace, + AccessTokenProvider: operator.NewSecretAccessTokenProvider(mgr.GetClient()), + ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processorGatewayCreator, operatorVersionProvider, clusterIdentifier), + StateApplier: state.NewStateApplier(mgr.GetAPIReader(), agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, operatorNamespace, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent") os.Exit(1) @@ -167,11 +176,53 @@ func main() { os.Exit(1) } - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + k8sClient := mgr.GetClient() + log := ctrl.Log.WithName("configurator") + operatorVersion, err := operatorVersionProvider.GetOperatorVersion() + if err != nil && !errors.Is(err, operator.ErrNotSemVer) { + setupLog.Error(err, "unable to read the running operator's version from environment variable") os.Exit(1) } + var configuratorGatewayCreator remote_configuration.ApiCreator = func(cbContainersCluster *operatorcontainerscarbonblackiov1.CBContainersAgent, accessToken string) (remote_configuration.ApiGateway, error) { + return gateway.NewDefaultGatewayCreator().CreateGateway(cbContainersCluster, accessToken) + } + + applier := remote_configuration.NewConfigurator( + k8sClient, + configuratorGatewayCreator, + log, + operator.NewSecretAccessTokenProvider(k8sClient), + operatorVersion, + operatorNamespace, + clusterIdentifier, + ) + applierController := controllers.NewRemoteConfigurationController(applier, log) + + var wg sync.WaitGroup + wg.Add(2) + + signalsContext := ctrl.SetupSignalHandler() + go func() { + defer wg.Done() + + setupLog.Info("starting manager") + if err := mgr.Start(signalsContext); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } + }() + go func() { + defer wg.Done() + + // TODO: Remove once the feature is ready for go-live + enableConfigurator := os.Getenv("ENABLE_REMOTE_CONFIGURATOR") + if enableConfigurator == "true" { + setupLog.Info("Starting remote configurator") + applierController.RunLoop(signalsContext) + } + }() + + wg.Wait() } func extractConfigurationVariables(mgr manager.Manager) (clusterIdentifier string, k8sVersion string) {