Skip to content

Commit

Permalink
Merge pull request #109 from sergeykostov/skostov-KRS-1086
Browse files Browse the repository at this point in the history 
Skostov krs 1086
  • Loading branch information
@sergeykostov
sergeykostov authored Jun 23, 2022
2 parents b976a48 + 86b5700 commit a26dd0a
Show file tree
Hide file tree
Showing 7 changed files with 166 additions and 157 deletions.
4 changes: 4 additions & 0 deletions api/v1/runtime_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ type CBContainersRuntimeResolverSpec struct {
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// +kubebuilder:default:=<>
Affinity *coreV1.Affinity `json:"affinity,omitempty"`
// +kubebuilder:default:="info"
LogLevel string `json:"logLevel,omitempty"`
}

type CBContainersRuntimeSensorSpec struct {
Expand All @@ -48,6 +50,8 @@ type CBContainersRuntimeSensorSpec struct {
Prometheus CBContainersPrometheusSpec `json:"prometheus,omitempty"`
// +kubebuilder:default:=2
VerbosityLevel *int `json:"verbosity_level,omitempty"`
// +kubebuilder:default:="info"
LogLevel string `json:"logLevel,omitempty"`
}

// CBContainersRuntimeProtectionSpec defines the desired state of CBContainersRuntime
Expand Down
70 changes: 34 additions & 36 deletions cbcontainers/state/components/runtime_resolver_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,34 +77,35 @@ func (obj *ResolverDeploymentK8sObject) MutateK8sObject(k8sObject client.Object,
deployment.Spec.Template.Spec.ServiceAccountName = commonState.RuntimeResolverServiceAccountName
deployment.Spec.Template.Spec.PriorityClassName = commonState.DataPlanePriorityClassName
deployment.Spec.Template.Spec.ImagePullSecrets = []coreV1.LocalObjectReference{{Name: commonState.RegistrySecretName}}
obj.mutateAnnotations(deployment, resolver)
obj.mutateVolumes(&deployment.Spec.Template.Spec)
obj.mutateAffinityAndNodeSelector(&deployment.Spec.Template.Spec, resolver)
obj.mutateContainersList(&deployment.Spec.Template.Spec,
resolver,
&agentSpec.Gateways.RuntimeEventsGateway,
agentSpec.Version,
agentSpec.AccessTokenSecretName,
runtimeProtection.InternalGrpcPort,
)

obj.mutateAnnotations(deployment, agentSpec)
obj.mutateVolumes(deployment, agentSpec)
obj.mutateAffinityAndNodeSelector(deployment, agentSpec)
obj.mutateContainersList(deployment, agentSpec)

return nil
}

func (obj *ResolverDeploymentK8sObject) mutateVolumes(templatePodSpec *coreV1.PodSpec) {
func (obj *ResolverDeploymentK8sObject) mutateVolumes(deployment *appsV1.Deployment, agentSpec *cbContainersV1.CBContainersAgentSpec) {
templatePodSpec := &deployment.Spec.Template.Spec
if templatePodSpec.Volumes == nil || len(templatePodSpec.Volumes) != 1 {
templatePodSpec.Volumes = make([]coreV1.Volume, 0)
}

commonState.MutateVolumesToIncludeRootCAsVolume(templatePodSpec)
}

func (obj *ResolverDeploymentK8sObject) mutateAffinityAndNodeSelector(templatePodSpec *coreV1.PodSpec, resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec) {
func (obj *ResolverDeploymentK8sObject) mutateAffinityAndNodeSelector(deployment *appsV1.Deployment, agentSpec *cbContainersV1.CBContainersAgentSpec) {
resolverSpec := &agentSpec.Components.RuntimeProtection.Resolver

templatePodSpec := &deployment.Spec.Template.Spec
templatePodSpec.Affinity = resolverSpec.Affinity
templatePodSpec.NodeSelector = resolverSpec.NodeSelector
}

func (obj *ResolverDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Deployment, resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec) {
func (obj *ResolverDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Deployment, agentSpec *cbContainersV1.CBContainersAgentSpec) {
resolverSpec := &agentSpec.Components.RuntimeProtection.Resolver

if deployment.ObjectMeta.Annotations == nil {
deployment.ObjectMeta.Annotations = make(map[string]string)
}
Expand All @@ -123,58 +124,55 @@ func (obj *ResolverDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Dep
}

func (obj *ResolverDeploymentK8sObject) mutateContainersList(
templatePodSpec *coreV1.PodSpec,
resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec,
eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec,
version,
accessTokenSecretName string,
desiredGRPCPortValue int32) {
deployment *appsV1.Deployment,
agentSpec *cbContainersV1.CBContainersAgentSpec) {

templatePodSpec := &deployment.Spec.Template.Spec
if len(templatePodSpec.Containers) != 1 {
container := coreV1.Container{}
templatePodSpec.Containers = []coreV1.Container{container}
}

obj.mutateContainer(&templatePodSpec.Containers[0], resolverSpec, eventsGatewaySpec,
version, accessTokenSecretName, desiredGRPCPortValue)
obj.mutateContainer(&templatePodSpec.Containers[0], agentSpec)
}

func (obj *ResolverDeploymentK8sObject) mutateContainer(
container *coreV1.Container,
resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec,
eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec,
version,
accessTokenSecretName string,
desiredGRPCPortValue int32) {
agentSpec *cbContainersV1.CBContainersAgentSpec) {

resolverSpec := &agentSpec.Components.RuntimeProtection.Resolver

container.Name = ResolverName
container.Resources = resolverSpec.Resources
commonState.MutateImage(container, resolverSpec.Image, version)
commonState.MutateImage(container, resolverSpec.Image, agentSpec.Version)
commonState.MutateContainerHTTPProbes(container, resolverSpec.Probes)
obj.mutateEnvVars(container, resolverSpec, eventsGatewaySpec, accessTokenSecretName, desiredGRPCPortValue)
obj.mutateContainerPorts(container, desiredGRPCPortValue)
obj.mutateEnvVars(container, agentSpec)
obj.mutateContainerPorts(container, agentSpec)
obj.mutateSecurityContext(container)
obj.mutateVolumesMounts(container)
}

func (obj *ResolverDeploymentK8sObject) mutateContainerPorts(container *coreV1.Container, desiredGRPCPortValue int32) {
func (obj *ResolverDeploymentK8sObject) mutateContainerPorts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
if container.Ports == nil || len(container.Ports) != 1 {
container.Ports = []coreV1.ContainerPort{{}}
}

container.Ports[0].Name = desiredDeploymentGRPCPortName
container.Ports[0].ContainerPort = desiredGRPCPortValue
container.Ports[0].ContainerPort = agentSpec.Components.RuntimeProtection.InternalGrpcPort
}

func (obj *ResolverDeploymentK8sObject) mutateEnvVars(
container *coreV1.Container,
resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec,
eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec,
accessTokenSecretName string,
desiredGRPCPortValue int32) {
container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

runtimeProtection := &agentSpec.Components.RuntimeProtection
resolverSpec := &runtimeProtection.Resolver
desiredGRPCPortValue := runtimeProtection.InternalGrpcPort
eventsGatewaySpec := &agentSpec.Gateways.RuntimeEventsGateway
accessTokenSecretName := agentSpec.AccessTokenSecretName

customEnvs := []coreV1.EnvVar{
{Name: "RUNTIME_KUBERNETES_RESOLVER_GRPC_PORT", Value: fmt.Sprintf("%d", desiredGRPCPortValue)},
{Name: "RUNTIME_KUBERNETES_RESOLVER_LOG_LEVEL", Value: runtimeProtection.Resolver.LogLevel},
{Name: "RUNTIME_KUBERNETES_RESOLVER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", resolverSpec.Prometheus.Port)},
{Name: "RUNTIME_KUBERNETES_RESOLVER_PROBES_PORT", Value: fmt.Sprintf("%d", resolverSpec.Probes.Port)},
{Name: "RUNTIME_KUBERNETES_RESOLVER_INITIALIZATION_TIMEOUT_MINUTES", Value: fmt.Sprintf("%d", desiredInitializationTimeoutMinutes)},
Expand Down
98 changes: 43 additions & 55 deletions cbcontainers/state/components/sensor_daemon_set.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,11 @@ const (
ClusterScanningContainerName = "cbcontainers-cluster-scanner"
daemonSetLabelKey = "app.kubernetes.io/name"

runtimeSensorVerbosityFlag = "-v"
runtimeSensorRunCommand = "/run_sensor.sh"
defaultDnsPolicy = coreV1.DNSClusterFirst
runtimeSensorDNSPolicy = coreV1.DNSClusterFirstWithHostNet
runtimeSensorHostNetwork = true
runtimeSensorHostPID = true
runtimeSensorRunCommand = "/run_sensor.sh"
defaultDnsPolicy = coreV1.DNSClusterFirst
runtimeSensorDNSPolicy = coreV1.DNSClusterFirstWithHostNet
runtimeSensorHostNetwork = true
runtimeSensorHostPID = true

desiredConnectionTimeoutSeconds = 60

Expand Down Expand Up @@ -95,12 +94,7 @@ func (obj *SensorDaemonSetK8sObject) MutateK8sObject(k8sObject client.Object, ag
obj.mutateAnnotations(daemonSet, agentSpec)
obj.mutateVolumes(daemonSet, agentSpec)
obj.mutateTolerations(daemonSet, agentSpec)
obj.mutateContainersList(&daemonSet.Spec.Template.Spec,
agentSpec,
agentSpec.Version,
agentSpec.AccessTokenSecretName,
runtimeProtection.InternalGrpcPort,
)
obj.mutateContainersList(daemonSet, agentSpec)

return nil
}
Expand Down Expand Up @@ -181,16 +175,13 @@ func (obj *SensorDaemonSetK8sObject) mutateTolerations(daemonSet *appsV1.DaemonS
daemonSet.Spec.Template.Spec.Tolerations = agentSpec.Components.Settings.DaemonSetsTolerations
}

func (obj *SensorDaemonSetK8sObject) mutateContainersList(
templatePodSpec *coreV1.PodSpec,
agentSpec *cbContainersV1.CBContainersAgentSpec,
version,
accessTokenSecretName string,
desiredGRPCPortValue int32) {
func (obj *SensorDaemonSetK8sObject) mutateContainersList(daemonSet *appsV1.DaemonSet, agentSpec *cbContainersV1.CBContainersAgentSpec) {

var runtimeContainer coreV1.Container
var clusterScannerContainer coreV1.Container

templatePodSpec := daemonSet.Spec.Template.Spec

desiredContainers := make([]coreV1.Container, 0, 2)
runtimeEnabled := false
clusterScannerEnabled := false
Expand Down Expand Up @@ -228,23 +219,22 @@ func (obj *SensorDaemonSetK8sObject) mutateContainersList(
if commonState.IsEnabled(agentSpec.Components.RuntimeProtection.Enabled) {
obj.mutateRuntimeContainer(
&templatePodSpec.Containers[obj.findContainerLocationByName(templatePodSpec.Containers, RuntimeContainerName)],
&agentSpec.Components.RuntimeProtection.Sensor, version, desiredGRPCPortValue)
agentSpec)
}

if commonState.IsEnabled(agentSpec.Components.ClusterScanning.Enabled) {
obj.mutateClusterScannerContainer(
&templatePodSpec.Containers[obj.findContainerLocationByName(templatePodSpec.Containers, ClusterScanningContainerName)],
&agentSpec.Components.ClusterScanning.ClusterScannerAgent, version, accessTokenSecretName,
&agentSpec.Gateways.HardeningEventsGateway)
agentSpec)
}
}

func (obj *SensorDaemonSetK8sObject) isStateChanged(actualContainersLength, desiredContainersLength int, runtimeEnabled, clusterScannerEnabled, runtimeMissing, clusterScannerMissing bool) bool {
// the actual containers length is different then the desired containers length.
// actual containers' length is different from desired containers' length.
// test cases
// there are more containers then the 2 allowed
// there are more containers than the 2 allowed
// there 0 containers when at least one component should be enabled
// the are different components amount then the desired count.
// there are different components amount then the desired count.
if actualContainersLength != desiredContainersLength {
return true
}
Expand Down Expand Up @@ -272,35 +262,34 @@ func (obj *SensorDaemonSetK8sObject) findContainerLocationByName(containers []co
return -1
}

func (obj *SensorDaemonSetK8sObject) mutateRuntimeContainer(
container *coreV1.Container,
sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec,
version string,
desiredGRPCPortValue int32) {
func (obj *SensorDaemonSetK8sObject) mutateRuntimeContainer(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
sensorSpec := &agentSpec.Components.RuntimeProtection.Sensor

container.Name = RuntimeContainerName
container.Resources = sensorSpec.Resources
container.Args = []string{runtimeSensorVerbosityFlag, fmt.Sprintf("%d", *sensorSpec.VerbosityLevel)}
container.Command = []string{runtimeSensorRunCommand}
commonState.MutateImage(container, sensorSpec.Image, version)
commonState.MutateImage(container, sensorSpec.Image, agentSpec.Version)
commonState.MutateContainerFileProbes(container, sensorSpec.Probes)
if commonState.IsEnabled(sensorSpec.Prometheus.Enabled) {
container.Ports = []coreV1.ContainerPort{{Name: "metrics", ContainerPort: int32(sensorSpec.Prometheus.Port)}}
}
obj.mutateRuntimeEnvVars(container, sensorSpec, desiredGRPCPortValue)
obj.mutateSecurityContext(container)
obj.mutateRuntimeEnvVars(container, agentSpec)
obj.mutateSecurityContext(container, agentSpec)
}

func (obj *SensorDaemonSetK8sObject) mutateRuntimeEnvVars(
container *coreV1.Container,
sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec,
desiredGRPCPortValue int32) {
func (obj *SensorDaemonSetK8sObject) mutateRuntimeEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

sensorSpec := &agentSpec.Components.RuntimeProtection.Sensor
runtimeProtection := &agentSpec.Components.RuntimeProtection
desiredGRPCPortValue := runtimeProtection.InternalGrpcPort

customEnvs := []coreV1.EnvVar{
{Name: "RUNTIME_KUBERNETES_SENSOR_GRPC_PORT", Value: fmt.Sprintf("%d", desiredGRPCPortValue)},
{Name: "RUNTIME_KUBERNETES_SENSOR_RESOLVER_ADDRESS", Value: resolverAddress},
{Name: "RUNTIME_KUBERNETES_SENSOR_RESOLVER_CONNECTION_TIMEOUT_SECONDS", Value: fmt.Sprintf("%d", desiredConnectionTimeoutSeconds)},
{Name: "RUNTIME_KUBERNETES_SENSOR_LIVENESS_PATH", Value: sensorSpec.Probes.LivenessPath},
{Name: "RUNTIME_KUBERNETES_SENSOR_READINESS_PATH", Value: sensorSpec.Probes.ReadinessPath},
{Name: "RUNTIME_KUBERNETES_SENSOR_LOG_LEVEL", Value: runtimeProtection.Sensor.LogLevel},
}

envVarBuilder := commonState.NewEnvVarBuilder().
Expand All @@ -309,7 +298,7 @@ func (obj *SensorDaemonSetK8sObject) mutateRuntimeEnvVars(
commonState.MutateEnvVars(container, envVarBuilder)
}

func (obj *SensorDaemonSetK8sObject) mutateSecurityContext(container *coreV1.Container) {
func (obj *SensorDaemonSetK8sObject) mutateSecurityContext(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
if container.SecurityContext == nil {
container.SecurityContext = &coreV1.SecurityContext{}
}
Expand All @@ -318,27 +307,26 @@ func (obj *SensorDaemonSetK8sObject) mutateSecurityContext(container *coreV1.Con
container.SecurityContext.RunAsUser = &sensorRunAsUser
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerContainer(
container *coreV1.Container,
clusterScannerSpec *cbContainersV1.CBContainersClusterScannerAgentSpec,
version string,
accessTokenSecretName string,
eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec) {
func (obj *SensorDaemonSetK8sObject) mutateClusterScannerContainer(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent

container.Name = ClusterScanningContainerName
container.Resources = clusterScannerSpec.Resources
commonState.MutateImage(container, clusterScannerSpec.Image, version)
commonState.MutateImage(container, clusterScannerSpec.Image, agentSpec.Version)
commonState.MutateContainerFileProbes(container, clusterScannerSpec.Probes)
if commonState.IsEnabled(clusterScannerSpec.Prometheus.Enabled) {
container.Ports = []coreV1.ContainerPort{{Name: "metrics", ContainerPort: int32(clusterScannerSpec.Prometheus.Port)}}
}
obj.mutateClusterScannerEnvVars(container, clusterScannerSpec, accessTokenSecretName, eventsGatewaySpec)
obj.mutateClusterScannerVolumesMounts(container, clusterScannerSpec)
obj.mutateSecurityContext(container)
obj.mutateClusterScannerEnvVars(container, agentSpec)
obj.mutateClusterScannerVolumesMounts(container, agentSpec)
obj.mutateSecurityContext(container, agentSpec)
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container,
clusterScannerSpec *cbContainersV1.CBContainersClusterScannerAgentSpec,
accessTokenSecretName string, eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec) {
func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {

clusterScannerSpec := &agentSpec.Components.ClusterScanning.ClusterScannerAgent

customEnvs := []coreV1.EnvVar{
{Name: "CLUSTER_SCANNER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", clusterScannerSpec.Prometheus.Port)},
{Name: "CLUSTER_SCANNER_IMAGE_SCANNING_REPORTER_HOST", Value: imageScanningReporterAddress},
Expand All @@ -354,8 +342,8 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerEnvVars(container *core
}

envVarBuilder := commonState.NewEnvVarBuilder().
WithCommonDataPlane(accessTokenSecretName).
WithEventsGateway(eventsGatewaySpec).
WithCommonDataPlane(agentSpec.AccessTokenSecretName).
WithEventsGateway(&agentSpec.Gateways.HardeningEventsGateway).
WithCustom(customEnvs...).
WithEnvVarFromResource("CLUSTER_SCANNER_LIMITS_MEMORY", ClusterScanningContainerName, "limits.memory").
WithEnvVarFromResource("CLUSTER_SCANNER_REQUESTS_MEMORY", ClusterScanningContainerName, "requests.memory").
Expand Down Expand Up @@ -385,8 +373,8 @@ func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumes(templatePodSpec
}
}

func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, clusterScannerSpec *cbContainersV1.CBContainersClusterScannerAgentSpec) {
containerRuntimes := getContainerRuntimes(clusterScannerSpec)
func (obj *SensorDaemonSetK8sObject) mutateClusterScannerVolumesMounts(container *coreV1.Container, agentSpec *cbContainersV1.CBContainersAgentSpec) {
containerRuntimes := getContainerRuntimes(&agentSpec.Components.ClusterScanning.ClusterScannerAgent)

if container.VolumeMounts == nil || len(container.VolumeMounts) != len(containerRuntimes)+1 {
container.VolumeMounts = make([]coreV1.VolumeMount, 0)
Expand Down
8 changes: 8 additions & 0 deletions config/crd/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4514,6 +4514,10 @@ spec:
resolver:
default: {}
properties:
logLevel:
default: "info"
format: byte
type: string
affinity:
default: {}
description: Affinity is a group of affinity scheduling
Expand Down Expand Up @@ -5597,6 +5601,10 @@ spec:
sensor:
default: {}
properties:
logLevel:
default: "info"
format: byte
type: string
daemonSetAnnotations:
additionalProperties:
type: string
Expand Down
6 changes: 6 additions & 0 deletions config/crd_v1beta1/bases/operator.containers.carbonblack.io_cbcontainersagents.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4216,6 +4216,9 @@ spec:
type: integer
resolver:
properties:
logLevel:
format: byte
type: string
affinity:
description: Affinity is a group of affinity scheduling
rules.
Expand Down Expand Up @@ -5232,6 +5235,9 @@ spec:
type: object
sensor:
properties:
logLevel:
format: byte
type: string
daemonSetAnnotations:
additionalProperties:
type: string
Expand Down
Loading

0 comments on commit a26dd0a

Please sign in to comment.