Inject otelcol sidecar into any namespace

Signed-off-by: Pavol Loffay <[email protected]>

.chloggen/inject-any-namespace.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Support sidecar injecton into any namespace.
+
+# One or more tracking issues related to the change
+issues: [199]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

README.md

@@ -87,7 +87,7 @@ The `CustomResource` for the `OpenTelemetryCollector` exposes a property named `
 
 #### Sidecar injection
 
-A sidecar with the OpenTelemetry Collector can be injected into pod-based workloads by setting the pod annotation `sidecar.opentelemetry.io/inject` to either `"true"`, or to the name of a concrete `OpenTelemetryCollector` from the same namespace, like in the following example:
+A sidecar with the OpenTelemetry Collector can be injected into pod-based workloads by setting the pod annotation `sidecar.opentelemetry.io/inject` to either `"true"`, or to the name of a concrete `OpenTelemetryCollector`, like in the following example:
 
 ```yaml
 kubectl apply -f - <<EOF
@@ -139,6 +139,13 @@ The annotation value can come either from the namespace, or from the pod. The mo
 * the pod annotation is used when it's set to a concrete instance name or to `"false"`
 * namespace annotation is used when the pod annotation is either absent or set to `"true"`, and the namespace is set to a concrete instance or to `"false"`
 
+The possible values for the annotation can be:
+
+* "true" - inject `OpenTelemetryCollector` resource from the namespace.
+* "sidecar-for-my-app" - name of `OpenTelemetryCollector` CR instance in the current namespace.
+* "my-other-namespace/my-instrumentation" - name and namespace of `OpenTelemetryCollector` CR instance in another namespace.
+* "false" - do not inject
+
 When using a pod-based workload, such as `Deployment` or `Statefulset`, make sure to add the annotation to the `PodTemplate` part. Like:
 
 ```yaml

config/manager/kustomization.yaml

@@ -1,2 +1,2 @@
 resources:
-- manager.yaml
+- manager.yaml

internal/config/main.go

@@ -41,6 +41,7 @@ type Config struct {
 	autoInstrumentationPythonImage string
 	collectorImage                 string
 	collectorConfigMapEntry        string
+	sidecarConfigPrepperImage      string
 	autoInstrumentationDotNetImage string
 	targetAllocatorConfigMapEntry  string
 	autoInstrumentationNodeJSImage string
@@ -74,6 +75,7 @@ func New(opts ...Option) Config {
 		autoDetectFrequency:            o.autoDetectFrequency,
 		collectorImage:                 o.collectorImage,
 		collectorConfigMapEntry:        o.collectorConfigMapEntry,
+		sidecarConfigPrepperImage:      o.sidecarConfigPrepperImage,
 		targetAllocatorImage:           o.targetAllocatorImage,
 		targetAllocatorConfigMapEntry:  o.targetAllocatorConfigMapEntry,
 		logger:                         o.logger,
@@ -145,6 +147,11 @@ func (c *Config) CollectorConfigMapEntry() string {
 	return c.collectorConfigMapEntry
 }
 
+// SidecarConfigPrepperImage represents the flag to override the OpenTelemetry Collector sidecar config prepper container image.
+func (c *Config) SidecarConfigPrepperImage() string {
+	return c.sidecarConfigPrepperImage
+}
+
 // TargetAllocatorImage represents the flag to override the OpenTelemetry TargetAllocator container image.
 func (c *Config) TargetAllocatorImage() string {
 	return c.targetAllocatorImage

internal/config/main_test.go

@@ -32,12 +32,14 @@ func TestNewConfig(t *testing.T) {
 	cfg := config.New(
 		config.WithCollectorImage("some-image"),
 		config.WithCollectorConfigMapEntry("some-config.yaml"),
+		config.WithSidecarConfigPrepperImage("prepperimage"),
 		config.WithPlatform(platform.Kubernetes),
 	)
 
 	// test
 	assert.Equal(t, "some-image", cfg.CollectorImage())
 	assert.Equal(t, "some-config.yaml", cfg.CollectorConfigMapEntry())
+	assert.Equal(t, "prepperimage", cfg.SidecarConfigPrepperImage())
 	assert.Equal(t, platform.Kubernetes, cfg.Platform())
 }
 

internal/config/options.go

@@ -39,6 +39,7 @@ type options struct {
 	autoInstrumentationPythonImage string
 	collectorImage                 string
 	collectorConfigMapEntry        string
+	sidecarConfigPrepperImage      string
 	targetAllocatorConfigMapEntry  string
 	targetAllocatorImage           string
 	onPlatformChange               changeHandler
@@ -69,6 +70,11 @@ func WithCollectorImage(s string) Option {
 		o.collectorImage = s
 	}
 }
+func WithSidecarConfigPrepperImage(s string) Option {
+	return func(o *options) {
+		o.sidecarConfigPrepperImage = s
+	}
+}
 func WithCollectorConfigMapEntry(s string) Option {
 	return func(o *options) {
 		o.collectorConfigMapEntry = s

internal/webhookhandler/webhookhandler_test.go

@@ -163,12 +163,13 @@ func TestShouldInjectSidecar(t *testing.T) {
 			// verify
 			assert.True(t, res.Allowed)
 			assert.Nil(t, res.AdmissionResponse.Result)
-			assert.Len(t, res.Patches, 3)
+			assert.Len(t, res.Patches, 4)
 
 			expectedMap := map[string]bool{
-				"/metadata/labels": false, // add a new label
-				"/spec/volumes":    false, // add a new volume with the configmap
-				"/spec/containers": false, // replace the containers, adding one new container
+				"/metadata/labels":     false, // add a new label
+				"/spec/volumes":        false, // add a new volume with the configmap
+				"/spec/containers":     false, // replace the containers, adding one new container
+				"/spec/initContainers": false, // replace the containers, adding one new container
 			}
 			for _, patch := range res.Patches {
 				// quick and dirty solution

main.go

@@ -85,6 +85,7 @@ func main() {
 		probeAddr                 string
 		enableLeaderElection      bool
 		collectorImage            string
+		sidecarConfigPrepperImage string
 		targetAllocatorImage      string
 		autoInstrumentationJava   string
 		autoInstrumentationNodeJS string
@@ -101,6 +102,7 @@ func main() {
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	pflag.StringVar(&collectorImage, "collector-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector), "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.")
+	pflag.StringVar(&sidecarConfigPrepperImage, "sidecar-config-prepper-image", "alpine:latest", "\"The default container image that is used to prepare config file for the collector config. The image uses sh, echo, base64 and cat commands")
 	pflag.StringVar(&targetAllocatorImage, "target-allocator-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/target-allocator:%s", v.TargetAllocator), "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.")
 	pflag.StringVar(&autoInstrumentationJava, "auto-instrumentation-java-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:%s", v.AutoInstrumentationJava), "The default OpenTelemetry Java instrumentation image. This image is used when no image is specified in the CustomResource.")
 	pflag.StringVar(&autoInstrumentationNodeJS, "auto-instrumentation-nodejs-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:%s", v.AutoInstrumentationNodeJS), "The default OpenTelemetry NodeJS instrumentation image. This image is used when no image is specified in the CustomResource.")
@@ -143,6 +145,7 @@ func main() {
 		config.WithLogger(ctrl.Log.WithName("config")),
 		config.WithVersion(v),
 		config.WithCollectorImage(collectorImage),
+		config.WithSidecarConfigPrepperImage(sidecarConfigPrepperImage),
 		config.WithTargetAllocatorImage(targetAllocatorImage),
 		config.WithAutoInstrumentationJavaImage(autoInstrumentationJava),
 		config.WithAutoInstrumentationNodeJSImage(autoInstrumentationNodeJS),

pkg/collector/reconcile/config_replace.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/install" // Package install has the side-effect of registering all builtin.
 	"gopkg.in/yaml.v2"
 
+	"github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector/adapters"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/naming"
 	ta "github.com/open-telemetry/opentelemetry-operator/pkg/targetallocator/adapters"
@@ -34,16 +35,16 @@ type Config struct {
 	PromConfig *promconfig.Config `yaml:"config"`
 }
 
-func ReplaceConfig(params Params) (string, error) {
-	if !params.Instance.Spec.TargetAllocator.Enabled {
-		return params.Instance.Spec.Config, nil
+func ReplaceConfig(instance v1alpha1.OpenTelemetryCollector) (string, error) {
+	if !instance.Spec.TargetAllocator.Enabled {
+		return instance.Spec.Config, nil
 	}
-	config, getStringErr := adapters.ConfigFromString(params.Instance.Spec.Config)
+	config, getStringErr := adapters.ConfigFromString(instance.Spec.Config)
 	if getStringErr != nil {
 		return "", getStringErr
 	}
 
-	promCfgMap, getCfgPromErr := ta.ConfigToPromConfig(params.Instance.Spec.Config)
+	promCfgMap, getCfgPromErr := ta.ConfigToPromConfig(instance.Spec.Config)
 	if getCfgPromErr != nil {
 		return "", getCfgPromErr
 	}
@@ -65,7 +66,7 @@ func ReplaceConfig(params Params) (string, error) {
 		escapedJob := url.QueryEscape(cfg.PromConfig.ScrapeConfigs[i].JobName)
 		cfg.PromConfig.ScrapeConfigs[i].ServiceDiscoveryConfigs = discovery.Configs{
 			&http.SDConfig{
-				URL: fmt.Sprintf("http://%s:80/jobs/%s/targets?collector_id=$POD_NAME", naming.TAService(params.Instance), escapedJob),
+				URL: fmt.Sprintf("http://%s:80/jobs/%s/targets?collector_id=$POD_NAME", naming.TAService(instance), escapedJob),
 			},
 		}
 	}

pkg/collector/reconcile/config_replace_test.go

@@ -29,7 +29,7 @@ func TestPrometheusParser(t *testing.T) {
 	assert.NoError(t, err)
 
 	t.Run("should update config with http_sd_config", func(t *testing.T) {
-		actualConfig, err := ReplaceConfig(param)
+		actualConfig, err := ReplaceConfig(param.Instance)
 		assert.NoError(t, err)
 
 		// prepare
@@ -63,7 +63,7 @@ func TestPrometheusParser(t *testing.T) {
 
 	t.Run("should not update config with http_sd_config", func(t *testing.T) {
 		param.Instance.Spec.TargetAllocator.Enabled = false
-		actualConfig, err := ReplaceConfig(param)
+		actualConfig, err := ReplaceConfig(param.Instance)
 		assert.NoError(t, err)
 
 		// prepare

pkg/collector/reconcile/configmap.go

@@ -74,7 +74,7 @@ func desiredConfigMap(_ context.Context, params Params) corev1.ConfigMap {
 	} else {
 		labels["app.kubernetes.io/version"] = "latest"
 	}
-	config, err := ReplaceConfig(params)
+	config, err := ReplaceConfig(params.Instance)
 	if err != nil {
 		params.Log.V(2).Info("failed to update prometheus config to use sharded targets: ", err)
 	}

pkg/sidecar/pod.go

@@ -16,6 +16,7 @@
 package sidecar
 
 import (
+	"encoding/base64"
 	"fmt"
 
 	"github.com/go-logr/logr"
@@ -24,6 +25,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1"
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/collector"
+	"github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile"
 	"github.com/open-telemetry/opentelemetry-operator/pkg/naming"
 )
 
@@ -33,8 +35,39 @@ const (
 
 // add a new sidecar container to the given pod, based on the given OpenTelemetryCollector.
 func add(cfg config.Config, logger logr.Logger, otelcol v1alpha1.OpenTelemetryCollector, pod corev1.Pod, attributes []corev1.EnvVar) (corev1.Pod, error) {
-	// add the container
 	volumes := collector.Volumes(cfg, otelcol)
+	volumes[0] = corev1.Volume{
+		Name: naming.ConfigMapVolume(),
+		VolumeSource: corev1.VolumeSource{
+			EmptyDir: &corev1.EmptyDirVolumeSource{},
+		},
+	}
+	volumeMount := corev1.VolumeMount{
+		Name:      naming.ConfigMapVolume(),
+		MountPath: "/conf",
+	}
+
+	otelColCfg, err := reconcile.ReplaceConfig(otelcol)
+	if err != nil {
+		return pod, err
+	}
+	otelColCfgStr := base64.StdEncoding.EncodeToString([]byte(otelColCfg))
+
+	// add the container
+	pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
+		Name:    "otc-container-config-prepper",
+		Image:   cfg.SidecarConfigPrepperImage(),
+		Command: []string{"/bin/sh"},
+		Args:    []string{"-c", "echo ${OTEL_CONFIG} | base64 -d > /conf/collector.yaml && cat /conf/collector.yaml"},
+		Env: []corev1.EnvVar{
+			{
+				Name:  "OTEL_CONFIG",
+				Value: otelColCfgStr,
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{volumeMount},
+	})
+
 	container := collector.Container(cfg, logger, otelcol)
 	if !hasResourceAttributeEnvVar(container.Env) {
 		container.Env = append(container.Env, attributes...)

pkg/sidecar/pod_test.go

@@ -15,9 +15,11 @@
 package sidecar
 
 import (
+	"encoding/base64"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -45,14 +47,41 @@ func TestAddSidecarWhenNoSidecarExists(t *testing.T) {
 			Name:      "otelcol-sample",
 			Namespace: "some-app",
 		},
+		Spec: v1alpha1.OpenTelemetryCollectorSpec{
+			Config: `
+receivers:
+exporters:
+processors:
+`,
+		},
 	}
-	cfg := config.New(config.WithCollectorImage("some-default-image"))
+	cfg := config.New(config.WithCollectorImage("some-default-image"), config.WithSidecarConfigPrepperImage("alpine:latest"))
 
 	// test
 	changed, err := add(cfg, logger, otelcol, pod, nil)
 
 	// verify
 	assert.NoError(t, err)
+	require.Len(t, changed.Spec.InitContainers, 1)
+	cfgBase64 := base64.StdEncoding.EncodeToString([]byte(otelcol.Spec.Config))
+	assert.Equal(t, changed.Spec.InitContainers[0], corev1.Container{
+		Name:    "otc-container-config-prepper",
+		Image:   "alpine:latest",
+		Command: []string{"/bin/sh"},
+		Args:    []string{"-c", "echo ${OTEL_CONFIG} | base64 -d > /conf/collector.yaml && cat /conf/collector.yaml"},
+		Env: []corev1.EnvVar{
+			{
+				Name:  "OTEL_CONFIG",
+				Value: cfgBase64,
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      "otc-internal",
+				MountPath: "/conf",
+			},
+		},
+	})
 	assert.Len(t, changed.Spec.Containers, 2)
 	assert.Len(t, changed.Spec.Volumes, 2)
 	assert.Equal(t, "some-app.otelcol-sample", changed.Labels["sidecar.opentelemetry.io/injected"])

pkg/sidecar/podmutator.go

@@ -106,7 +106,14 @@ func (p *sidecarPodMutator) getCollectorInstance(ctx context.Context, ns corev1.
 	}
 
 	otelcol := v1alpha1.OpenTelemetryCollector{}
-	err := p.client.Get(ctx, types.NamespacedName{Name: ann, Namespace: ns.Name}, &otelcol)
+	var nsnOtelcol types.NamespacedName
+	instNamespace, instName, namespaced := strings.Cut(ann, "/")
+	if namespaced {
+		nsnOtelcol = types.NamespacedName{Name: instName, Namespace: instNamespace}
+	} else {
+		nsnOtelcol = types.NamespacedName{Name: ann, Namespace: ns.Name}
+	}
+	err := p.client.Get(ctx, nsnOtelcol, &otelcol)
 	if err != nil {
 		return otelcol, err
 	}

tests/e2e/instrumentation-dotnet/01-assert.yaml

@@ -49,6 +49,7 @@ spec:
       name: opentelemetry-auto-instrumentation
   - name: otc-container
   initContainers:
+  - name: otc-container-config-prepper
   - name: opentelemetry-auto-instrumentation
 status:
   phase: Running

tests/e2e/instrumentation-java-other-ns/01-assert.yaml

@@ -43,6 +43,7 @@ spec:
       name: opentelemetry-auto-instrumentation
   - name: otc-container
   initContainers:
+  - name: otc-container-config-prepper
   - name: opentelemetry-auto-instrumentation
 status:
   phase: Running

tests/e2e/instrumentation-java/01-assert.yaml

@@ -43,6 +43,7 @@ spec:
       name: opentelemetry-auto-instrumentation
   - name: otc-container
   initContainers:
+  - name: otc-container-config-prepper
   - name: opentelemetry-auto-instrumentation
 status:
   phase: Running

tests/e2e/instrumentation-nodejs/01-assert.yaml

@@ -39,6 +39,7 @@ spec:
       name: opentelemetry-auto-instrumentation
   - name: otc-container
   initContainers:
+  - name: otc-container-config-prepper
   - name: opentelemetry-auto-instrumentation
 status:
   phase: Running

tests/e2e/instrumentation-python/01-assert.yaml

@@ -45,6 +45,7 @@ spec:
       name: opentelemetry-auto-instrumentation
   - name: otc-container
   initContainers:
+  - name: otc-container-config-prepper
   - name: opentelemetry-auto-instrumentation
 status:
   phase: Running