Stress: build our own custom otel collector (#7900)

Build our own custom otel distribution using their official `otb` tool.

It reduces the amount of code significantly by cutting it down to azure monitor and removing all other contrib components. As a bonus we can also control the Docker images used, which is helpful for other things.

tools/stress-cluster/services/otelcollector/Dockerfile

@@ -1,8 +1,18 @@
-FROM mcr.microsoft.com/oss/otel/opentelemetry-collector-contrib:0.94.0 as otel
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21 as build
+RUN apt update -y && apt upgrade -y && apt install -y build-essential git
 
-FROM busybox:latest as busybox
-COPY --from=otel / /
-ADD ./otel-collector-config.yml /otel-collector-config.yml
+# this lets us build a custom otel-collector image, with a minimal set of 
+# adapters, etc...
+RUN go install go.opentelemetry.io/collector/cmd/builder@latest
+COPY ./otel-builder.yml /otel-builder.yml
+# builds to /tmp/dist
+RUN builder --config=/otel-builder.yml
+
+# this is the actual image we'll upload - it's got a shell so we can
+# run startup.sh but is otherwise pristine.
+FROM mcr.microsoft.com/cbl-mariner/busybox:2.0
+COPY --from=build /tmp/dist/otelcol-custom /otelcol-custom
+COPY ./otel-collector-config.yml /otel-collector-config.yml
 ADD ./startup.sh /startup.sh
 
 EXPOSE 4317

tools/stress-cluster/services/otelcollector/README.md

@@ -0,0 +1,16 @@
+# OpenTelemetry custom distro for stress testing
+
+This `Dockerfile` builds the image we use for our OpenTelemetry collector in the stress testing cluster.
+
+It includes only the Azure Monitor and 'debug' exporters, which shrinks it down quite a bit and uses only mariner-based images, for both building and for the final app image.
+
+To test this out locally:
+
+1. Create a .env file that looks like this:
+
+   ```bash
+   # make sure you bring in the quotes
+   APPLICATIONINSIGHTS_CONNECTION_STRING='<appinsights connection string from the Azure portal>'
+   ```
+
+2. Run ./localtest.sh

tools/stress-cluster/services/otelcollector/localtest.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
 
-docker build -t oteltest .
+set -ex
+
+docker build --no-cache -t oteltest .
+
 docker run -it \
     -e ENV_FILE=/.env \
     -v `pwd`/.env:/.env \
+    -P \
     oteltest
 

tools/stress-cluster/services/otelcollector/otel-builder.yml

@@ -0,0 +1,16 @@
+# This is used as an input to the `ocb` app that lets you build a customized OpenTelemetry collector, 
+# as described here: https://opentelemetry.io/docs/collector/custom-collector/
+dist:
+  name: otelcol-custom
+  description: Local OpenTelemetry Collector binary
+  output_path: /tmp/dist
+  otelcol_version: 0.96.0
+exporters:
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/azuremonitorexporter v0.96.0
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.96.0
+
+receivers:
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.96.0
+
+processors:
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.96.0

tools/stress-cluster/services/otelcollector/otel-collector-config.yml

@@ -11,18 +11,18 @@ service:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [logging, azuremonitor]
+      exporters: [debug, azuremonitor]
     metrics:
       receivers: [otlp]
       processors: [batch]
       exporters: [azuremonitor]
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [logging, azuremonitor]
+      exporters: [debug, azuremonitor]
 
 exporters:
-  logging:
+  debug:
     verbosity: "${env:OTEL_LOG_LEVEL}"
   azuremonitor:
     connection_string: "${env:APPLICATIONINSIGHTS_CONNECTION_STRING}"

tools/stress-cluster/services/otelcollector/startup.sh

@@ -4,4 +4,4 @@ set -ex
 
 source $ENV_FILE
 export APPLICATIONINSIGHTS_CONNECTION_STRING
-/otelcol-contrib --config otel-collector-config.yml $@
+/otelcol-custom --config otel-collector-config.yml $@