Skip to content

Commit

Permalink
Python auto-instrumentation: handle musl base containers
Browse files Browse the repository at this point in the history
Build and and inject musl based python auto-instrumentation if proper
annotation is configured:

python-wheel-kind: "musllinux"

Refs open-telemetry#2264
  • Loading branch information
xrmx committed Oct 7, 2024
1 parent 91759b0 commit db273cf
Show file tree
Hide file tree
Showing 11 changed files with 401 additions and 17 deletions.
18 changes: 14 additions & 4 deletions autoinstrumentation/python/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# To build one auto-instrumentation image for Python, please:
# - Ensure the packages are installed in the `/autoinstrumentation` directory. This is required as when instrumenting the pod,
# one init container will be created to copy all the content in `/autoinstrumentation` directory to your app's container. Then
# - Ensure the packages are installed in the `/autoinstrumentation,{musl}` directory. This is required as when instrumenting the pod,
# one init container will be created to copy all the content in `/autoinstrumentation{,-musl}` directory to your app's container. Then
# update the `PYTHONPATH` environment variable accordingly. To achieve this, you can mimic the one in `autoinstrumentation/python/Dockerfile`
# by using multi-stage builds. In the first stage, install all the required packages in one custom directory with `pip install --target`.
# Then in the second stage, copy the directory to `/autoinstrumentation`.
# Then in the second stage, copy the directory to `/autoinstrumentation{,-musl}`.
# - Ensure you have `opentelemetry-distro` and `opentelemetry-instrumentation` or your customized alternatives installed.
# Those two packages are essential to Python auto-instrumentation.
# - Grant the necessary access to `/autoinstrumentation` directory. `chmod -R go+r /autoinstrumentation`
# - Grant the necessary access to `/autoinstrumentation{,-musl}` directory. `chmod -R go+r /autoinstrumentation`
# - For auto-instrumentation by container injection, the Linux command cp is
# used and must be availabe in the image.
FROM python:3.11 AS build
Expand All @@ -17,8 +17,18 @@ ADD requirements.txt .

RUN mkdir workspace && pip install --target workspace -r requirements.txt

FROM python:3.11-alpine AS build-musl

WORKDIR /operator-build

ADD requirements.txt .

RUN mkdir workspace && pip install --target workspace -r requirements.txt

FROM busybox

COPY --from=build /operator-build/workspace /autoinstrumentation
COPY --from=build-musl /operator-build/workspace /autoinstrumentation-musl

RUN chmod -R go+r /autoinstrumentation
RUN chmod -R go+r /autoinstrumentation-musl
1 change: 1 addition & 0 deletions pkg/instrumentation/annotation.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const (
annotationInjectNodeJSContainersName = "instrumentation.opentelemetry.io/nodejs-container-names"
annotationInjectPython = "instrumentation.opentelemetry.io/inject-python"
annotationInjectPythonContainersName = "instrumentation.opentelemetry.io/python-container-names"
annotationPythonWheelKind = "instrumentation.opentelemetry.io/python-wheel-kind"
annotationInjectDotNet = "instrumentation.opentelemetry.io/inject-dotnet"
annotationDotNetRuntime = "instrumentation.opentelemetry.io/otel-dotnet-auto-runtime"
annotationInjectDotnetContainersName = "instrumentation.opentelemetry.io/dotnet-container-names"
Expand Down
1 change: 1 addition & 0 deletions pkg/instrumentation/podmutator.go
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c
}
if pm.config.EnablePythonAutoInstrumentation() || inst == nil {
insts.Python.Instrumentation = inst
insts.Python.AdditionalAnnotations = map[string]string{annotationPythonWheelKind: annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationPythonWheelKind)}
} else {
logger.Error(nil, "support for Python auto instrumentation is not enabled")
pm.Recorder.Event(pod.DeepCopy(), "Warning", "InstrumentationRequestRejected", "support for Python auto instrumentation is not enabled")
Expand Down
36 changes: 25 additions & 11 deletions pkg/instrumentation/python.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,22 @@ import (
)

const (
envPythonPath = "PYTHONPATH"
envOtelTracesExporter = "OTEL_TRACES_EXPORTER"
envOtelMetricsExporter = "OTEL_METRICS_EXPORTER"
envOtelExporterOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
pythonPathPrefix = "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation"
pythonPathSuffix = "/otel-auto-instrumentation-python"
pythonInstrMountPath = "/otel-auto-instrumentation-python"
pythonVolumeName = volumeName + "-python"
pythonInitContainerName = initContainerName + "-python"
envPythonPath = "PYTHONPATH"
envOtelTracesExporter = "OTEL_TRACES_EXPORTER"
envOtelMetricsExporter = "OTEL_METRICS_EXPORTER"
envOtelExporterOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
manyLinuxAutoInstrumentationSrc = "/autoinstrumentation/."
muslLinuxAutoInstrumentationSrc = "/autoinstrumentation-musl/."
pythonPathPrefix = "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation"
pythonPathSuffix = "/otel-auto-instrumentation-python"
pythonInstrMountPath = "/otel-auto-instrumentation-python"
pythonVolumeName = volumeName + "-python"
pythonInitContainerName = initContainerName + "-python"
wheelKindManyLinux = "manylinux"
wheelKindMuslLinux = "musllinux"
)

func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (corev1.Pod, error) {
func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int, wheelKind string) (corev1.Pod, error) {
// caller checks if there is at least one container.
container := &pod.Spec.Containers[index]

Expand All @@ -43,6 +47,16 @@ func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (cor
return pod, err
}

autoInstrumentationSrc := ""
switch wheelKind {
case "", wheelKindManyLinux:
autoInstrumentationSrc = manyLinuxAutoInstrumentationSrc
case wheelKindMuslLinux:
autoInstrumentationSrc = muslLinuxAutoInstrumentationSrc
default:
return pod, fmt.Errorf("provided instrumentation.opentelemetry.io/python-wheel-kind annotation value '%s' is not supported", wheelKind)
}

// inject Python instrumentation spec env vars.
for _, env := range pythonSpec.Env {
idx := getIndexOfEnv(container.Env, env.Name)
Expand Down Expand Up @@ -106,7 +120,7 @@ func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (cor
pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
Name: pythonInitContainerName,
Image: pythonSpec.Image,
Command: []string{"cp", "-r", "/autoinstrumentation/.", pythonInstrMountPath},
Command: []string{"cp", "-r", autoInstrumentationSrc, pythonInstrMountPath},
Resources: pythonSpec.Resources,
VolumeMounts: []corev1.VolumeMount{{
Name: pythonVolumeName,
Expand Down
161 changes: 160 additions & 1 deletion pkg/instrumentation/python_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func TestInjectPythonSDK(t *testing.T) {
name string
v1alpha1.Python
pod corev1.Pod
wheelKind string
expected corev1.Pod
err error
}{
Expand All @@ -42,6 +43,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -114,6 +116,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -187,6 +190,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -259,6 +263,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -331,6 +336,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
Expand Down Expand Up @@ -403,6 +409,7 @@ func TestInjectPythonSDK(t *testing.T) {
},
},
},
wheelKind: "manylinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
Expand All @@ -419,11 +426,163 @@ func TestInjectPythonSDK(t *testing.T) {
},
err: fmt.Errorf("the container defines env var value via ValueFrom, envVar: %s", envPythonPath),
},
{
name: "musllinux wheelKind defined",
Python: v1alpha1.Python{Image: "foo/bar:1"},
pod: corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{},
},
},
},
wheelKind: "musllinux",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: pythonVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{
SizeLimit: &defaultVolumeLimitSize,
},
},
},
},
InitContainers: []corev1.Container{
{
Name: "opentelemetry-auto-instrumentation-python",
Image: "foo/bar:1",
Command: []string{"cp", "-r", "/autoinstrumentation-musl/.", "/otel-auto-instrumentation-python"},
VolumeMounts: []corev1.VolumeMount{{
Name: "opentelemetry-auto-instrumentation-python",
MountPath: "/otel-auto-instrumentation-python",
}},
},
},
Containers: []corev1.Container{
{
VolumeMounts: []corev1.VolumeMount{
{
Name: "opentelemetry-auto-instrumentation-python",
MountPath: "/otel-auto-instrumentation-python",
},
},
Env: []corev1.EnvVar{
{
Name: "PYTHONPATH",
Value: fmt.Sprintf("%s:%s", "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation", "/otel-auto-instrumentation-python"),
},
{
Name: "OTEL_EXPORTER_OTLP_PROTOCOL",
Value: "http/protobuf",
},
{
Name: "OTEL_TRACES_EXPORTER",
Value: "otlp",
},
{
Name: "OTEL_METRICS_EXPORTER",
Value: "otlp",
},
},
},
},
},
},
err: nil,
},
{
name: "wheelKind not defined",
Python: v1alpha1.Python{Image: "foo/bar:1"},
pod: corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{},
},
},
},
wheelKind: "",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: pythonVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{
SizeLimit: &defaultVolumeLimitSize,
},
},
},
},
InitContainers: []corev1.Container{
{
Name: "opentelemetry-auto-instrumentation-python",
Image: "foo/bar:1",
Command: []string{"cp", "-r", "/autoinstrumentation/.", "/otel-auto-instrumentation-python"},
VolumeMounts: []corev1.VolumeMount{{
Name: "opentelemetry-auto-instrumentation-python",
MountPath: "/otel-auto-instrumentation-python",
}},
},
},
Containers: []corev1.Container{
{
VolumeMounts: []corev1.VolumeMount{
{
Name: "opentelemetry-auto-instrumentation-python",
MountPath: "/otel-auto-instrumentation-python",
},
},
Env: []corev1.EnvVar{
{
Name: "PYTHONPATH",
Value: fmt.Sprintf("%s:%s", "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation", "/otel-auto-instrumentation-python"),
},
{
Name: "OTEL_EXPORTER_OTLP_PROTOCOL",
Value: "http/protobuf",
},
{
Name: "OTEL_TRACES_EXPORTER",
Value: "otlp",
},
{
Name: "OTEL_METRICS_EXPORTER",
Value: "otlp",
},
},
},
},
},
},
err: nil,
},
{
name: "wheelKind not supported",
Python: v1alpha1.Python{Image: "foo/bar:1"},
pod: corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{},
},
},
},
wheelKind: "not supported",
expected: corev1.Pod{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{},
},
},
},
err: fmt.Errorf("provided instrumentation.opentelemetry.io/python-wheel-kind annotation value 'not supported' is not supported"),
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pod, err := injectPythonSDK(test.Python, test.pod, 0)
pod, err := injectPythonSDK(test.Python, test.pod, 0, test.wheelKind)
assert.Equal(t, test.expected, pod)
assert.Equal(t, test.err, err)
})
Expand Down
2 changes: 1 addition & 1 deletion pkg/instrumentation/sdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func (i *sdkInjector) inject(ctx context.Context, insts languageInstrumentations

for _, container := range insts.Python.Containers {
index := getContainerIndex(container, pod)
pod, err = injectPythonSDK(otelinst.Spec.Python, pod, index)
pod, err = injectPythonSDK(otelinst.Spec.Python, pod, index, insts.Python.AdditionalAnnotations[annotationPythonWheelKind])
if err != nil {
i.logger.Info("Skipping Python SDK injection", "reason", err.Error(), "container", pod.Spec.Containers[index].Name)
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
name: sidecar
spec:
config: |
receivers:
otlp:
protocols:
grpc:
http:
processors:
exporters:
debug:
service:
pipelines:
traces:
receivers: [otlp]
exporters: [debug]
mode: sidecar
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
name: python-musl
spec:
env:
- name: OTEL_EXPORTER_OTLP_TIMEOUT
value: "20"
- name: OTEL_TRACES_SAMPLER
value: parentbased_traceidratio
- name: OTEL_TRACES_SAMPLER_ARG
value: "0.85"
- name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED
value: "true"
exporter:
endpoint: http://localhost:4317
propagators:
- jaeger
- b3
sampler:
type: parentbased_traceidratio
argument: "0.25"
python:
env:
- name: OTEL_LOG_LEVEL
value: "debug"
- name: OTEL_TRACES_EXPORTER
value: otlp
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: http://localhost:4318
Loading

0 comments on commit db273cf

Please sign in to comment.