diff --git a/.chloggen/2033-nginx-autoinstrumentation.yaml b/.chloggen/2033-nginx-autoinstrumentation.yaml new file mode 100755 index 0000000000..064b01724b --- /dev/null +++ b/.chloggen/2033-nginx-autoinstrumentation.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action) +component: operator + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Implementation of new Nginx autoinstrumentation. + +# One or more tracking issues related to the change +issues: [2033] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.github/workflows/publish-images.yaml b/.github/workflows/publish-images.yaml index 707f98ce39..c98f03866f 100644 --- a/.github/workflows/publish-images.yaml +++ b/.github/workflows/publish-images.yaml @@ -31,6 +31,7 @@ jobs: grep -v '\#' versions.txt | grep autoinstrumentation-dotnet | awk -F= '{print "AUTO_INSTRUMENTATION_DOTNET_VERSION="$2}' >> $GITHUB_ENV grep -v '\#' versions.txt | grep autoinstrumentation-go | awk -F= '{print "AUTO_INSTRUMENTATION_GO_VERSION="$2}' >> $GITHUB_ENV grep -v '\#' versions.txt | grep autoinstrumentation-apache-httpd | awk -F= '{print "AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION="$2}' >> $GITHUB_ENV + grep -v '\#' versions.txt | grep autoinstrumentation-apache-httpd | awk -F= '{print "AUTO_INSTRUMENTATION_NGINX_VERSION="$2}' >> $GITHUB_ENV echo "VERSION_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV echo "VERSION=$(git describe --tags | sed 's/^v//')" >> $GITHUB_ENV @@ -98,5 +99,6 @@ jobs: AUTO_INSTRUMENTATION_DOTNET_VERSION=${{ env.AUTO_INSTRUMENTATION_DOTNET_VERSION }} AUTO_INSTRUMENTATION_GO_VERSION=${{ env.AUTO_INSTRUMENTATION_GO_VERSION }} AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION=${{ env.AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION }} + AUTO_INSTRUMENTATION_NGINX_VERSION=${{ env.AUTO_INSTRUMENTATION_NGINX_VERSION }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index 4dfecd17a6..cff812ec5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,10 +31,11 @@ ARG AUTO_INSTRUMENTATION_NODEJS_VERSION ARG AUTO_INSTRUMENTATION_PYTHON_VERSION ARG AUTO_INSTRUMENTATION_DOTNET_VERSION ARG AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION +ARG AUTO_INSTRUMENTATION_NGINX_VERSION ARG AUTO_INSTRUMENTATION_GO_VERSION # Build -RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -ldflags="-X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationApacheHttpd=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION}" -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -ldflags="-X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationApacheHttpd=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} -X ${VERSION_PKG}.autoInstrumentationNginx=${AUTO_INSTRUMENTATION_NGINX_VERSION}" -a -o manager main.go ######## Start a new stage from scratch ####### FROM scratch diff --git a/Makefile b/Makefile index 149e3624c9..d625a02823 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ AUTO_INSTRUMENTATION_PYTHON_VERSION ?= "$(shell grep -v '\#' versions.txt | grep AUTO_INSTRUMENTATION_DOTNET_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-dotnet | awk -F= '{print $$2}')" AUTO_INSTRUMENTATION_GO_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-go | awk -F= '{print $$2}')" AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-apache-httpd | awk -F= '{print $$2}')" -LD_FLAGS ?= "-X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION}" +AUTO_INSTRUMENTATION_NGINX_VERSION ?= "$(shell grep -v '\#' versions.txt | grep autoinstrumentation-nginx | awk -F= '{print $$2}')" +LD_FLAGS ?= "-X ${VERSION_PKG}.version=${VERSION} -X ${VERSION_PKG}.buildDate=${VERSION_DATE} -X ${VERSION_PKG}.otelCol=${OTELCOL_VERSION} -X ${VERSION_PKG}.targetAllocator=${TARGETALLOCATOR_VERSION} -X ${VERSION_PKG}.operatorOpAMPBridge=${OPERATOR_OPAMP_BRIDGE_VERSION} -X ${VERSION_PKG}.autoInstrumentationJava=${AUTO_INSTRUMENTATION_JAVA_VERSION} -X ${VERSION_PKG}.autoInstrumentationNodeJS=${AUTO_INSTRUMENTATION_NODEJS_VERSION} -X ${VERSION_PKG}.autoInstrumentationPython=${AUTO_INSTRUMENTATION_PYTHON_VERSION} -X ${VERSION_PKG}.autoInstrumentationDotNet=${AUTO_INSTRUMENTATION_DOTNET_VERSION} -X ${VERSION_PKG}.autoInstrumentationGo=${AUTO_INSTRUMENTATION_GO_VERSION} -X ${VERSION_PKG}.autoInstrumentationApacheHttpd=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} -X ${VERSION_PKG}.autoInstrumentationNginx=${AUTO_INSTRUMENTATION_NGINX_VERSION}" ARCH ?= $(shell go env GOARCH) # Image URL to use all building/pushing image targets @@ -235,7 +236,7 @@ scorecard-tests: operator-sdk # buildx is used to ensure same results for arm based systems (m1/2 chips) .PHONY: container container: - docker buildx build --load --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg OPERATOR_OPAMP_BRIDGE_VERSION=${OPERATOR_OPAMP_BRIDGE_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} --build-arg AUTO_INSTRUMENTATION_GO_VERSION=${AUTO_INSTRUMENTATION_GO_VERSION} --build-arg AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} . + docker buildx build --load --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg OPERATOR_OPAMP_BRIDGE_VERSION=${OPERATOR_OPAMP_BRIDGE_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} --build-arg AUTO_INSTRUMENTATION_GO_VERSION=${AUTO_INSTRUMENTATION_GO_VERSION} --build-arg AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION=${AUTO_INSTRUMENTATION_APACHE_HTTPD_VERSION} --build-arg AUTO_INSTRUMENTATION_NGINX_VERSION=${AUTO_INSTRUMENTATION_NGINX_VERSION} . # Push the container image, used only for local dev purposes .PHONY: container-push diff --git a/README.md b/README.md index 7cce244cd0..9490745815 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ When using sidecar mode the OpenTelemetry collector container will have the envi ### OpenTelemetry auto-instrumentation injection -The operator can inject and configure OpenTelemetry auto-instrumentation libraries. Currently Apache HTTPD, DotNet, Go, Java, NodeJS and Python are supported. +The operator can inject and configure OpenTelemetry auto-instrumentation libraries. Currently Apache HTTPD, DotNet, Go, Java, Nginx, NodeJS and Python are supported. To use auto-instrumentation, configure an `Instrumentation` resource with the configuration for the SDK and instrumentation. @@ -291,6 +291,11 @@ Apache HTTPD: instrumentation.opentelemetry.io/inject-apache-httpd: "true" ``` +Nginx: +```bash +instrumentation.opentelemetry.io/inject-nginx: "true" +``` + OpenTelemetry SDK environment variables only: ```bash instrumentation.opentelemetry.io/inject-sdk: "true" @@ -449,6 +454,8 @@ spec: image: your-customized-auto-instrumentation-image:go apacheHttpd: image: your-customized-auto-instrumentation-image:apache-httpd + nginx: + image: your-customized-auto-instrumentation-image:nginx ``` The Dockerfiles for auto-instrumentation can be found in [autoinstrumentation directory](./autoinstrumentation). @@ -474,6 +481,26 @@ metadata: ``` List of all available attributes can be found at [otel-webserver-module](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/otel-webserver-module) +#### Using Nginx autoinstrumentation + +For `Nginx` autoinstrumentation, Nginx versions 1.22.0, 1.23.0, and 1.23.1 are supported at this time. The Nginx configuration file is expected to be `/etc/nginx/nginx.conf` by default, if it's different, see following example on how to change it. Instrumentation at this time also expects, that `conf.d` directory is present in the directory, where configuration file resides and that there is a `include /conf.d/*.conf;` directive in the `http { ... }` section of Nginx configuration file (like it is in the default configuration file of Nginx). You can also adjust OpenTelemetry SDK attributes. Example: + +```yaml +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: my-instrumentation + nginx: + image: your-customized-auto-instrumentation-image:nginx # if custom instrumentation image is needed + configFile: /my/custom-dir/custom-nginx.conf + attrs: + - name: NginxModuleOtelMaxQueueSize + value: "4096" + - name: ... + value: ... +``` +List of all available attributes can be found at [otel-webserver-module](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/otel-webserver-module) + #### Inject OpenTelemetry SDK environment variables only You can configure the OpenTelemetry SDK for applications which can't currently be autoinstrumented by using `inject-sdk` in place of `inject-python` or `inject-java`, for example. This will inject environment variables like `OTEL_RESOURCE_ATTRIBUTES`, `OTEL_TRACES_SAMPLER`, and `OTEL_EXPORTER_OTLP_ENDPOINT`, that you can configure in the `Instrumentation`, but will not actually provide the SDK. @@ -499,6 +526,7 @@ If a language is enabled by default its gate only needs to be supplied when disa | DotNet | `operator.autoinstrumentation.dotnet` | enabled | | ApacheHttpD | `operator.autoinstrumentation.apache-httpd` | enabled | | Go | `operator.autoinstrumentation.go` | disabled | +| Nginx | `operator.autoinstrumentation.nginx` | disabled | Language not specified in the table are always supported and cannot be disabled. diff --git a/apis/v1alpha1/instrumentation_webhook_test.go b/apis/v1alpha1/instrumentation_webhook_test.go index 0ccd12346a..9791a93014 100644 --- a/apis/v1alpha1/instrumentation_webhook_test.go +++ b/apis/v1alpha1/instrumentation_webhook_test.go @@ -30,6 +30,7 @@ func TestInstrumentationDefaultingWebhook(t *testing.T) { AnnotationDefaultAutoInstrumentationPython: "python-img:1", AnnotationDefaultAutoInstrumentationDotNet: "dotnet-img:1", AnnotationDefaultAutoInstrumentationApacheHttpd: "apache-httpd-img:1", + AnnotationDefaultAutoInstrumentationNginx: "nginx-img:1", }, }, } @@ -39,6 +40,7 @@ func TestInstrumentationDefaultingWebhook(t *testing.T) { assert.Equal(t, "python-img:1", inst.Spec.Python.Image) assert.Equal(t, "dotnet-img:1", inst.Spec.DotNet.Image) assert.Equal(t, "apache-httpd-img:1", inst.Spec.ApacheHttpd.Image) + assert.Equal(t, "nginx-img:1", inst.Spec.Nginx.Image) } func TestInstrumentationValidatingWebhook(t *testing.T) { diff --git a/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml index 5677655ad9..588e52044d 100644 --- a/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml +++ b/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml @@ -299,7 +299,7 @@ spec: - --enable-leader-election - --zap-log-level=info - --zap-time-encoding=rfc3339nano - - --feature-gates=+operator.autoinstrumentation.go + - --feature-gates=+operator.autoinstrumentation.go,+operator.autoinstrumentation.nginx image: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator:0.85.0 livenessProbe: httpGet: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 80999a45ab..918d8a41a4 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -33,4 +33,4 @@ spec: - "--enable-leader-election" - "--zap-log-level=info" - "--zap-time-encoding=rfc3339nano" - - "--feature-gates=+operator.autoinstrumentation.go" + - "--feature-gates=+operator.autoinstrumentation.go,+operator.autoinstrumentation.nginx" diff --git a/internal/config/main.go b/internal/config/main.go index 64d2beec8a..951340136b 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -44,6 +44,7 @@ type Config struct { autoInstrumentationDotNetImage string autoInstrumentationGoImage string autoInstrumentationApacheHttpdImage string + autoInstrumentationNginxImage string targetAllocatorConfigMapEntry string autoInstrumentationNodeJSImage string autoInstrumentationJavaImage string @@ -85,6 +86,7 @@ func New(opts ...Option) Config { autoInstrumentationPythonImage: o.autoInstrumentationPythonImage, autoInstrumentationDotNetImage: o.autoInstrumentationDotNetImage, autoInstrumentationApacheHttpdImage: o.autoInstrumentationApacheHttpdImage, + autoInstrumentationNginxImage: o.autoInstrumentationNginxImage, labelsFilter: o.labelsFilter, } } @@ -184,6 +186,11 @@ func (c *Config) AutoInstrumentationApacheHttpdImage() string { return c.autoInstrumentationApacheHttpdImage } +// AutoInstrumentationNginxImage returns OpenTelemetry Nginx auto-instrumentation container image. +func (c *Config) AutoInstrumentationNginxImage() string { + return c.autoInstrumentationNginxImage +} + // LabelsFilter Returns the filters converted to regex strings used to filter out unwanted labels from propagations. func (c *Config) LabelsFilter() []string { return c.labelsFilter diff --git a/internal/config/options.go b/internal/config/options.go index 13d88b508a..3d0e207aff 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -38,6 +38,7 @@ type options struct { autoInstrumentationNodeJSImage string autoInstrumentationPythonImage string autoInstrumentationApacheHttpdImage string + autoInstrumentationNginxImage string collectorImage string collectorConfigMapEntry string targetAllocatorConfigMapEntry string @@ -145,6 +146,12 @@ func WithAutoInstrumentationApacheHttpdImage(s string) Option { } } +func WithAutoInstrumentationNginxImage(s string) Option { + return func(o *options) { + o.autoInstrumentationNginxImage = s + } +} + func WithLabelFilters(labelFilters []string) Option { return func(o *options) { diff --git a/internal/version/main.go b/internal/version/main.go index 6b4883518e..7e85af2f12 100644 --- a/internal/version/main.go +++ b/internal/version/main.go @@ -31,6 +31,7 @@ var ( autoInstrumentationPython string autoInstrumentationDotNet string autoInstrumentationApacheHttpd string + autoInstrumentationNginx string autoInstrumentationGo string ) @@ -48,6 +49,7 @@ type Version struct { AutoInstrumentationDotNet string `json:"auto-instrumentation-dotnet"` AutoInstrumentationGo string `json:"auto-instrumentation-go"` AutoInstrumentationApacheHttpd string `json:"auto-instrumentation-apache-httpd"` + AutoInstrumentationNginx string `json:"auto-instrumentation-nginx"` } // Get returns the Version object with the relevant information. @@ -65,12 +67,13 @@ func Get() Version { AutoInstrumentationDotNet: AutoInstrumentationDotNet(), AutoInstrumentationGo: AutoInstrumentationGo(), AutoInstrumentationApacheHttpd: AutoInstrumentationApacheHttpd(), + AutoInstrumentationNginx: AutoInstrumentationNginx(), } } func (v Version) String() string { return fmt.Sprintf( - "Version(Operator='%v', BuildDate='%v', OpenTelemetryCollector='%v', Go='%v', TargetAllocator='%v', OperatorOpAMPBridge='%v', AutoInstrumentationJava='%v', AutoInstrumentationNodeJS='%v', AutoInstrumentationPython='%v', AutoInstrumentationDotNet='%v', AutoInstrumentationGo='%v', AutoInstrumentationApacheHttpd='%v')", + "Version(Operator='%v', BuildDate='%v', OpenTelemetryCollector='%v', Go='%v', TargetAllocator='%v', OperatorOpAMPBridge='%v', AutoInstrumentationJava='%v', AutoInstrumentationNodeJS='%v', AutoInstrumentationPython='%v', AutoInstrumentationDotNet='%v', AutoInstrumentationGo='%v', AutoInstrumentationApacheHttpd='%v', AutoInstrumentationNginx='%v')", v.Operator, v.BuildDate, v.OpenTelemetryCollector, @@ -83,6 +86,7 @@ func (v Version) String() string { v.AutoInstrumentationDotNet, v.AutoInstrumentationGo, v.AutoInstrumentationApacheHttpd, + v.AutoInstrumentationNginx, ) } @@ -154,6 +158,13 @@ func AutoInstrumentationApacheHttpd() string { return "0.0.0" } +func AutoInstrumentationNginx() string { + if len(autoInstrumentationNginx) > 0 { + return autoInstrumentationNginx + } + return "0.0.0" +} + func AutoInstrumentationGo() string { if len(autoInstrumentationGo) > 0 { return autoInstrumentationGo diff --git a/main.go b/main.go index 61cb218a74..2f04fb0835 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,7 @@ func main() { autoInstrumentationPython string autoInstrumentationDotNet string autoInstrumentationApacheHttpd string + autoInstrumentationNginx string autoInstrumentationGo string labelsFilter []string webhookPort int @@ -122,6 +123,7 @@ func main() { pflag.StringVar(&autoInstrumentationDotNet, "auto-instrumentation-dotnet-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-dotnet:%s", v.AutoInstrumentationDotNet), "The default OpenTelemetry DotNet instrumentation image. This image is used when no image is specified in the CustomResource.") pflag.StringVar(&autoInstrumentationGo, "auto-instrumentation-go-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-go-instrumentation/autoinstrumentation-go:%s", v.AutoInstrumentationGo), "The default OpenTelemetry Go instrumentation image. This image is used when no image is specified in the CustomResource.") pflag.StringVar(&autoInstrumentationApacheHttpd, "auto-instrumentation-apache-httpd-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-apache-httpd:%s", v.AutoInstrumentationApacheHttpd), "The default OpenTelemetry Apache HTTPD instrumentation image. This image is used when no image is specified in the CustomResource.") + pflag.StringVar(&autoInstrumentationNginx, "auto-instrumentation-nginx-image", fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-apache-httpd:%s", v.AutoInstrumentationNginx), "The default OpenTelemetry Nginx instrumentation image. This image is used when no image is specified in the CustomResource.") pflag.StringArrayVar(&labelsFilter, "labels", []string{}, "Labels to filter away from propagating onto deploys") pflag.IntVar(&webhookPort, "webhook-port", 9443, "The port the webhook endpoint binds to.") pflag.StringVar(&tlsOpt.minVersion, "tls-min-version", "VersionTLS12", "Minimum TLS version supported. Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants.") @@ -142,6 +144,7 @@ func main() { "auto-instrumentation-dotnet", autoInstrumentationDotNet, "auto-instrumentation-go", autoInstrumentationGo, "auto-instrumentation-apache-httpd", autoInstrumentationApacheHttpd, + "auto-instrumentation-nginx", autoInstrumentationNginx, "feature-gates", flagset.Lookup(featuregate.FeatureGatesFlag).Value.String(), "build-date", v.BuildDate, "go-version", v.Go, @@ -171,6 +174,7 @@ func main() { config.WithAutoInstrumentationDotNetImage(autoInstrumentationDotNet), config.WithAutoInstrumentationGoImage(autoInstrumentationGo), config.WithAutoInstrumentationApacheHttpdImage(autoInstrumentationApacheHttpd), + config.WithAutoInstrumentationNginxImage(autoInstrumentationNginx), config.WithAutoDetect(ad), config.WithLabelFilters(labelsFilter), ) @@ -257,7 +261,7 @@ func main() { otelv1alpha1.AnnotationDefaultAutoInstrumentationDotNet: autoInstrumentationDotNet, otelv1alpha1.AnnotationDefaultAutoInstrumentationGo: autoInstrumentationGo, otelv1alpha1.AnnotationDefaultAutoInstrumentationApacheHttpd: autoInstrumentationApacheHttpd, - }, + otelv1alpha1.AnnotationDefaultAutoInstrumentationNginx: autoInstrumentationNginx}, }, }).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "Instrumentation") @@ -324,6 +328,7 @@ func addDependencies(_ context.Context, mgr ctrl.Manager, cfg config.Config, v v DefaultAutoInstDotNet: cfg.AutoInstrumentationDotNetImage(), DefaultAutoInstGo: cfg.AutoInstrumentationDotNetImage(), DefaultAutoInstApacheHttpd: cfg.AutoInstrumentationApacheHttpdImage(), + DefaultAutoInstNginx: cfg.AutoInstrumentationNginxImage(), Client: mgr.GetClient(), Recorder: mgr.GetEventRecorderFor("opentelemetry-operator"), } diff --git a/pkg/featuregate/featuregate.go b/pkg/featuregate/featuregate.go index 200c997a01..b9e6231eca 100644 --- a/pkg/featuregate/featuregate.go +++ b/pkg/featuregate/featuregate.go @@ -61,6 +61,12 @@ var ( featuregate.WithRegisterDescription("controls whether the operator supports Apache HTTPD auto-instrumentation"), featuregate.WithRegisterFromVersion("v0.80.0"), ) + EnableNginxAutoInstrumentationSupport = featuregate.GlobalRegistry().MustRegister( + "operator.autoinstrumentation.nginx", + featuregate.StageAlpha, + featuregate.WithRegisterDescription("controls whether the operator supports Nginx auto-instrumentation"), + featuregate.WithRegisterFromVersion("v0.86.0"), + ) EnableMultiInstrumentationSupport = featuregate.GlobalRegistry().MustRegister( "operator.autoinstrumentation.multi-instrumentation", diff --git a/pkg/instrumentation/annotation.go b/pkg/instrumentation/annotation.go index bcf62fec0a..28ef7bf3d5 100644 --- a/pkg/instrumentation/annotation.go +++ b/pkg/instrumentation/annotation.go @@ -40,6 +40,8 @@ const ( annotationInjectSdkContainersName = "instrumentation.opentelemetry.io/sdk-container-names" annotationInjectApacheHttpd = "instrumentation.opentelemetry.io/inject-apache-httpd" annotationInjectApacheHttpdContainersName = "instrumentation.opentelemetry.io/apache-httpd-container-names" + annotationInjectNginx = "instrumentation.opentelemetry.io/inject-nginx" + annotationInjectNginxContainersName = "instrumentation.opentelemetry.io/inject-nginx-container-names" ) // annotationValue returns the effective annotationInjectJava value, based on the annotations from the pod and namespace. diff --git a/pkg/instrumentation/nginx.go b/pkg/instrumentation/nginx.go new file mode 100644 index 0000000000..9a16520d6d --- /dev/null +++ b/pkg/instrumentation/nginx.go @@ -0,0 +1,352 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package instrumentation + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/go-logr/logr" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" + corev1 "k8s.io/api/core/v1" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" +) + +const ( + nginxDefaultConfigFile = "/etc/nginx/nginx.conf" + nginxAgentCloneContainerName = "otel-agent-source-container-clone" + nginxAgentInitContainerName = "otel-agent-attach-nginx" + nginxAgentVolume = "otel-nginx-agent" + nginxAgentConfigVolume = "otel-nginx-conf-dir" + nginxAgentConfigFile = "opentemetry_agent.conf" + nginxAgentDirectory = "/opt/opentelemetry-webserver" + nginxAgentSubDirectory = "/agent" + nginxAgentDirFull = nginxAgentDirectory + nginxAgentSubDirectory + nginxAgentConfigDirectory = "/source-conf" + nginxAgentConfDirFull = nginxAgentDirectory + nginxAgentConfigDirectory + nginxAttributesEnvVar = "OTEL_NGINX_AGENT_CONF" + nginxServiceInstanceId = "<>" + nginxServiceInstanceIdEnvVar = "OTEL_NGINX_SERVICE_INSTANCE_ID" + nginxVersionEnvVar = "NGINX_VERSION" + nginxLibraryPathEnv = "LD_LIBRARY_PATH" +) + +/* + Nginx injection is different from other languages in: + - OpenTelemetry parameters are not passed as environmental variables, but via a configuration file + - OpenTelemetry module needs to be specified in the Nginx config file, but that is already specified by + an author of the original image configuration and the configuration must be preserved + + Therefore, following approach is taken: + 1) Inject an init container created as a *clone* of the application container and copy config file and referenced + configuration directory to an empty shared volume + 2) Inject a second init container with the OpenTelemetry module itself - i.e. instrumentation image + 3) Take the Nginx configuration file saved on volume and inject reference to OpenTelemetry module into the config + 4) On the same volume, inject a configuration file for OpenTelemetry module + 5) Copy OpenTelemetry module from second init container (instrumentation image) to another shared volume + 6) Inject mounting of volumes / files into appropriate directories in the application container +*/ + +func injectNginxSDK(_ logr.Logger, nginxSpec v1alpha1.Nginx, pod corev1.Pod, index int, otlpEndpoint string, resourceMap map[string]string) corev1.Pod { + + // caller checks if there is at least one container + container := &pod.Spec.Containers[index] + + // inject env vars + for _, env := range nginxSpec.Env { + idx := getIndexOfEnv(container.Env, env.Name) + if idx == -1 { + container.Env = append(container.Env, env) + } + } + + // First make a clone of the instrumented container to take the existing Nginx configuration from + // and create init container from it + if isNginxInitContainerMissing(pod, nginxAgentCloneContainerName) { + // Inject volume for original Nginx configuration + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}) + + nginxConfFile := getNginxConfFile(nginxSpec.ConfigFile) + nginxConfDir := getNginxConfDir(nginxSpec.ConfigFile) + + // from original Nginx container, we need + // 1) original configuration files, which then get modified in the instrumentation process + // 2) version of Nginx to select the proper version of OTel modules. + // - run Nginx with -v to get the version + // - store the version into a file where instrumentation initContainer can pick it up + nginxCloneScriptTemplate := + ` +cp -r %[2]s/* %[3]s && +export %[4]s=$( { nginx -v ; } 2>&1 ) && echo ${%[4]s##*/} > %[3]s/version.txt +` + nginxAgentCommands := prepareCommandFromTemplate(nginxCloneScriptTemplate, + nginxConfFile, + nginxConfDir, + nginxAgentConfDirFull, + nginxVersionEnvVar, + ) + + cloneContainer := container.DeepCopy() + cloneContainer.Name = nginxAgentCloneContainerName + cloneContainer.Command = []string{"/bin/sh", "-c"} + cloneContainer.Args = []string{nginxAgentCommands} + cloneContainer.VolumeMounts = append(cloneContainer.VolumeMounts, corev1.VolumeMount{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }) + // remove resource requirements since those are then reserved for the lifetime of a pod + // and we definitely do not need them for the init container for cp command + cloneContainer.Resources = nginxSpec.Resources + // remove livenessProbe, readinessProbe, and startupProbe, since not supported on init containers + cloneContainer.LivenessProbe = nil + cloneContainer.ReadinessProbe = nil + cloneContainer.StartupProbe = nil + + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *cloneContainer) + + // drop volume mount with volume-provided Nginx config from original container + // since it could over-write configuration provided by the injection + idxFound := -1 + for idx, volume := range container.VolumeMounts { + if strings.Contains(volume.MountPath, nginxConfDir) { // potentially passes config, which we want to pass to init copy only + idxFound = idx + break + } + } + if idxFound >= 0 { + volumeMounts := container.VolumeMounts + volumeMounts = append(volumeMounts[:idxFound], volumeMounts[idxFound+1:]...) + container.VolumeMounts = volumeMounts + } + + // Inject volumes info instrumented container - Nginx config dir + Nginx agent + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: nginxAgentConfigVolume, + MountPath: nginxConfDir, + }) + } + + // Inject second init container with instrumentation image + // Create / update config files + // Copy OTEL module to a shared volume + if isNginxInitContainerMissing(pod, nginxAgentInitContainerName) { + // Inject volume for agent + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}) + + // Following is the template for a shell script, which does the actual instrumentation + // It does following: + // 1) Copies Nginx OTel modules from the webserver agent image + // 2) Picks-up the Nginx version stored by the clone of original container (see comment there) + // 3) Finds out which directory to use for logs + // 4) Configures the directory in logging configuration file of OTel modules + // 5) Creates a configuration file for OTel modules + // 6) In that configuration file, set SID parameter to pod name (in env var OTEL_NGINX_SERVICE_INSTANCE_ID) + // 7) In Nginx config file, inject directive to load OTel module + // 8) In Nginx config file, enable use of env var OTEL_RESOURCE_ATTRIBUTES in Nginx process + // (by default, env vars are hidden to Nginx process, they need to be enabled specifically) + // 9) Move OTel module configuration file to Nginx configuration directory. + + // Each line of the script MUST end with \n ! + nginxAgentI13nScript := + ` +NGINX_AGENT_DIR_FULL=$1 \n +NGINX_AGENT_CONF_DIR_FULL=$2 \n +NGINX_CONFIG_FILE=$3 \n +NGINX_SID_PLACEHOLDER=$4 \n +NGINX_SID_VALUE=$5 \n +echo "Input Parameters: $@" \n +set -x \n +\n +cp -ar /opt/opentelemetry/* ${NGINX_AGENT_DIR_FULL} \n +\n +NGINX_VERSION=$(cat ${NGINX_AGENT_CONF_DIR_FULL}/version.txt) \n +NGINX_AGENT_LOG_DIR=$(echo "${NGINX_AGENT_DIR_FULL}/logs" | sed 's,/,\\/,g') \n +\n +cat ${NGINX_AGENT_DIR_FULL}/conf/appdynamics_sdk_log4cxx.xml.template | sed 's,__agent_log_dir__,'${NGINX_AGENT_LOG_DIR}',g' > ${NGINX_AGENT_DIR_FULL}/conf/appdynamics_sdk_log4cxx.xml \n +echo -e $OTEL_NGINX_AGENT_CONF > ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf \n +sed -i "s,${NGINX_SID_PLACEHOLDER},${OTEL_NGINX_SERVICE_INSTANCE_ID},g" ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf \n +sed -i "1s,^,load_module ${NGINX_AGENT_DIR_FULL}/WebServerModule/Nginx/${NGINX_VERSION}/ngx_http_opentelemetry_module.so;\\n,g" ${NGINX_AGENT_CONF_DIR_FULL}/${NGINX_CONFIG_FILE} \n +sed -i "1s,^,env OTEL_RESOURCE_ATTRIBUTES;\\n,g" ${NGINX_AGENT_CONF_DIR_FULL}/${NGINX_CONFIG_FILE} \n +mv ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf ${NGINX_AGENT_CONF_DIR_FULL}/conf.d \n + ` + + nginxAgentI13nCommand := "echo -e $OTEL_NGINX_I13N_SCRIPT > " + nginxAgentDirFull + "/nginx_instrumentation.sh && " + + "chmod +x " + nginxAgentDirFull + "/nginx_instrumentation.sh && " + + "cat " + nginxAgentDirFull + "/nginx_instrumentation.sh && " + + fmt.Sprintf(nginxAgentDirFull+"/nginx_instrumentation.sh \"%s\" \"%s\" \"%s\" \"%s\"", + nginxAgentDirFull, + nginxAgentConfDirFull, + getNginxConfFile(nginxSpec.ConfigFile), + nginxServiceInstanceId, + ) + + pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ + Name: nginxAgentInitContainerName, + Image: nginxSpec.Image, + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxAgentI13nCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: getNginxOtelConfig(pod, nginxSpec, index, otlpEndpoint, resourceMap), + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxAgentI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + Resources: nginxSpec.Resources, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + SecurityContext: pod.Spec.Containers[index].SecurityContext, + }) + + found := false + for i, e := range container.Env { + if e.Name == nginxLibraryPathEnv { + container.Env[i].Value = e.Value + ":" + nginxAgentDirFull + "/sdk_lib/lib" + found = true + break + } + } + if !found { + container.Env = append(container.Env, corev1.EnvVar{ + Name: nginxLibraryPathEnv, + Value: nginxAgentDirFull + "/sdk_lib/lib", + }) + } + } + + return pod +} + +// Calculate if we already inject InitContainers. +func isNginxInitContainerMissing(pod corev1.Pod, containerName string) bool { + for _, initContainer := range pod.Spec.InitContainers { + if initContainer.Name == containerName { + return false + } + } + return true +} + +// Calculate Nginx agent configuration file based on attributes provided by the injection rules +// and by the pod values. +func getNginxOtelConfig(pod corev1.Pod, nginxSpec v1alpha1.Nginx, index int, otelEndpoint string, resourceMap map[string]string) string { + + if otelEndpoint == "" { + otelEndpoint = "http://localhost:4317/" + } + serviceName := chooseServiceName(pod, resourceMap, index) + serviceNamespace := pod.GetNamespace() + if len(serviceNamespace) == 0 { + serviceNamespace = resourceMap[string(semconv.K8SNamespaceNameKey)] + if len(serviceNamespace) == 0 { + serviceNamespace = "nginx" + } + } + + // Namespace name override TBD + + attrMap := map[string]string{ + "NginxModuleEnabled": "ON", + "NginxModuleOtelSpanExporter": "otlp", + "NginxModuleOtelExporterEndpoint": otelEndpoint, + "NginxModuleServiceName": serviceName, + "NginxModuleServiceNamespace": serviceNamespace, + "NginxModuleServiceInstanceId": nginxServiceInstanceId, + "NginxModuleResolveBackends": "ON", + "NginxModuleTraceAsError": "ON", + } + for _, attr := range nginxSpec.Attrs { + attrMap[attr.Name] = attr.Value + } + + configFileContent := "" + + keys := make([]string, 0, len(attrMap)) + for key := range attrMap { + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { + configFileContent += fmt.Sprintf("%s %s;\n", key, attrMap[key]) + } + + return configFileContent +} + +func getNginxConfDir(configuredFile string) string { + nginxConfFile := nginxDefaultConfigFile + if configuredFile != "" { + nginxConfFile = configuredFile + } + configDir := filepath.Dir(nginxConfFile) + return configDir +} + +func getNginxConfFile(configuredFile string) string { + nginxConfFile := nginxDefaultConfigFile + if configuredFile != "" { + nginxConfFile = configuredFile + } + configFilenameOnly := filepath.Base(nginxConfFile) + return configFilenameOnly +} + +func prepareCommandFromTemplate(template string, params ...any) string { + command := fmt.Sprintf(template, + params..., + ) + + command = strings.Replace(command, "\n", " ", -1) + command = strings.Replace(command, "\t", " ", -1) + command = strings.TrimLeft(command, " ") + command = strings.TrimRight(command, " ") + + return command +} diff --git a/pkg/instrumentation/nginx_test.go b/pkg/instrumentation/nginx_test.go new file mode 100644 index 0000000000..8a78b41ae8 --- /dev/null +++ b/pkg/instrumentation/nginx_test.go @@ -0,0 +1,657 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package instrumentation + +import ( + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" +) + +var nginxSdkInitContainerTestCommand = "echo -e $OTEL_NGINX_I13N_SCRIPT > /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && chmod +x /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && cat /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh \"/opt/opentelemetry-webserver/agent\" \"/opt/opentelemetry-webserver/source-conf\" \"nginx.conf\" \"<>\"" +var nginxSdkInitContainerTestCommandCustomFile = "echo -e $OTEL_NGINX_I13N_SCRIPT > /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && chmod +x /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && cat /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh \"/opt/opentelemetry-webserver/agent\" \"/opt/opentelemetry-webserver/source-conf\" \"custom-nginx.conf\" \"<>\"" +var nginxSdkInitContainerI13nScript = "\nNGINX_AGENT_DIR_FULL=$1\t\\n\nNGINX_AGENT_CONF_DIR_FULL=$2 \\n\nNGINX_CONFIG_FILE=$3 \\n\nNGINX_SID_PLACEHOLDER=$4 \\n\nNGINX_SID_VALUE=$5 \\n\necho \"Input Parameters: $@\" \\n\nset -x \\n\n\\n\ncp -ar /opt/opentelemetry/* ${NGINX_AGENT_DIR_FULL} \\n\n\\n\nNGINX_VERSION=$(cat ${NGINX_AGENT_CONF_DIR_FULL}/version.txt) \\n\nNGINX_AGENT_LOG_DIR=$(echo \"${NGINX_AGENT_DIR_FULL}/logs\" | sed 's,/,\\\\/,g') \\n\n\\n\ncat ${NGINX_AGENT_DIR_FULL}/conf/appdynamics_sdk_log4cxx.xml.template | sed 's,__agent_log_dir__,'${NGINX_AGENT_LOG_DIR}',g' > ${NGINX_AGENT_DIR_FULL}/conf/appdynamics_sdk_log4cxx.xml \\n\necho -e $OTEL_NGINX_AGENT_CONF > ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf \\n\nsed -i \"s,${NGINX_SID_PLACEHOLDER},${OTEL_NGINX_SERVICE_INSTANCE_ID},g\" ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf \\n\nsed -i \"1s,^,load_module ${NGINX_AGENT_DIR_FULL}/WebServerModule/Nginx/${NGINX_VERSION}/ngx_http_opentelemetry_module.so;\\\\n,g\" ${NGINX_AGENT_CONF_DIR_FULL}/${NGINX_CONFIG_FILE} \\n\nsed -i \"1s,^,env OTEL_RESOURCE_ATTRIBUTES;\\\\n,g\" ${NGINX_AGENT_CONF_DIR_FULL}/${NGINX_CONFIG_FILE} \\n\nmv ${NGINX_AGENT_CONF_DIR_FULL}/opentelemetry_agent.conf ${NGINX_AGENT_CONF_DIR_FULL}/conf.d \\n\n\t\t" + +func TestInjectNginxSDK(t *testing.T) { + + tests := []struct { + name string + v1alpha1.Nginx + pod corev1.Pod + expected corev1.Pod + }{ + { + name: "Clone Container not present", + Nginx: v1alpha1.Nginx{ + Image: "foo/bar:1", + Attrs: []corev1.EnvVar{ + { + Name: "NginxModuleOtelMaxQueueSize", + Value: "4096", + }, + }, + }, + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + }, + expected: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "foo/bar:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelMaxQueueSize 4096;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName nginx-service-name;\nNginxModuleServiceNamespace req-namespace;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + }, + }, + }, + }, + }, + }, + // === Test ConfigFile configuration ============================= + { + name: "ConfigFile honored", + Nginx: v1alpha1.Nginx{ + Image: "foo/bar:1", + ConfigFile: "/opt/nginx/custom-nginx.conf", + }, + pod: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /opt/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "foo/bar:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommandCustomFile}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName nginx-service-name;\nNginxModuleServiceNamespace req-namespace;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/opt/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + }, + }, + }, + }, + }}, + // === Test Removal of probes ============================= + { + name: "Probes removed on clone init container", + Nginx: v1alpha1.Nginx{ + Image: "foo/bar:1", + }, + pod: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + ReadinessProbe: &corev1.Probe{}, + StartupProbe: &corev1.Probe{}, + LivenessProbe: &corev1.Probe{}, + }, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "foo/bar:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName nginx-service-name;\nNginxModuleServiceNamespace req-namespace;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + ReadinessProbe: &corev1.Probe{}, + StartupProbe: &corev1.Probe{}, + LivenessProbe: &corev1.Probe{}, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + }, + }, + }, + }, + }, + }, + // Pod Namespace specified + { + name: "Pod Namespace specified", + Nginx: v1alpha1.Nginx{Image: "foo/bar:1"}, + pod: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "my-namespace", + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "my-namespace", + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "foo/bar:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName nginx-service-name;\nNginxModuleServiceNamespace my-namespace;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + }, + }, + }, + }, + }, + }, + } + + resourceMap := map[string]string{ + string(semconv.K8SDeploymentNameKey): "nginx-service-name", + string(semconv.K8SNamespaceNameKey): "req-namespace", + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pod := injectNginxSDK(logr.Discard(), test.Nginx, test.pod, 0, "http://otlp-endpoint:4317", resourceMap) + assert.Equal(t, test.expected, pod) + }) + } +} + +func TestInjectNginxUnknownNamespace(t *testing.T) { + + tests := []struct { + name string + v1alpha1.Nginx + pod corev1.Pod + expected corev1.Pod + }{ + { + name: "Clone Container not present, unknown namespace", + Nginx: v1alpha1.Nginx{Image: "foo/bar:1"}, + pod: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {}, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: nginxAgentConfigVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: nginxAgentVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "foo/bar:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName nginx-service-name;\nNginxModuleServiceNamespace nginx;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + }, + }, + }, + }, + }, + }, + } + + resourceMap := map[string]string{ + string(semconv.K8SDeploymentNameKey): "nginx-service-name", + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pod := injectNginxSDK(logr.Discard(), test.Nginx, test.pod, 0, "http://otlp-endpoint:4317", resourceMap) + assert.Equal(t, test.expected, pod) + }) + } +} + +func TestNginxInitContainerMissing(t *testing.T) { + tests := []struct { + name string + pod corev1.Pod + expected bool + }{ + { + name: "InitContainer_Already_Inject", + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "istio-init", + }, + { + Name: nginxAgentInitContainerName, + }, + }, + }, + }, + expected: false, + }, + { + name: "InitContainer_Absent_1", + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "istio-init", + }, + }, + }, + }, + expected: true, + }, + { + name: "InitContainer_Absent_2", + pod: corev1.Pod{ + Spec: corev1.PodSpec{}, + }, + expected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := isNginxInitContainerMissing(test.pod, nginxAgentInitContainerName) + assert.Equal(t, test.expected, result) + }) + } +} diff --git a/pkg/instrumentation/podmutator.go b/pkg/instrumentation/podmutator.go index 06ff988d6f..49f93d7067 100644 --- a/pkg/instrumentation/podmutator.go +++ b/pkg/instrumentation/podmutator.go @@ -55,6 +55,7 @@ type languageInstrumentations struct { Python instrumentationWithContainers DotNet instrumentationWithContainers ApacheHttpd instrumentationWithContainers + Nginx instrumentationWithContainers Go instrumentationWithContainers Sdk instrumentationWithContainers } @@ -78,6 +79,9 @@ func (langInsts languageInstrumentations) isSingleInstrumentationEnabled() bool if langInsts.ApacheHttpd.Instrumentation != nil { count++ } + if langInsts.Nginx.Instrumentation != nil { + count++ + } if langInsts.Go.Instrumentation != nil { count++ } @@ -120,6 +124,11 @@ func (langInsts languageInstrumentations) areContainerNamesConfiguredForMultiple instrWithoutContainers += isInstrWithoutContainers(langInsts.ApacheHttpd) allContainers = append(allContainers, langInsts.ApacheHttpd.Containers) } + if langInsts.Nginx.Instrumentation != nil { + instrWithContainers += isInstrWithContainers(langInsts.Nginx) + instrWithoutContainers += isInstrWithoutContainers(langInsts.Nginx) + allContainers = append(allContainers, langInsts.Nginx.Containers) + } if langInsts.Go.Instrumentation != nil { instrWithContainers += isInstrWithContainers(langInsts.Go) instrWithoutContainers += isInstrWithoutContainers(langInsts.Go) @@ -171,6 +180,9 @@ func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers( if langInsts.ApacheHttpd.Instrumentation != nil { langInsts.ApacheHttpd.Containers = containers } + if langInsts.Nginx.Instrumentation != nil { + langInsts.Nginx.Containers = containers + } if langInsts.Go.Instrumentation != nil { langInsts.Go.Containers = containers } @@ -282,6 +294,18 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c pm.Recorder.Event(pod.DeepCopy(), "Warning", "InstrumentationRequestRejected", "support for Apache HTTPD auto instrumentation is not enabled") } + if inst, err = pm.getInstrumentationInstance(ctx, ns, pod, annotationInjectNginx); err != nil { + // we still allow the pod to be created, but we log a message to the operator's logs + logger.Error(err, "failed to select an OpenTelemetry Instrumentation instance for this pod") + return pod, err + } + if featuregate.EnableNginxAutoInstrumentationSupport.IsEnabled() || inst == nil { + insts.Nginx.Instrumentation = inst + } else { + logger.Error(nil, "support for Nginx auto instrumentation is not enabled") + pm.Recorder.Event(pod.DeepCopy(), "Warning", "InstrumentationRequestRejected", "support for Nginx auto instrumentation is not enabled") + } + if inst, err = pm.getInstrumentationInstance(ctx, ns, pod, annotationInjectSdk); err != nil { // we still allow the pod to be created, but we log a message to the operator's logs logger.Error(err, "failed to select an OpenTelemetry Instrumentation instance for this pod") @@ -291,7 +315,9 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c if insts.Java.Instrumentation == nil && insts.NodeJS.Instrumentation == nil && insts.Python.Instrumentation == nil && insts.DotNet.Instrumentation == nil && insts.Go.Instrumentation == nil && insts.ApacheHttpd.Instrumentation == nil && + insts.Nginx.Instrumentation == nil && insts.Sdk.Instrumentation == nil { + logger.V(1).Info("annotation not present in deployment, skipping instrumentation injection") return pod, nil } @@ -305,6 +331,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c insts.DotNet.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectDotnetContainersName) insts.Go.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectGoContainersName) insts.ApacheHttpd.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectApacheHttpdContainersName) + insts.Nginx.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectNginxContainersName) insts.Sdk.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectSdkContainersName) // We check if provided annotations and instrumentations are valid diff --git a/pkg/instrumentation/podmutator_test.go b/pkg/instrumentation/podmutator_test.go index 4eca4610e7..eedbb3b05c 100644 --- a/pkg/instrumentation/podmutator_test.go +++ b/pkg/instrumentation/podmutator_test.go @@ -2846,6 +2846,254 @@ func TestMutatePod(t *testing.T) { }) }, }, + + { + name: "nginx injection, true", + ns: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "req-namespace", + }, + }, + inst: v1alpha1.Instrumentation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Namespace: "req-namespace", + }, + Spec: v1alpha1.InstrumentationSpec{ + Nginx: v1alpha1.Nginx{ + Image: "otel/nginx-inj:1", + Attrs: []corev1.EnvVar{{ + Name: "NginxModuleOtelMaxQueueSize", + Value: "4096", + }}, + }, + Exporter: v1alpha1.Exporter{ + Endpoint: "http://otlp-endpoint:4317", + }, + Env: []corev1.EnvVar{}, + }, + }, + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Annotations: map[string]string{ + annotationInjectNginx: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + }, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Annotations: map[string]string{ + annotationInjectNginx: "true", + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "otel-nginx-conf-dir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "otel-nginx-agent", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "otel/nginx-inj:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelMaxQueueSize 4096;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName my-nginx-6c44bcbdd;\nNginxModuleServiceNamespace req-namespace;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "nginx", + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + { + Name: "OTEL_SERVICE_NAME", + Value: "my-nginx-6c44bcbdd", + }, + { + Name: "OTEL_EXPORTER_OTLP_ENDPOINT", + Value: "http://otlp-endpoint:4317", + }, + { + Name: "OTEL_RESOURCE_ATTRIBUTES_NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "OTEL_RESOURCE_ATTRIBUTES", + Value: "k8s.container.name=nginx,k8s.namespace.name=req-namespace,k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=my-nginx-6c44bcbdd", + }, + }, + }, + }, + }, + }, + setFeatureGates: func(t *testing.T) { + originalVal := featuregate.EnableNginxAutoInstrumentationSupport.IsEnabled() + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), true)) + t.Cleanup(func() { + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), originalVal)) + }) + }, + }, + { + name: "nginx injection feature gate disabled", + ns: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-disabled", + }, + }, + inst: v1alpha1.Instrumentation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Namespace: "nginx-disabled", + }, + Spec: v1alpha1.InstrumentationSpec{ + Nginx: v1alpha1.Nginx{ + Image: "otel/nginx-inj:1", + Env: []corev1.EnvVar{ + { + Name: "OTEL_LOG_LEVEL", + Value: "debug", + }, + { + Name: "OTEL_EXPORTER_OTLP_ENDPOINT", + Value: "http://localhost:4317", + }, + }, + Attrs: []corev1.EnvVar{{ + Name: "NginxModuleOtelMaxQueueSize", + Value: "4096", + }}, + }, + Exporter: v1alpha1.Exporter{ + Endpoint: "http://otlp-endpoint:4317", + }, + Env: []corev1.EnvVar{ + { + Name: "OTEL_EXPORTER_OTLP_TIMEOUT", + Value: "20", + }, + { + Name: "OTEL_TRACES_SAMPLER", + Value: "parentbased_traceidratio", + }, + { + Name: "OTEL_TRACES_SAMPLER_ARG", + Value: "0.85", + }, + }, + }, + }, + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Annotations: map[string]string{ + annotationInjectNginx: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + }, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + Annotations: map[string]string{ + annotationInjectNginx: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + }, + }, + }, + }, + setFeatureGates: func(t *testing.T) { + originalVal := featuregate.EnableNginxAutoInstrumentationSupport.IsEnabled() + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), false)) + t.Cleanup(func() { + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), originalVal)) + }) + }, + }, + { name: "missing annotation", ns: corev1.Namespace{ diff --git a/pkg/instrumentation/sdk.go b/pkg/instrumentation/sdk.go index 2a064cb2a2..dd53ca0710 100644 --- a/pkg/instrumentation/sdk.go +++ b/pkg/instrumentation/sdk.go @@ -173,6 +173,23 @@ func (i *sdkInjector) inject(ctx context.Context, insts languageInstrumentations pod = i.injectCommonSDKConfig(ctx, otelinst, ns, pod, index, index) } } + + if insts.Nginx.Instrumentation != nil { + otelinst := *insts.Nginx.Instrumentation + i.logger.V(1).Info("injecting Nginx instrumentation into pod", "otelinst-namespace", otelinst.Namespace, "otelinst-name", otelinst.Name) + + nginxContainers := insts.Nginx.Containers + + for _, container := range strings.Split(nginxContainers, ",") { + index := getContainerIndex(container, pod) + // Nginx agent is configured via config files rather than env vars. + // Therefore, service name, otlp endpoint and other attributes are passed to the agent injection method + pod = injectNginxSDK(i.logger, otelinst.Spec.Nginx, pod, index, otelinst.Spec.Endpoint, i.createResourceMap(ctx, otelinst, ns, pod, index)) + pod = i.injectCommonEnvVar(otelinst, pod, index) + pod = i.injectCommonSDKConfig(ctx, otelinst, ns, pod, index, index) + } + } + if insts.Sdk.Instrumentation != nil { otelinst := *insts.Sdk.Instrumentation i.logger.V(1).Info("injecting sdk-only instrumentation into pod", "otelinst-namespace", otelinst.Namespace, "otelinst-name", otelinst.Name) diff --git a/pkg/instrumentation/sdk_test.go b/pkg/instrumentation/sdk_test.go index e16decacd6..c19e9cd04e 100644 --- a/pkg/instrumentation/sdk_test.go +++ b/pkg/instrumentation/sdk_test.go @@ -1388,6 +1388,168 @@ func TestInjectApacheHttpd(t *testing.T) { } } +func TestInjectNginx(t *testing.T) { + + tests := []struct { + name string + insts languageInstrumentations + pod corev1.Pod + expected corev1.Pod + }{ + { + name: "injection enabled, exporter set", + insts: languageInstrumentations{ + Nginx: instrumentationWithContainers{ + Instrumentation: &v1alpha1.Instrumentation{ + Spec: v1alpha1.InstrumentationSpec{ + Nginx: v1alpha1.Nginx{ + Image: "img:1", + Attrs: []corev1.EnvVar{{ + Name: "NginxModuleOtelMaxQueueSize", + Value: "4096", + }}, + }, + Exporter: v1alpha1.Exporter{ + Endpoint: "http://otlp-endpoint:4317", + }, + }, + }, + Containers: "", + }, + }, + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "app", + }, + }, + }, + }, + expected: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-6c44bcbdd", + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "otel-nginx-conf-dir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "otel-nginx-agent", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: nginxAgentCloneContainerName, + Image: "", + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }}, + }, + { + Name: nginxAgentInitContainerName, + Image: "img:1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{nginxSdkInitContainerTestCommand}, + Env: []corev1.EnvVar{ + { + Name: nginxAttributesEnvVar, + Value: "NginxModuleEnabled ON;\nNginxModuleOtelExporterEndpoint http://otlp-endpoint:4317;\nNginxModuleOtelMaxQueueSize 4096;\nNginxModuleOtelSpanExporter otlp;\nNginxModuleResolveBackends ON;\nNginxModuleServiceInstanceId <>;\nNginxModuleServiceName my-nginx-6c44bcbdd;\nNginxModuleServiceNamespace nginx;\nNginxModuleTraceAsError ON;\n", + }, + { + Name: "OTEL_NGINX_I13N_SCRIPT", + Value: nginxSdkInitContainerI13nScript, + }, + { + Name: nginxServiceInstanceIdEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: nginxAgentConfDirFull, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "app", + VolumeMounts: []corev1.VolumeMount{ + { + Name: nginxAgentVolume, + MountPath: nginxAgentDirFull, + }, + { + Name: nginxAgentConfigVolume, + MountPath: "/etc/nginx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "LD_LIBRARY_PATH", + Value: "/opt/opentelemetry-webserver/agent/sdk_lib/lib", + }, + { + Name: "OTEL_SERVICE_NAME", + Value: "my-nginx-6c44bcbdd", + }, + { + Name: "OTEL_EXPORTER_OTLP_ENDPOINT", + Value: "http://otlp-endpoint:4317", + }, + { + Name: "OTEL_RESOURCE_ATTRIBUTES_NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "OTEL_RESOURCE_ATTRIBUTES", + Value: "k8s.container.name=app,k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=my-nginx-6c44bcbdd", + }, + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + inj := sdkInjector{ + logger: logr.Discard(), + } + pod := inj.inject(context.Background(), test.insts, corev1.Namespace{}, test.pod) + assert.Equal(t, test.expected, pod) + }) + } +} + func TestInjectSdkOnly(t *testing.T) { inst := v1alpha1.Instrumentation{ Spec: v1alpha1.InstrumentationSpec{ diff --git a/pkg/instrumentation/upgrade/upgrade.go b/pkg/instrumentation/upgrade/upgrade.go index a3fed661ef..0b45be4fb0 100644 --- a/pkg/instrumentation/upgrade/upgrade.go +++ b/pkg/instrumentation/upgrade/upgrade.go @@ -36,6 +36,7 @@ type InstrumentationUpgrade struct { DefaultAutoInstPython string DefaultAutoInstDotNet string DefaultAutoInstApacheHttpd string + DefaultAutoInstNginx string DefaultAutoInstGo string } @@ -152,5 +153,18 @@ func (u *InstrumentationUpgrade) upgrade(_ context.Context, inst v1alpha1.Instru u.Recorder.Event(inst.DeepCopy(), "Warning", "InstrumentationUpgradeRejected", "support for Apache HTTPD auto instrumentation is not enabled") } } + autoInstNginx := inst.Annotations[v1alpha1.AnnotationDefaultAutoInstrumentationNginx] + if autoInstNginx != "" { + if featuregate.EnableNginxAutoInstrumentationSupport.IsEnabled() { + // upgrade the image only if the image matches the annotation + if inst.Spec.Nginx.Image == autoInstNginx { + inst.Spec.Nginx.Image = u.DefaultAutoInstNginx + inst.Annotations[v1alpha1.AnnotationDefaultAutoInstrumentationNginx] = u.DefaultAutoInstNginx + } + } else { + u.Logger.Error(nil, "support for Nginx auto instrumentation is not enabled") + u.Recorder.Event(inst.DeepCopy(), "Warning", "InstrumentationUpgradeRejected", "support for Nginx auto instrumentation is not enabled") + } + } return inst } diff --git a/pkg/instrumentation/upgrade/upgrade_test.go b/pkg/instrumentation/upgrade/upgrade_test.go index 88857742d5..f8d989810b 100644 --- a/pkg/instrumentation/upgrade/upgrade_test.go +++ b/pkg/instrumentation/upgrade/upgrade_test.go @@ -44,6 +44,12 @@ func TestUpgrade(t *testing.T) { require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableApacheHTTPAutoInstrumentationSupport.ID(), originalVal)) }) + originalVal = featuregate.EnableNginxAutoInstrumentationSupport.IsEnabled() + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), true)) + t.Cleanup(func() { + require.NoError(t, colfeaturegate.GlobalRegistry().Set(featuregate.EnableNginxAutoInstrumentationSupport.ID(), originalVal)) + }) + nsName := strings.ToLower(t.Name()) err := k8sClient.Create(context.Background(), &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -63,6 +69,7 @@ func TestUpgrade(t *testing.T) { v1alpha1.AnnotationDefaultAutoInstrumentationDotNet: "dotnet:1", v1alpha1.AnnotationDefaultAutoInstrumentationGo: "go:1", v1alpha1.AnnotationDefaultAutoInstrumentationApacheHttpd: "apache-httpd:1", + v1alpha1.AnnotationDefaultAutoInstrumentationNginx: "nginx:1", }, }, Spec: v1alpha1.InstrumentationSpec{ @@ -78,6 +85,7 @@ func TestUpgrade(t *testing.T) { assert.Equal(t, "dotnet:1", inst.Spec.DotNet.Image) assert.Equal(t, "go:1", inst.Spec.Go.Image) assert.Equal(t, "apache-httpd:1", inst.Spec.ApacheHttpd.Image) + assert.Equal(t, "nginx:1", inst.Spec.Nginx.Image) err = k8sClient.Create(context.Background(), inst) require.NoError(t, err) @@ -89,6 +97,7 @@ func TestUpgrade(t *testing.T) { DefaultAutoInstDotNet: "dotnet:2", DefaultAutoInstGo: "go:2", DefaultAutoInstApacheHttpd: "apache-httpd:2", + DefaultAutoInstNginx: "nginx:2", Client: k8sClient, } err = up.ManagedInstances(context.Background()) @@ -112,4 +121,6 @@ func TestUpgrade(t *testing.T) { assert.Equal(t, "go:2", updated.Spec.Go.Image) assert.Equal(t, "apache-httpd:2", updated.Annotations[v1alpha1.AnnotationDefaultAutoInstrumentationApacheHttpd]) assert.Equal(t, "apache-httpd:2", updated.Spec.ApacheHttpd.Image) + assert.Equal(t, "nginx:2", updated.Annotations[v1alpha1.AnnotationDefaultAutoInstrumentationNginx]) + assert.Equal(t, "nginx:2", updated.Spec.Nginx.Image) } diff --git a/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-collector.yaml b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-collector.yaml new file mode 100644 index 0000000000..f258017082 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-collector.yaml @@ -0,0 +1,21 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: sidecar +spec: + mode: sidecar + config: | + receivers: + otlp: + protocols: + grpc: + http: + processors: + exporters: + logging: + service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging] diff --git a/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-instrumentation.yaml b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-instrumentation.yaml new file mode 100644 index 0000000000..9c8297f897 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/00-install-instrumentation.yaml @@ -0,0 +1,17 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: nginx +spec: + exporter: + endpoint: http://localhost:4317 + propagators: + - jaeger + - b3 + sampler: + type: parentbased_traceidratio + argument: "0.25" + nginx: + attrs: + - name: NginxModuleOtelMaxQueueSize + value: "4096" diff --git a/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-assert.yaml b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-assert.yaml new file mode 100644 index 0000000000..ff21ccd679 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-assert.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + instrumentation.opentelemetry.io/inject-nginx: 'true' + sidecar.opentelemetry.io/inject: 'true' + labels: + app: my-nginx +spec: + containers: + - env: + - name: LD_LIBRARY_PATH + value: /opt:/opt/opentelemetry-webserver/agent/sdk_lib/lib + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: '0.25' + - name: OTEL_RESOURCE_ATTRIBUTES + name: myapp + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + - name: otel-nginx-agent + mountPath: /opt/opentelemetry-webserver/agent + - name: otel-nginx-conf-dir + mountPath: /etc/nginx + - args: + - --config=env:OTEL_CONFIG + name: otc-container + initContainers: + - args: + - cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt + name: otel-agent-source-container-clone + - name: otel-agent-attach-nginx + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + args: + - echo -e $OTEL_NGINX_I13N_SCRIPT > /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && chmod +x /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && cat /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh "/opt/opentelemetry-webserver/agent" "/opt/opentelemetry-webserver/source-conf" "nginx.conf" "<>" +status: + phase: Running diff --git a/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-install-app.yaml b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-install-app.yaml new file mode 100644 index 0000000000..6ea7c97659 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-contnr-secctx/01-install-app.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nginx +spec: + selector: + matchLabels: + app: my-nginx + replicas: 1 + template: + metadata: + labels: + app: my-nginx + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nginx: "true" + spec: + containers: + - name: myapp + image: nginxinc/nginx-unprivileged:1.23.1 + imagePullPolicy: Always + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + ports: + - containerPort: 8765 + env: + - name: LD_LIBRARY_PATH + value: /opt + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 250m + memory: 100Mi + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + # user nginx; + worker_processes 1; + events { + worker_connections 10240; + } + http { + include /etc/nginx/conf.d/*.conf; + server { + listen 8765; + server_name localhost; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location /api/customer/ { + proxy_pass http://localhost:8282/api/customer/; + } + location /api/vendor/ { + proxy_pass http://localhost:8383/api/vendor/; + } + + location /seznam { + proxy_pass http://www.seznam.cz/; + } + } + + } + +--- diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-collector.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-collector.yaml new file mode 100644 index 0000000000..6606f291da --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-collector.yaml @@ -0,0 +1,22 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: sidecar +spec: + mode: sidecar + config: | + receivers: + otlp: + protocols: + grpc: + http: + processors: + exporters: + logging: + service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging] diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-instrumentation.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-instrumentation.yaml new file mode 100644 index 0000000000..1f5296bc0b --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/00-install-instrumentation.yaml @@ -0,0 +1,19 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: nginx +spec: + exporter: + endpoint: http://localhost:4317 + propagators: + - jaeger + - b3 + sampler: + type: parentbased_traceidratio + argument: "0.25" + nginx: + attrs: + - name: NginxModuleOtelMaxQueueSize + value: "4096" + diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-assert.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-assert.yaml new file mode 100644 index 0000000000..f66a3ebb39 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-assert.yaml @@ -0,0 +1,77 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: v1 +kind: Pod +metadata: + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nginx: "true" + instrumentation.opentelemetry.io/container-names: "myapp,myrabbit" + labels: + app: my-nginx +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - env: + - name: LD_LIBRARY_PATH + value: /opt:/opt/opentelemetry-webserver/agent/sdk_lib/lib + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.25" + - name: OTEL_RESOURCE_ATTRIBUTES + name: myapp + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + - mountPath: /opt/opentelemetry-webserver/agent + name: otel-nginx-agent + - mountPath: /etc/nginx + name: otel-nginx-conf-dir + - env: + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.25" + - name: OTEL_RESOURCE_ATTRIBUTES + name: myrabbit + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + - args: + - --config=env:OTEL_CONFIG + name: otc-container +status: + phase: Running diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-install-app.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-install-app.yaml new file mode 100644 index 0000000000..03ccf5c338 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/01-install-app.yaml @@ -0,0 +1,93 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nginx +spec: + selector: + matchLabels: + app: my-nginx + replicas: 1 + template: + metadata: + labels: + app: my-nginx + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nginx: "true" + instrumentation.opentelemetry.io/container-names: "myapp,myrabbit" + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - name: myapp + image: nginxinc/nginx-unprivileged:1.23.1 + imagePullPolicy: Always + ports: + - containerPort: 8765 + env: + - name: LD_LIBRARY_PATH + value: /opt + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + - name: myrabbit + image: rabbitmq + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + # user nginx; + worker_processes 1; + events { + worker_connections 10240; + } + http { + include /etc/nginx/conf.d/*.conf; + server { + listen 8765; + server_name localhost; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location /api/customer/ { + proxy_pass http://localhost:8282/api/customer/; + } + location /api/vendor/ { + proxy_pass http://localhost:8383/api/vendor/; + } + + location /seznam { + proxy_pass http://www.seznam.cz/; + } + } + + } + +--- \ No newline at end of file diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-assert.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-assert.yaml new file mode 100644 index 0000000000..ea590a88d2 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-assert.yaml @@ -0,0 +1,74 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: v1 +kind: Pod +metadata: + annotations: + instrumentation.opentelemetry.io/inject-nginx: "true" + sidecar.opentelemetry.io/inject: "true" + labels: + app: my-nginx +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - env: + - name: LD_LIBRARY_PATH + value: /opt:/opt/opentelemetry-webserver/agent/sdk_lib/lib + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.25" + - name: OTEL_RESOURCE_ATTRIBUTES + name: myapp + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + - mountPath: /opt/opentelemetry-webserver/agent + name: otel-nginx-agent + - mountPath: /etc/nginx + name: otel-nginx-conf-dir + - env: + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.25" + - name: OTEL_RESOURCE_ATTRIBUTES + name: myrabbit + - args: + - --config=env:OTEL_CONFIG + name: otc-container +status: + phase: Running diff --git a/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-install-app.yaml b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-install-app.yaml new file mode 100644 index 0000000000..98766fdb6f --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx-multicontainer/02-install-app.yaml @@ -0,0 +1,92 @@ +# skipping test, see https://github.com/open-telemetry/opentelemetry-operator/issues/1936 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nginx +spec: + selector: + matchLabels: + app: my-nginx + replicas: 1 + template: + metadata: + labels: + app: my-nginx + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nginx: "true" + instrumentation.opentelemetry.io/container-names: "myrabbit" + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - name: myapp + image: nginxinc/nginx-unprivileged:1.23.1 + imagePullPolicy: Always + ports: + - containerPort: 8765 + env: + - name: LD_LIBRARY_PATH + value: /opt + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + imagePullPolicy: Always + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + - name: myrabbit + image: rabbitmq + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + # user nginx; + worker_processes 1; + events { + worker_connections 10240; + } + http { + include /etc/nginx/conf.d/*.conf; + server { + listen 8765; + server_name localhost; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location /api/customer/ { + proxy_pass http://localhost:8282/api/customer/; + } + location /api/vendor/ { + proxy_pass http://localhost:8383/api/vendor/; + } + + location /seznam { + proxy_pass http://www.seznam.cz/; + } + } + + } + +--- \ No newline at end of file diff --git a/tests/e2e-instrumentation/instrumentation-nginx/00-install-collector.yaml b/tests/e2e-instrumentation/instrumentation-nginx/00-install-collector.yaml new file mode 100644 index 0000000000..f258017082 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx/00-install-collector.yaml @@ -0,0 +1,21 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: sidecar +spec: + mode: sidecar + config: | + receivers: + otlp: + protocols: + grpc: + http: + processors: + exporters: + logging: + service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging] diff --git a/tests/e2e-instrumentation/instrumentation-nginx/00-install-instrumentation.yaml b/tests/e2e-instrumentation/instrumentation-nginx/00-install-instrumentation.yaml new file mode 100644 index 0000000000..9c8297f897 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx/00-install-instrumentation.yaml @@ -0,0 +1,17 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: nginx +spec: + exporter: + endpoint: http://localhost:4317 + propagators: + - jaeger + - b3 + sampler: + type: parentbased_traceidratio + argument: "0.25" + nginx: + attrs: + - name: NginxModuleOtelMaxQueueSize + value: "4096" diff --git a/tests/e2e-instrumentation/instrumentation-nginx/01-assert.yaml b/tests/e2e-instrumentation/instrumentation-nginx/01-assert.yaml new file mode 100644 index 0000000000..38474a2a09 --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx/01-assert.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + instrumentation.opentelemetry.io/inject-nginx: 'true' + sidecar.opentelemetry.io/inject: 'true' + labels: + app: my-nginx +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - env: + - name: LD_LIBRARY_PATH + value: /opt:/opt/opentelemetry-webserver/agent/sdk_lib/lib + - name: OTEL_SERVICE_NAME + value: my-nginx + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317 + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_PROPAGATORS + value: jaeger,b3 + - name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio + - name: OTEL_TRACES_SAMPLER_ARG + value: '0.25' + - name: OTEL_RESOURCE_ATTRIBUTES + name: myapp + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + - name: otel-nginx-agent + mountPath: /opt/opentelemetry-webserver/agent + - name: otel-nginx-conf-dir + mountPath: /etc/nginx + - args: + - --config=env:OTEL_CONFIG + name: otc-container + initContainers: + - args: + - cp -r /etc/nginx/* /opt/opentelemetry-webserver/source-conf && export NGINX_VERSION=$( { nginx -v ; } 2>&1 ) && echo ${NGINX_VERSION##*/} > /opt/opentelemetry-webserver/source-conf/version.txt + name: otel-agent-source-container-clone + - name: otel-agent-attach-nginx + args: + - echo -e $OTEL_NGINX_I13N_SCRIPT > /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && chmod +x /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && cat /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh && /opt/opentelemetry-webserver/agent/nginx_instrumentation.sh "/opt/opentelemetry-webserver/agent" "/opt/opentelemetry-webserver/source-conf" "nginx.conf" "<>" +status: + phase: Running diff --git a/tests/e2e-instrumentation/instrumentation-nginx/01-install-app.yaml b/tests/e2e-instrumentation/instrumentation-nginx/01-install-app.yaml new file mode 100644 index 0000000000..bd0530e56e --- /dev/null +++ b/tests/e2e-instrumentation/instrumentation-nginx/01-install-app.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nginx +spec: + selector: + matchLabels: + app: my-nginx + replicas: 1 + template: + metadata: + labels: + app: my-nginx + annotations: + sidecar.opentelemetry.io/inject: "true" + instrumentation.opentelemetry.io/inject-nginx: "true" + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - name: myapp + image: nginxinc/nginx-unprivileged:1.23.1 + imagePullPolicy: Always + ports: + - containerPort: 8765 + env: + - name: LD_LIBRARY_PATH + value: /opt + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + imagePullPolicy: Always + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 250m + memory: 100Mi + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + # user nginx; + worker_processes 1; + events { + worker_connections 10240; + } + http { + include /etc/nginx/conf.d/*.conf; + server { + listen 8765; + server_name localhost; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + location /api/customer/ { + proxy_pass http://localhost:8282/api/customer/; + } + location /api/vendor/ { + proxy_pass http://localhost:8383/api/vendor/; + } + + location /seznam { + proxy_pass http://www.seznam.cz/; + } + } + + } + +--- diff --git a/versions.txt b/versions.txt index 086276688e..877032de70 100644 --- a/versions.txt +++ b/versions.txt @@ -35,3 +35,7 @@ autoinstrumentation-go=v0.3.0-alpha # Represents the current release of Apache HTTPD instrumentation. # Should match autoinstrumentation/apache-httpd/version.txt autoinstrumentation-apache-httpd=1.0.3 + +# Represents the current release of Apache HTTPD instrumentation. +# Should match autoinstrumentation/apache-httpd/version.txt +autoinstrumentation-nginx=1.0.3