OpenTelemetry support

.github/workflows/ci.yaml

@@ -103,14 +103,15 @@ jobs:
           REGISTRY: ingress-controller
         run: |
           echo "building images..."
-          make clean-image build image image-chroot
+          make clean-image build image image-chroot image-modules
           make -C test/e2e-image image
 
           echo "creating images cache..."
           docker save \
             nginx-ingress-controller:e2e \
             ingress-controller/controller:1.0.0-dev \
             ingress-controller/controller-chroot:1.0.0-dev \
+            ingress-controller/opentelemetry:1.0.0-dev \
             | pigz > docker.tar.gz
 
       - name: cache

Makefile

@@ -139,6 +139,15 @@ build-plugin:  ## Build ingress-nginx krew plugin.
 		build/build-plugin.sh
 
 
+.PHONY: image-modules
+image-modules: image-module-opentelemetry ## Builds images for all the modules
+
+image-module-%: ## Builds the image for the required module
+	echo "Building $* docker image ($(ARCH))..."
+	@docker build \
+		--no-cache \
+		-t $(REGISTRY)/$*:$(TAG) images/$*/rootfs
+
 .PHONY: clean
 clean: ## Remove .gocache directory.
 	rm -rf bin/ .gocache/ .cache/

internal/ingress/annotations/annotations.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/loadbalancing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/log"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/mirror"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/opentracing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/portinredirect"
@@ -92,6 +93,7 @@ type Ingress struct {
 	ExternalAuth       authreq.Config
 	EnableGlobalAuth   bool
 	HTTP2PushPreload   bool
+	OpenTelemetry      opentelemetry.Config
 	Opentracing        opentracing.Config
 	Proxy              proxy.Config
 	ProxySSL           proxyssl.Config
@@ -142,6 +144,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"ExternalAuth":         authreq.NewParser(cfg),
 			"EnableGlobalAuth":     authreqglobal.NewParser(cfg),
 			"HTTP2PushPreload":     http2pushpreload.NewParser(cfg),
+			"OpenTelemetry":        opentelemetry.NewParser(cfg),
 			"Opentracing":          opentracing.NewParser(cfg),
 			"Proxy":                proxy.NewParser(cfg),
 			"ProxySSL":             proxyssl.NewParser(cfg),

internal/ingress/annotations/opentelemetry/main.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	networking "k8s.io/api/networking/v1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type opentelemetry struct {
+	r resolver.Resolver
+}
+
+// Config contains the configuration to be used in the Ingress
+type Config struct {
+	Enabled      bool `json:"enabled"`
+	Set          bool `json:"set"`
+	TrustEnabled bool `json:"trust-enabled"`
+	TrustSet     bool `json:"trust-set"`
+}
+
+// Equal tests for equality between two Config types
+func (bd1 *Config) Equal(bd2 *Config) bool {
+	if bd1.Set != bd2.Set {
+		return false
+	}
+
+	if bd1.Enabled != bd2.Enabled {
+		return false
+	}
+
+	if bd1.TrustEnabled != bd2.TrustEnabled {
+		return false
+	}
+
+	return true
+}
+
+// NewParser creates a new OpenTelemetry annotation parser
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return opentelemetry{r}
+}
+
+func (s opentelemetry) Parse(ing *networking.Ingress) (interface{}, error) {
+	enabled, err := parser.GetBoolAnnotation("enable-opentelemetry", ing)
+	if err != nil {
+		return &Config{Set: false, Enabled: false}, nil
+	}
+
+	trustSpan, err := parser.GetBoolAnnotation("opentelemetry-trust-incoming-span", ing)
+	if err != nil {
+		return &Config{Set: true, Enabled: enabled}, nil
+	}
+
+	return &Config{Set: true, Enabled: enabled, TrustSet: true, TrustEnabled: trustSpan}, nil
+}

internal/ingress/annotations/opentelemetry/main_test.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	"testing"
+
+	api "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+func buildIngress() *networking.Ingress {
+	defaultBackend := networking.IngressBackend{
+		Service: &networking.IngressServiceBackend{
+			Name: "default-backend",
+			Port: networking.ServiceBackendPort{
+				Number: 80,
+			},
+		},
+	}
+
+	return &networking.Ingress{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: networking.IngressSpec{
+			DefaultBackend: &networking.IngressBackend{
+				Service: &networking.IngressServiceBackend{
+					Name: "default-backend",
+					Port: networking.ServiceBackendPort{
+						Number: 80,
+					},
+				},
+			},
+			Rules: []networking.IngressRule{
+				{
+					Host: "foo.bar.com",
+					IngressRuleValue: networking.IngressRuleValue{
+						HTTP: &networking.HTTPIngressRuleValue{
+							Paths: []networking.HTTPIngressPath{
+								{
+									Path:    "/foo",
+									Backend: defaultBackend,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestIngressAnnotationOpenTelemetrySetTrue(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if !openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+}
+
+func TestIngressAnnotationOpenTelemetrySetFalse(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with explicitly set to false
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "false"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be false, got true")
+	}
+}
+
+func TestIngressAnnotationOpenTelemetryUnset(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with no annotation specified
+	data := map[string]string{}
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	_, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+}

internal/ingress/controller/config/config.go

@@ -559,6 +559,28 @@ type Configuration struct {
 	// Default: true
 	OpentracingTrustIncomingSpan bool `json:"opentracing-trust-incoming-span"`
 
+	// EnableOpenTelemetry enables the nginx OpenTelemetry extension
+	// https://github.com/open-telemetry/opentelemetry-cpp-contrib
+	// By default this is disabled
+	EnableOpenTelemetry bool `json:"enable-opentelemetry"`
+
+	// OpenTelemetryOperationName specifies a custom name for the server span
+	OpenTelemetryOperationName string `json:"opentelemetry-operation-name"`
+
+	// OpenTelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans
+	// If false, incoming span headers will be rejected
+	// Default: true
+	OpenTelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"`
+
+	// OtlpExporterHost defines the host of the OpenTelemetry collector instance
+	// where the data will be transmitted
+	OtlpCollectorHost string `json:"otlp-collector-host"`
+
+	// OtlpExporterPost defines the port of the OpenTelemetry collector instance
+	// where the data will be transmitted
+	// Default: 4318
+	OtlpCollectorPort int `json:"otlp-collector-port"`
+
 	// ZipkinCollectorHost specifies the host to use when uploading traces
 	ZipkinCollectorHost string `json:"zipkin-collector-host"`
 
@@ -908,6 +930,7 @@ func NewDefault() Configuration {
 		BindAddressIpv4:                        defBindAddress,
 		BindAddressIpv6:                        defBindAddress,
 		OpentracingTrustIncomingSpan:           true,
+		OpenTelemetryTrustIncomingSpan:         true,
 		ZipkinCollectorPort:                    9411,
 		ZipkinServiceName:                      "nginx",
 		ZipkinSampleRate:                       1.0,

internal/ingress/controller/nginx.go

@@ -667,6 +667,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		return err
 	}
 
+	err = createOpenTelemetryCfg(cfg, "/etc/nginx/opentelemetry.toml")
+	if err != nil {
+		return err
+	}
+
 	err = createOpentracingCfg(cfg)
 	if err != nil {
 		return err
@@ -1036,6 +1041,42 @@ func configureCertificates(rawServers []*ingress.Server) error {
 	return nil
 }
 
+const otlpTmpl = `
+exporter = "otlp"
+processor = "simple"
+
+{{- if .OtlpCollectorHost }}
+[exporters.otlp]
+host = "{{.OtlpCollectorHost}}"
+{{ if .OtlpCollectorPort }}port = {{.OtlpCollectorPort}}{{- end }}
+{{- end }}
+
+[service]
+name = "ingress-nginx"
+`
+
+func createOpenTelemetryCfg(cfg ngx_config.Configuration, configPath string) error {
+	var tmpl *template.Template
+	var err error
+
+	if !cfg.EnableOpenTelemetry {
+		return nil
+	}
+
+	tmpl, _ = template.New("opentelemetry").Parse(otlpTmpl)
+
+	tmplBuf := bytes.NewBuffer(make([]byte, 0))
+	err = tmpl.Execute(tmplBuf, cfg)
+	if err != nil {
+		return err
+	}
+
+	// Expand possible environment variables before writing the configuration to file.
+	expanded := os.ExpandEnv(tmplBuf.String())
+
+	return os.WriteFile(configPath, []byte(expanded), file.ReadWriteByUser)
+}
+
 const zipkinTmpl = `{
   "service_name": "{{ .ZipkinServiceName }}",
   "collector_host": "{{ .ZipkinCollectorHost }}",