add pomerium_otel tracer extension

.bazelignore

@@ -0,0 +1,4 @@
+bazel-bin
+bazel-envoy-custom
+bazel-out
+bazel-testlogs

.gitignore

@@ -3,4 +3,5 @@ clang.bazelrc
 user.bazelrc
 compile_commands.json
 .vscode
-user.bazelrc
+.cache
+user.bazelrc

BUILD

@@ -13,6 +13,7 @@ envoy_cc_binary(
     deps = [
         "//source/extensions/http/early_header_mutation/trace_context:pomerium_trace_context",
         "//source/extensions/request_id/uuidx:pomerium_uuidx",
+        "//source/extensions/tracers/pomerium_otel",
         "@envoy//source/exe:envoy_main_entry_lib",
     ],
 )

WORKSPACE

@@ -12,6 +12,7 @@ http_archive(
     patch_tool = "patch",
     patches = [
         "//:patches/0001-fix-otel-grpc-trace-exporter.patch",
+        "//:patches/0002-opentelemetry-tracer-lib-visibility.patch",
     ],
     strip_prefix = "envoy-" + envoy_version,
     url = "https://github.com/envoyproxy/envoy/archive/refs/tags/v" + envoy_version + ".zip",

patches/0002-opentelemetry-tracer-lib-visibility.patch

@@ -0,0 +1,10 @@
+diff --git a/source/extensions/tracers/opentelemetry/BUILD b/source/extensions/tracers/opentelemetry/BUILD
+index 0b6e75abea..0cac8092aa 100644
+--- a/source/extensions/tracers/opentelemetry/BUILD
++++ b/source/extensions/tracers/opentelemetry/BUILD
+@@ -24,4 +24,5 @@ envoy_cc_extension(
+
+ envoy_cc_library(
++    visibility = ["//visibility:public"],
+     name = "opentelemetry_tracer_lib",
+     srcs = [

source/extensions/http/early_header_mutation/trace_context/BUILD

@@ -23,6 +23,7 @@ envoy_cc_library(
         ":trace_context_cc_proto",
         "@envoy//envoy/http:early_header_mutation_interface",
         "@envoy//envoy/registry",
+        "@envoy//source/common/common:base64_lib",
         "@envoy//source/common/common:logger_lib",
         "@envoy//source/common/http:header_mutation_lib",
     ],
@@ -45,6 +46,7 @@ envoy_cc_test(
     deps = [
         ":pomerium_trace_context",
         ":trace_context_cc_proto",
+        "@com_google_absl//absl/random",
         "@envoy//source/common/common:random_generator_lib",
         "@envoy//test/mocks/runtime:runtime_mocks",
         "@envoy//test/mocks/stream_info:stream_info_mocks",

source/extensions/http/early_header_mutation/trace_context/trace_context.cc

@@ -1,5 +1,6 @@
 #include "source/extensions/http/early_header_mutation/trace_context/trace_context.h"
 #include "source/common/common/logger.h"
+#include "source/common/common/base64.h"
 
 namespace Envoy::Extensions::Http::EarlyHeaderMutation {
 
@@ -23,7 +24,8 @@ bool TraceContext::mutate(Envoy::Http::RequestHeaderMap& headers,
    *
    * If the x-pomerium-traceparent header is present, the sampling decision will
    * be set in the x-pomerium-sampling-decision header. This header is then read
-   * by the uuidx extension to force the desired trace decision if necessary.
+   * by the uuidx extension to ensure consistency in request IDs, and by the
+   * pomerium_otel extension to force a sampling decision in newly created spans.
    */
 
   headers.remove(pomerium_external_parent_header);
@@ -37,7 +39,33 @@ bool TraceContext::mutate(Envoy::Http::RequestHeaderMap& headers,
       if (auto traceparent = values[0]->value().getStringView(); traceparent.size() == 55) {
         headers.setCopy(pomerium_external_parent_header, traceparent.substr(36, 16));
       }
+    } else if (headers.getPathValue().starts_with("/oauth2/callback")) {
+      // oauth2 callback is a special case, we can't encode the traceparent in the usual way since
+      // the query parameters are standard and managed by oauth2 clients
+      const auto state = params.getFirstValue(Envoy::Http::LowerCaseString("state"));
+      if (state.has_value()) {
+        // The Pomerium state format looks like:
+        // nonce|timestamp|trace_id+flags|encrypted_data(redirect_url)+mac(nonce,ts)
+        // The trace ID segment can be empty if this request was not traced. If so, the delimiter
+        // is still present (e.g. the state will be "nonce|timestamp||encrypted_data").
+        const std::string stateDecoded =
+            Base64Url::decode(StringUtil::removeTrailingCharacters(state.value(), '='));
+        // The encrypted data is not base64-encoded like the other fields, so read only up to the
+        // third delimiter, instead of trying to split the entire string.
+        const std::vector<absl::string_view> segments =
+            absl::StrSplit(stateDecoded, absl::MaxSplits('|', 3));
+        if (segments.size() == 4) {
+          const auto traceidDecoded = Base64Url::decode(segments[2]);
+          if (traceidDecoded.size() == 17) { // 16 byte trace ID + 1 byte flags
+            const auto traceID = traceidDecoded.substr(0, 16);
+            const auto flags = traceidDecoded.at(16);
+            headers.setCopy(pomerium_traceid_header, absl::BytesToHexString(traceID));
+            headers.setCopy(pomerium_sampling_decision_header, (flags & 1) == 1 ? "1" : "0");
+          }
+        }
+      }
     }
+
     return true;
   }
 
@@ -56,12 +84,8 @@ bool TraceContext::mutate(Envoy::Http::RequestHeaderMap& headers,
   auto flags = absl::HexStringToBytes(flags_hex).front();
   if (flags & 1) { // sampled
     headers.setCopy(pomerium_sampling_decision_header, "1");
-    ENVOY_LOG(debug, "pomerium_traceparent={}, forcing trace decision (on)",
-              pomerium_traceparent.value());
   } else { // not sampled
     headers.setCopy(pomerium_sampling_decision_header, "0");
-    ENVOY_LOG(debug, "pomerium_traceparent={}, forcing sampling decision (off)",
-              pomerium_traceparent.value());
   }
 
   headers.setCopy(pomerium_traceparent_header, pomerium_traceparent.value());

source/extensions/http/early_header_mutation/trace_context/trace_context.h

@@ -20,6 +20,7 @@ class TraceContext : public Envoy::Http::EarlyHeaderMutation,
   Envoy::Http::LowerCaseString pomerium_sampling_decision_header{"x-pomerium-sampling-decision"};
   Envoy::Http::LowerCaseString pomerium_traceparent_header{"x-pomerium-traceparent"};
   Envoy::Http::LowerCaseString pomerium_tracestate_header{"x-pomerium-tracestate"};
+  Envoy::Http::LowerCaseString pomerium_traceid_header{"x-pomerium-traceid"};
   Envoy::Http::LowerCaseString pomerium_external_parent_header{"x-pomerium-external-parent-span"};
   Envoy::Http::LowerCaseString traceparent_header{"traceparent"};
 };

source/extensions/http/early_header_mutation/trace_context/trace_context_test.cc

@@ -2,6 +2,8 @@
 
 #include "test/mocks/stream_info/mocks.h"
 #include "test/test_common/utility.h"
+#include "source/common/common/base64.h"
+#include "absl/random/random.h"
 
 #include "gtest/gtest.h"
 
@@ -60,23 +62,23 @@ TEST(TraceContextTest, Mutate) {
   }
   {
     Envoy::Http::TestRequestHeaderMapImpl request_headers{
-        {":path", fmt::format("/foo/bar?pomerium_traceparent=invalid")},
+        {":path", "/foo/bar?pomerium_traceparent=invalid"},
     };
     EXPECT_TRUE(tc.mutate(request_headers, stream_info));
     EXPECT_FALSE(request_headers.has("x-pomerium-sampling-decision"));
     EXPECT_FALSE(request_headers.has("x-pomerium-traceparent"));
   }
   {
     Envoy::Http::TestRequestHeaderMapImpl request_headers{
-        {":path", fmt::format("/foo/bar")},
+        {":path", "/foo/bar"},
     };
     EXPECT_TRUE(tc.mutate(request_headers, stream_info));
     EXPECT_FALSE(request_headers.has("x-pomerium-sampling-decision"));
     EXPECT_FALSE(request_headers.has("x-pomerium-traceparent"));
   }
   {
     Envoy::Http::TestRequestHeaderMapImpl request_headers{
-        {":path", fmt::format("/foo/bar?pomerium_traceparent=00-1-2-??")},
+        {":path", "/foo/bar?pomerium_traceparent=00-1-2-??"},
     };
     EXPECT_TRUE(tc.mutate(request_headers, stream_info));
     EXPECT_FALSE(request_headers.has("x-pomerium-sampling-decision"));
@@ -90,6 +92,26 @@ TEST(TraceContextTest, Mutate) {
     EXPECT_EQ("2222222222222222", request_headers.get_("x-pomerium-external-parent-span"));
     EXPECT_FALSE(request_headers.has("x-pomerium-traceparent"));
   }
+  {
+    absl::BitGen bitgen;
+    const auto traceid_bytes =
+        absl::StrCat(absl::HexStringToBytes("11111111111111111111111111111111"), 1);
+    EXPECT_EQ(traceid_bytes.size(), 17);
+    const auto encoded_traceid = Base64Url::encode(traceid_bytes.c_str(), traceid_bytes.size());
+    char random_bytes[64];
+    for (int i = 0; i < 64; i++) {
+      random_bytes[i] = absl::Uniform<char>(bitgen, -128, 127);
+    }
+    const auto state = absl::StrCat("foo|bar|", encoded_traceid, "|", random_bytes);
+    auto state_encoded = Base64Url::encode(state.c_str(), state.size());
+    Base64::completePadding(state_encoded); // match go base64url encoding
+    Envoy::Http::TestRequestHeaderMapImpl request_headers{
+        {":path", absl::StrCat("/oauth2/callback?code=xyz&state=", state_encoded)},
+    };
+    EXPECT_TRUE(tc.mutate(request_headers, stream_info));
+    EXPECT_EQ("11111111111111111111111111111111", request_headers.get_("x-pomerium-traceid"));
+    EXPECT_EQ("1", request_headers.get_("x-pomerium-sampling-decision"));
+  }
 }
 
 } // namespace Envoy::Extensions::Http::EarlyHeaderMutation

source/extensions/tracers/pomerium_otel/BUILD

@@ -0,0 +1,65 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_cc_library",
+    "envoy_cc_test",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+envoy_cc_library(
+    name = "pomerium_otel",
+    srcs = [
+        "config.cc",
+        "span.cc",
+        "tracer_impl.cc",
+    ],
+    hdrs = [
+        "config.h",
+        "span.h",
+        "tracer_impl.h",
+        "typeutils.h",
+    ],
+    repository = "@envoy",
+    deps = [
+        ":pomerium_otel_cc_proto",
+        "@envoy//source/common/common:logger_lib",
+        "@envoy//source/common/common:utility_lib",
+        "@envoy//source/extensions/tracers/opentelemetry:opentelemetry_tracer_lib",
+    ],
+)
+
+cc_proto_library(
+    name = "pomerium_otel_cc_proto",
+    deps = ["pomerium_otel_proto"],
+)
+
+proto_library(
+    name = "pomerium_otel_proto",
+    srcs = ["pomerium_otel.proto"],
+    deps = [
+        "@com_envoyproxy_protoc_gen_validate//validate:validate_proto",
+        "@com_github_cncf_xds//udpa/annotations:pkg",
+        "@envoy_api//envoy/annotations:pkg",
+        "@envoy_api//envoy/config/core/v3:pkg",
+    ],
+)
+
+envoy_cc_test(
+    name = "pomerium_otel_test",
+    srcs = [
+        "pomerium_otel_test.cc",
+    ],
+    repository = "@envoy",
+    deps = [
+        ":pomerium_otel",
+        ":pomerium_otel_cc_proto",
+        "@envoy//test/mocks/server:tracer_factory_context_mocks",
+        "@envoy//test/mocks/stream_info:stream_info_mocks",
+        "@envoy//test/mocks/thread_local:thread_local_mocks",
+        "@envoy//test/mocks/tracing:tracing_mocks",
+        "@envoy//test/mocks/upstream:cluster_manager_mocks",
+        "@envoy//test/test_common:utility_lib",
+    ],
+)

source/extensions/tracers/pomerium_otel/config.cc

@@ -0,0 +1,19 @@
+#include "source/extensions/tracers/pomerium_otel/config.h"
+
+#include "envoy/registry/registry.h"
+#include "source/extensions/tracers/pomerium_otel/tracer_impl.h"
+
+namespace Envoy::Extensions::Tracers::OpenTelemetry {
+
+PomeriumOpenTelemetryTracerFactory::PomeriumOpenTelemetryTracerFactory()
+    : FactoryBase("envoy.tracers.pomerium_otel") {}
+
+Tracing::DriverSharedPtr PomeriumOpenTelemetryTracerFactory::createTracerDriverTyped(
+    const pomerium::extensions::OpenTelemetryConfig& proto_config,
+    Server::Configuration::TracerFactoryContext& context) {
+  return std::make_shared<PomeriumDriver>(toBaseConfig(proto_config), context);
+}
+
+REGISTER_FACTORY(PomeriumOpenTelemetryTracerFactory, Server::Configuration::TracerFactory);
+
+} // namespace Envoy::Extensions::Tracers::OpenTelemetry

source/extensions/tracers/pomerium_otel/config.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "source/extensions/tracers/pomerium_otel/pomerium_otel.pb.h"
+#include "source/extensions/tracers/pomerium_otel/typeutils.h"
+
+#include "source/extensions/tracers/common/factory_base.h"
+
+namespace Envoy::Extensions::Tracers::OpenTelemetry {
+
+class PomeriumOpenTelemetryTracerFactory
+    : Logger::Loggable<Logger::Id::tracing>,
+      public Common::FactoryBase<pomerium::extensions::OpenTelemetryConfig> {
+public:
+  PomeriumOpenTelemetryTracerFactory();
+
+private:
+  // FactoryBase
+  Tracing::DriverSharedPtr
+  createTracerDriverTyped(const pomerium::extensions::OpenTelemetryConfig& proto_config,
+                          Server::Configuration::TracerFactoryContext& context) override;
+};
+
+} // namespace Envoy::Extensions::Tracers::OpenTelemetry

source/extensions/tracers/pomerium_otel/pomerium_otel.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package pomerium.extensions;
+
+import "envoy/config/core/v3/extension.proto";
+import "envoy/config/core/v3/grpc_service.proto";
+import "envoy/config/core/v3/http_service.proto";
+import "udpa/annotations/migrate.proto";
+import "udpa/annotations/status.proto";
+
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+message OpenTelemetryConfig {
+  envoy.config.core.v3.GrpcService                   grpc_service       = 1 [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"];
+  envoy.config.core.v3.HttpService                   http_service       = 3 [(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"];
+  string                                             service_name       = 2;
+  repeated envoy.config.core.v3.TypedExtensionConfig resource_detectors = 4;
+  envoy.config.core.v3.TypedExtensionConfig          sampler            = 5;
+}