Add an OpenTelemetry (gRPC) access logger extension (#15105)

This PR adds an OTLP logger to Envoy, which allows users to flexibly configure their log structure (according to OpenTelemetry protos).
The API users the common gRPC configuration, used in ALS, for gRPC settings. The flexible OT fields (body, attributes) are completely configurable and are filled with request/response information based on Envoy access log formatting.
There are still a couple of TODOs -

The 'filter_state_objects_to_log' from the common gRPC config isn't added to the log
Resource attributes are not configurable, and only Envoy node info is added.
Note: this extension was added as v3alpha API.

Risk Level: Low - a new extension, so unlikely to break any existing setups (marked as v3alpha API).
Testing: Unit tests.
Docs Changes:
Release Notes: access log: added a new :ref:OpenTelemetry access logger <envoy_v3_api_msg_extensions.access_loggers.open_telemetry.v3alpha.OpenTelemetryAccessLogConfig> extension, allowing a flexible log structure with native Envoy access log formatting.

Signed-off-by: Itamar Kaminski <[email protected]>

api/BUILD

@@ -155,6 +155,7 @@ proto_library(
         "//envoy/data/tap/v3:pkg",
         "//envoy/extensions/access_loggers/file/v3:pkg",
         "//envoy/extensions/access_loggers/grpc/v3:pkg",
+        "//envoy/extensions/access_loggers/open_telemetry/v3alpha:pkg",
         "//envoy/extensions/access_loggers/stderror/v3:pkg",
         "//envoy/extensions/access_loggers/stdoutput/v3:pkg",
         "//envoy/extensions/access_loggers/wasm/v3:pkg",

api/bazel/external_proto_deps.bzl

@@ -14,6 +14,7 @@ EXTERNAL_PROTO_IMPORT_BAZEL_DEP_MAP = {
     "metrics.proto": "@prometheus_metrics_model//:client_model",
     "opencensus/proto/trace/v1/trace.proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto",
     "opencensus/proto/trace/v1/trace_config.proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto",
+    "opentelemetry/proto/common/v1/common.proto": "@opentelemetry_proto//:logs",
 }
 
 # This maps from the Bazel proto_library target to the Go language binding target for external dependencies.
@@ -22,6 +23,7 @@ EXTERNAL_PROTO_GO_BAZEL_DEP_MAP = {
     "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:expr_go_proto",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go",
+    "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_go_proto",
 }
 
 # This maps from the Bazel proto_library target to the C++ language binding target for external dependencies.
@@ -30,6 +32,7 @@ EXTERNAL_PROTO_CC_BAZEL_DEP_MAP = {
     "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:syntax_cc_proto",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_cc",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto_cc",
+    "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_cc_proto",
 }
 
 # This maps from the Bazel proto_library target to the Python language binding target for external dependencies.
@@ -38,4 +41,5 @@ EXTERNAL_PROTO_PY_BAZEL_DEP_MAP = {
     "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:syntax_py_proto",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_py",
     "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto_py",
+    "@opentelemetry_proto//:logs": "@opentelemetry_proto//:logs_py_proto",
 }

api/bazel/repositories.bzl

@@ -110,10 +110,10 @@ go_proto_library(
 """
 
 OPENTELEMETRY_LOGS_BUILD_CONTENT = """
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_cc//cc:defs.bzl", "cc_proto_library")
+load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 
-proto_library(
+api_cc_py_proto_library(
     name = "logs",
     srcs = [
         "opentelemetry/proto/collector/logs/v1/logs_service.proto",
@@ -124,9 +124,10 @@ proto_library(
     visibility = ["//visibility:public"],
 )
 
-cc_proto_library(
-    name = "logs_cc_proto",
-    deps = [":logs"],
+go_proto_library(
+    name = "logs_go_proto",
+    importpath = "logs",
+    proto = ":logs",
     visibility = ["//visibility:public"],
 )
 """

api/envoy/extensions/access_loggers/open_telemetry/v3alpha/BUILD

@@ -0,0 +1,13 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "//envoy/extensions/access_loggers/grpc/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+        "@opentelemetry_proto//:logs",
+    ],
+)

api/envoy/extensions/access_loggers/open_telemetry/v3alpha/logs_service.proto

@@ -0,0 +1,42 @@
+syntax = "proto3";
+
+package envoy.extensions.access_loggers.open_telemetry.v3alpha;
+
+import "envoy/extensions/access_loggers/grpc/v3/als.proto";
+
+import "opentelemetry/proto/common/v1/common.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.access_loggers.open_telemetry.v3alpha";
+option java_outer_classname = "LogsServiceProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).work_in_progress = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: OpenTelemetry (gRPC) Access Log]
+
+// Configuration for the built-in *envoy.access_loggers.open_telemetry*
+// :ref:`AccessLog <envoy_api_msg_config.accesslog.v3.AccessLog>`. This configuration will
+// populate `opentelemetry.proto.collector.v1.logs.ExportLogsServiceRequest.resource_logs <https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/collector/logs/v1/logs_service.proto>`_.
+// OpenTelemetry `Resource <https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/resource/v1/resource.proto>`_
+// attributes are filled with Envoy node info. In addition, the request start time is set in the
+// dedicated field.
+// [#extension: envoy.access_loggers.open_telemetry]
+// [#comment:TODO(itamarkam): allow configuration for resource attributes.]
+message OpenTelemetryAccessLogConfig {
+  // [#comment:TODO(itamarkam): add 'filter_state_objects_to_log' to logs.]
+  grpc.v3.CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}];
+
+  // OpenTelemetry `LogResource <https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/logs/v1/logs.proto>`_
+  // fields, following `Envoy access logging formatting <https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage>`_.
+  //
+  // See 'body' in the LogResource proto for more details.
+  // Example: ``body { string_value: "%PROTOCOL%" }``.
+  opentelemetry.proto.common.v1.AnyValue body = 2;
+
+  // See 'attributes' in the LogResource proto for more details.
+  // Example: ``attributes { values { key: "user_agent" value { string_value: "%REQ(USER-AGENT)%" } } }``.
+  opentelemetry.proto.common.v1.KeyValueList attributes = 3;
+}

api/versioning/BUILD

@@ -38,6 +38,7 @@ proto_library(
         "//envoy/data/tap/v3:pkg",
         "//envoy/extensions/access_loggers/file/v3:pkg",
         "//envoy/extensions/access_loggers/grpc/v3:pkg",
+        "//envoy/extensions/access_loggers/open_telemetry/v3alpha:pkg",
         "//envoy/extensions/access_loggers/stderror/v3:pkg",
         "//envoy/extensions/access_loggers/stdoutput/v3:pkg",
         "//envoy/extensions/access_loggers/wasm/v3:pkg",

docs/root/api-v3/config/accesslog/accesslog.rst

@@ -9,3 +9,4 @@ Access loggers
 
   v3/*
   ../../extensions/access_loggers/*/v3/*
+  ../../extensions/access_loggers/*/v3alpha/*

docs/root/intro/arch_overview/observability/access_logging.rst

@@ -73,5 +73,6 @@ Further reading
 * File :ref:`access log sink <envoy_v3_api_msg_extensions.access_loggers.file.v3.FileAccessLog>`.
 * gRPC :ref:`Access Log Service (ALS) <envoy_v3_api_msg_extensions.access_loggers.grpc.v3.HttpGrpcAccessLogConfig>`
   sink.
+* OpenTelemetry (gRPC) :ref:`LogsService <envoy_v3_api_msg_extensions.access_loggers.open_telemetry.v3alpha.OpenTelemetryAccessLogConfig>`
 * Stdoutput :ref:`access log sink <envoy_v3_api_msg_extensions.access_loggers.stdoutput.v3.StdoutputAccessLog>`
 * Stderror :ref:`access log sink <envoy_v3_api_msg_extensions.access_loggers.stderror.v3.StderrorAccessLog>`

docs/root/version_history/current.rst

@@ -111,6 +111,8 @@ Removed Config or Runtime
 
 New Features
 ------------
+
+* access log: added a new :ref:`OpenTelemetry access logger <envoy_v3_api_msg_extensions.access_loggers.open_telemetry.v3alpha.OpenTelemetryAccessLogConfig>` extension, allowing a flexible log structure with native Envoy access log formatting.
 * access log: added the new response flag `NC` for upstream cluster not found. The error flag is set when the http or tcp route is found for the request but the cluster is not available.
 * access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
 * access log: added support for cross platform writing to :ref:`standard output <envoy_v3_api_msg_extensions.access_loggers.stdoutput.v3.StdoutputAccessLog>` and :ref:`standard error <envoy_v3_api_msg_extensions.access_loggers.stderror.v3.StderrorAccessLog>`.

generated_api_shadow/BUILD

@@ -155,6 +155,7 @@ proto_library(
         "//envoy/data/tap/v3:pkg",
         "//envoy/extensions/access_loggers/file/v3:pkg",
         "//envoy/extensions/access_loggers/grpc/v3:pkg",
+        "//envoy/extensions/access_loggers/open_telemetry/v3alpha:pkg",
         "//envoy/extensions/access_loggers/stderror/v3:pkg",
         "//envoy/extensions/access_loggers/stdoutput/v3:pkg",
         "//envoy/extensions/access_loggers/wasm/v3:pkg",