formatter: print request header without query string

CODEOWNERS

@@ -179,3 +179,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/filters/common/ext_authz @esmet @gsagula @dio
 /*/extensions/filters/http/ext_authz @esmet @gsagula @dio
 /*/extensions/filters/network/ext_authz @esmet @gsagula @dio
+# Formatters
+/*/extensions/formatter/req_without_query @dio @tsaarni

api/BUILD

@@ -242,6 +242,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",

api/envoy/config/core/v3/substitution_format_string.proto

@@ -107,5 +107,6 @@ message SubstitutionFormatString {
 
   // Specifies a collection of Formatter plugins that can be called from the access log configuration.
   // See the formatters extensions documentation for details.
+  // [#extension-category: envoy.formatter]
   repeated TypedExtensionConfig formatters = 6;
 }

api/envoy/extensions/formatter/req_without_query/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/formatter/req_without_query/v3/req_without_query.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package envoy.extensions.formatter.req_without_query.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.formatter.req_without_query.v3";
+option java_outer_classname = "ReqWithoutQueryProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Formatter extension for printing request without query string]
+// [#extension: envoy.formatter.req_without_query]
+
+// ReqWithoutQuery formatter extension implements REQ_WITHOUT_QUERY command operator that
+// works the same way as :ref:`REQ <config_access_log_format_req>` except that it will
+// remove the query string. It is used to avoid logging any sensitive information into
+// the access log.
+
+// %REQ_WITHOUT_QUERY(X?Y):Z%
+//   An HTTP request header where X is the main HTTP header, Y is the alternative one, and Z is an
+//   optional parameter denoting string truncation up to Z characters long. The value is taken from
+//   the HTTP request header named X first and if it's not set, then request header Y is used. If
+//   none of the headers are present '-' symbol will be in the log.
+
+// [#not-implemented-hide:]
+// Placeholder for future configuration for the ReqWithoutQuery formatter.
+message ReqWithoutQuery {
+}

api/versioning/BUILD

@@ -125,6 +125,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",

bazel/envoy_library.bzl

@@ -77,6 +77,7 @@ EXTENSION_CATEGORIES = [
     "envoy.filters.listener",
     "envoy.filters.network",
     "envoy.filters.udp_listener",
+    "envoy.formatter",
     "envoy.grpc_credentials",
     "envoy.guarddog_actions",
     "envoy.health_checkers",

docs/root/configuration/observability/access_log/usage.rst

@@ -407,6 +407,8 @@ The following command operators are supported:
 %DOWNSTREAM_LOCAL_PORT%
     Similar to **%DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%**, but only extracts the port portion of the **%DOWNSTREAM_LOCAL_ADDRESS%**
 
+.. _config_access_log_format_req:
+
 %REQ(X?Y):Z%
   HTTP
     An HTTP request header where X is the main HTTP header, Y is the alternative one, and Z is an

generated_api_shadow/BUILD

@@ -242,6 +242,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg",
         "//envoy/extensions/internal_redirect/previous_routes/v3:pkg",

source/extensions/extensions_build_config.bzl

@@ -255,6 +255,12 @@ EXTENSIONS = {
     #
 
     "envoy.tls.cert_validator.spiffe":                  "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config",
+
+    #
+    # Formatter
+    #
+
+    "envoy.formatter.req_without_query":                "//source/extensions/formatter/req_without_query:config",
 }
 
 # These can be changed to ["//visibility:public"], for  downstream builds which

source/extensions/formatter/req_without_query/BUILD

@@ -0,0 +1,37 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+# Access log formatter that strips query string from request path
+# Public docs: docs/root/TODO(tsaarni)
+
+envoy_cc_library(
+    name = "req_without_query_lib",
+    srcs = ["req_without_query.cc"],
+    hdrs = ["req_without_query.h"],
+    deps = [
+        "//source/common/formatter:substitution_formatter_lib",
+        "//source/common/protobuf:utility_lib",
+    ],
+)
+
+envoy_cc_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    category = "envoy.formatter",
+    security_posture = "unknown",
+    status = "alpha",
+    deps = [
+        "//include/envoy/registry",
+        "//source/extensions/formatter/req_without_query:req_without_query_lib",
+        "@envoy_api//envoy/extensions/formatter/req_without_query/v3:pkg_cc_proto",
+    ],
+)

source/extensions/formatter/req_without_query/config.cc

@@ -0,0 +1,26 @@
+#include "extensions/formatter/req_without_query/config.h"
+
+#include "envoy/extensions/formatter/req_without_query/v3/req_without_query.pb.h"
+
+#include "extensions/formatter/req_without_query/req_without_query.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+::Envoy::Formatter::CommandParserPtr
+ReqWithoutQueryFactory::createCommandParserFromProto(const Protobuf::Message&) {
+  return std::make_unique<ReqWithoutQueryCommandParser>();
+}
+
+ProtobufTypes::MessagePtr ReqWithoutQueryFactory::createEmptyConfigProto() {
+  return std::make_unique<envoy::extensions::formatter::req_without_query::v3::ReqWithoutQuery>();
+}
+
+std::string ReqWithoutQueryFactory::name() const { return "envoy.formatter.req_without_query"; }
+
+REGISTER_FACTORY(ReqWithoutQueryFactory, ReqWithoutQueryFactory::CommandParserFactory);
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy

source/extensions/formatter/req_without_query/config.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "common/formatter/substitution_formatter.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+class ReqWithoutQueryFactory : public ::Envoy::Formatter::CommandParserFactory {
+public:
+  ::Envoy::Formatter::CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message&) override;
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override;
+  std::string name() const override;
+};
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy

source/extensions/formatter/req_without_query/req_without_query.cc

@@ -0,0 +1,90 @@
+#include "extensions/formatter/req_without_query/req_without_query.h"
+
+#include <string>
+
+#include "common/protobuf/utility.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+void stripQueryString(std::string& path) {
+  const size_t query_pos = path.find('?');
+  path = std::string(path.data(), query_pos != path.npos ? query_pos : path.size());
+}
+
+void truncate(std::string& str, absl::optional<uint32_t> max_length) {
+  if (!max_length) {
+    return;
+  }
+
+  str = str.substr(0, max_length.value());
+}
+
+ReqWithoutQuery::ReqWithoutQuery(const std::string& main_header,
+                                 const std::string& alternative_header,
+                                 absl::optional<size_t> max_length)
+    : main_header_(main_header), alternative_header_(alternative_header), max_length_(max_length) {}
+
+absl::optional<std::string> ReqWithoutQuery::format(const Http::RequestHeaderMap& request,
+                                                    const Http::ResponseHeaderMap&,
+                                                    const Http::ResponseTrailerMap&,
+                                                    const StreamInfo::StreamInfo&,
+                                                    absl::string_view) const {
+  const Http::HeaderEntry* header = findHeader(request);
+  if (!header) {
+    return absl::nullopt;
+  }
+
+  std::string val = std::string(header->value().getStringView());
+  stripQueryString(val);
+  truncate(val, max_length_);
+
+  return val;
+}
+
+ProtobufWkt::Value ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request,
+                                                const Http::ResponseHeaderMap&,
+                                                const Http::ResponseTrailerMap&,
+                                                const StreamInfo::StreamInfo&,
+                                                absl::string_view) const {
+  const Http::HeaderEntry* header = findHeader(request);
+  if (!header) {
+    return ValueUtil::nullValue();
+  }
+
+  std::string val = std::string(header->value().getStringView());
+  stripQueryString(val);
+  truncate(val, max_length_);
+  return ValueUtil::stringValue(val);
+}
+
+const Http::HeaderEntry* ReqWithoutQuery::findHeader(const Http::HeaderMap& headers) const {
+  const auto header = headers.get(main_header_);
+
+  if (header.empty() && !alternative_header_.get().empty()) {
+    const auto alternate_header = headers.get(alternative_header_);
+    // TODO(https://github.com/envoyproxy/envoy/issues/13454): Potentially log all header values.
+    return alternate_header.empty() ? nullptr : alternate_header[0];
+  }
+
+  return header.empty() ? nullptr : header[0];
+}
+
+::Envoy::Formatter::FormatterProviderPtr
+ReqWithoutQueryCommandParser::parse(const std::string& token, size_t, size_t) const {
+  if (absl::StartsWith(token, "REQ_WITHOUT_QUERY(")) {
+    std::string main_header, alternative_header;
+    absl::optional<size_t> max_length;
+
+    Envoy::Formatter::SubstitutionFormatParser::parseCommandHeader(
+        token, ReqWithoutQueryParamStart, main_header, alternative_header, max_length);
+    return std::make_unique<ReqWithoutQuery>(main_header, alternative_header, max_length);
+  }
+
+  return nullptr;
+}
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy

source/extensions/formatter/req_without_query/req_without_query.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/config/typed_config.h"
+#include "envoy/registry/registry.h"
+
+#include "common/formatter/substitution_formatter.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+class ReqWithoutQuery : public ::Envoy::Formatter::FormatterProvider {
+public:
+  ReqWithoutQuery(const std::string& main_header, const std::string& alternative_header,
+                  absl::optional<size_t> max_length);
+
+  absl::optional<std::string> format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&,
+                                     const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&,
+                                     absl::string_view) const override;
+  ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&,
+                                 const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&,
+                                 absl::string_view) const override;
+
+private:
+  const Http::HeaderEntry* findHeader(const Http::HeaderMap& headers) const;
+
+  Http::LowerCaseString main_header_;
+  Http::LowerCaseString alternative_header_;
+  absl::optional<size_t> max_length_;
+};
+
+class ReqWithoutQueryCommandParser : public ::Envoy::Formatter::CommandParser {
+public:
+  ReqWithoutQueryCommandParser() = default;
+  ::Envoy::Formatter::FormatterProviderPtr parse(const std::string& token, size_t,
+                                                 size_t) const override;
+
+private:
+  static const size_t ReqWithoutQueryParamStart{sizeof("REQ_WITHOUT_QUERY(") - 1};
+};
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy