Implement omit_empty_values for access log formatters

api/envoy/config/core/v3/substitution_format_string.proto

@@ -58,4 +58,11 @@ message SubstitutionFormatString {
     //
     google.protobuf.Struct json_format = 2 [(validate.rules).message = {required: true}];
   }
+
+  // If set to true, when command operators are evaluated to null,
+  //
+  // * for ``text_format``, the output of the empty operator is changed from ``-`` to an
+  //   empty string, so that empty values are omitted entirely.
+  // * for ``json_format`` the keys with null values are omitted in the output structure.
+  bool omit_empty_values = 3;
 }

docs/root/configuration/observability/access_log/usage.rst

@@ -116,7 +116,9 @@ are noted.
 
 Note that if a value is not set/empty, the logs will contain a ``-`` character or, for JSON logs,
 the string ``"-"``. For typed JSON logs unset values are represented as ``null`` values and empty
-strings are rendered as ``""``.
+strings are rendered as ``""``. :ref:`omit_empty_values
+<envoy_v3_api_field_config.core.v3.SubstitutionFormatString.omit_empty_values>` option could be used
+to omit empty values entirely.
 
 Unless otherwise noted, command operators produce string outputs for typed JSON logs.
 

docs/root/version_history/current.rst

@@ -61,6 +61,7 @@ New Features
 * access log: added a :ref:`dynamic metadata filter<envoy_v3_api_msg_config.accesslog.v3.MetadataFilter>` for access logs, which filters whether to log based on matching dynamic metadata.
 * access log: added support for :ref:`%DOWNSTREAM_PEER_FINGERPRINT_1% <config_access_log_format_response_flags>` as a response flag.
 * access log: added support for nested objects in :ref:`JSON logging mode <config_access_log_format_dictionaries>`.
+* access log: added :ref:`omit_empty_values<envoy_v3_api_field_config.core.v3.SubstitutionFormatString.omit_empty_values>` option to omit unset value from formatted log.
 * build: enable building envoy :ref:`arm64 images <arm_binaries>` by buildx tool in x86 CI platform.
 * dns_filter: added support for answering :ref:`service record<envoy_v3_api_msg_data.dns.v3.DnsTable.DnsService>` queries.
 * dynamic_forward_proxy: added :ref:`use_tcp_for_dns_lookups<envoy_v3_api_field_extensions.common.dynamic_forward_proxy.v3.DnsCacheConfig.use_tcp_for_dns_lookups>` option to use TCP for DNS lookups in order to match the DNS options for :ref:`Clusters<envoy_v3_api_msg_config.cluster.v3.Cluster>`.

include/envoy/formatter/substitution_formatter.h

@@ -51,13 +51,14 @@ class FormatterProvider {
    * @param response_trailers supplies the response trailers.
    * @param stream_info supplies the stream info.
    * @param local_reply_body supplies the local reply body.
-   * @return std::string containing a single value extracted from the given headers/trailers/stream.
+   * @return absl::optional<std::string> optional string containing a single value extracted from
+   * the given headers/trailers/stream.
    */
-  virtual std::string format(const Http::RequestHeaderMap& request_headers,
-                             const Http::ResponseHeaderMap& response_headers,
-                             const Http::ResponseTrailerMap& response_trailers,
-                             const StreamInfo::StreamInfo& stream_info,
-                             absl::string_view local_reply_body) const PURE;
+  virtual absl::optional<std::string> format(const Http::RequestHeaderMap& request_headers,
+                                             const Http::ResponseHeaderMap& response_headers,
+                                             const Http::ResponseTrailerMap& response_trailers,
+                                             const StreamInfo::StreamInfo& stream_info,
+                                             absl::string_view local_reply_body) const PURE;
   /**
    * Extract a value from the provided headers/trailers/stream, preserving the value's type.
    * @param request_headers supplies the request headers.

source/common/formatter/substitution_format_string.cc

@@ -7,17 +7,17 @@ namespace Formatter {
 
 FormatterPtr
 SubstitutionFormatStringUtils::createJsonFormatter(const ProtobufWkt::Struct& struct_format,
-                                                   bool preserve_types) {
-  return std::make_unique<JsonFormatterImpl>(struct_format, preserve_types);
+                                                   bool preserve_types, bool omit_empty_values) {
+  return std::make_unique<JsonFormatterImpl>(struct_format, preserve_types, omit_empty_values);
 }
 
 FormatterPtr SubstitutionFormatStringUtils::fromProtoConfig(
     const envoy::config::core::v3::SubstitutionFormatString& config) {
   switch (config.format_case()) {
   case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat:
-    return std::make_unique<FormatterImpl>(config.text_format());
+    return std::make_unique<FormatterImpl>(config.text_format(), config.omit_empty_values());
   case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: {
-    return createJsonFormatter(config.json_format(), true);
+    return createJsonFormatter(config.json_format(), true, config.omit_empty_values());
   }
   default:
     NOT_REACHED_GCOVR_EXCL_LINE;

source/common/formatter/substitution_format_string.h

@@ -25,7 +25,7 @@ class SubstitutionFormatStringUtils {
    * Generate a Json formatter object from proto::Struct config
    */
   static FormatterPtr createJsonFormatter(const ProtobufWkt::Struct& struct_format,
-                                          bool preserve_types);
+                                          bool preserve_types, bool omit_empty_values);
 };
 
 } // namespace Formatter