diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md index fff64a494971..b8f509b2194c 100644 --- a/REPO_LAYOUT.md +++ b/REPO_LAYOUT.md @@ -89,6 +89,8 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE `Envoy::Extensions::ListenerFilters` namespace. * [filters/network/](/source/extensions/filters/network): L4 network filters which use the `Envoy::Extensions::NetworkFilters` namespace. + * [formatters](/source/extensions/formatters): Access log formatters which use the + `Envoy::Extensions::Formatters` namespace. * [grpc_credentials/](/source/extensions/grpc_credentials): Custom gRPC credentials which use the `Envoy::Extensions::GrpcCredentials` namespace. * [health_checker/](/source/extensions/health_checker): Custom health checkers which use the diff --git a/api/envoy/config/core/v3/substitution_format_string.proto b/api/envoy/config/core/v3/substitution_format_string.proto index fa14db45ddda..9802309df60d 100644 --- a/api/envoy/config/core/v3/substitution_format_string.proto +++ b/api/envoy/config/core/v3/substitution_format_string.proto @@ -3,7 +3,9 @@ syntax = "proto3"; package envoy.config.core.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/extension.proto"; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -18,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Configuration to use multiple :ref:`command operators ` // to generate a new string in either plain text or JSON format. -// [#next-free-field: 6] +// [#next-free-field: 7] message SubstitutionFormatString { oneof format { option (validate.required) = true; @@ -103,4 +105,8 @@ message SubstitutionFormatString { // content_type: "text/html; charset=UTF-8" // string content_type = 4; + + // Specifies a collection of Formatter plugins that can be called from the access log configuration. + // See the formatters extensions documentation for details. + repeated TypedExtensionConfig formatters = 6; } diff --git a/api/envoy/config/core/v4alpha/substitution_format_string.proto b/api/envoy/config/core/v4alpha/substitution_format_string.proto index 383d5dfd5e6d..abd8089362c6 100644 --- a/api/envoy/config/core/v4alpha/substitution_format_string.proto +++ b/api/envoy/config/core/v4alpha/substitution_format_string.proto @@ -3,7 +3,9 @@ syntax = "proto3"; package envoy.config.core.v4alpha; import "envoy/config/core/v4alpha/base.proto"; +import "envoy/config/core/v4alpha/extension.proto"; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -19,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // Configuration to use multiple :ref:`command operators ` // to generate a new string in either plain text or JSON format. -// [#next-free-field: 6] +// [#next-free-field: 7] message SubstitutionFormatString { option (udpa.annotations.versioning).previous_message_type = "envoy.config.core.v3.SubstitutionFormatString"; @@ -92,4 +94,8 @@ message SubstitutionFormatString { // content_type: "text/html; charset=UTF-8" // string content_type = 4; + + // Specifies a collection of Formatter plugins that can be called from the access log configuration. + // See the formatters extensions documentation for details. + repeated TypedExtensionConfig formatters = 6; } diff --git a/docs/root/extending/extending.rst b/docs/root/extending/extending.rst index 7f2e676e20eb..aaf83ce7b516 100644 --- a/docs/root/extending/extending.rst +++ b/docs/root/extending/extending.rst @@ -26,6 +26,7 @@ types including: * :ref:`Compression libraries ` * :ref:`Bootstrap extensions ` * :ref:`Fatal actions ` +* :ref:`Formatters ` As of this writing there is no high level extension developer documentation. The :repo:`existing extensions ` are a good way to learn what is possible. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 3ef7e0a8cd04..f487da21458e 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -19,6 +19,7 @@ Removed Config or Runtime New Features ------------ +* access log: added the :ref:`formatters ` extension point for custom formatters (command operators). * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. Deprecated diff --git a/generated_api_shadow/envoy/config/core/v3/substitution_format_string.proto b/generated_api_shadow/envoy/config/core/v3/substitution_format_string.proto index fa14db45ddda..9802309df60d 100644 --- a/generated_api_shadow/envoy/config/core/v3/substitution_format_string.proto +++ b/generated_api_shadow/envoy/config/core/v3/substitution_format_string.proto @@ -3,7 +3,9 @@ syntax = "proto3"; package envoy.config.core.v3; import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/extension.proto"; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -18,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Configuration to use multiple :ref:`command operators ` // to generate a new string in either plain text or JSON format. -// [#next-free-field: 6] +// [#next-free-field: 7] message SubstitutionFormatString { oneof format { option (validate.required) = true; @@ -103,4 +105,8 @@ message SubstitutionFormatString { // content_type: "text/html; charset=UTF-8" // string content_type = 4; + + // Specifies a collection of Formatter plugins that can be called from the access log configuration. + // See the formatters extensions documentation for details. + repeated TypedExtensionConfig formatters = 6; } diff --git a/generated_api_shadow/envoy/config/core/v4alpha/substitution_format_string.proto b/generated_api_shadow/envoy/config/core/v4alpha/substitution_format_string.proto index 8fc44f05a9c3..922fd6b819a0 100644 --- a/generated_api_shadow/envoy/config/core/v4alpha/substitution_format_string.proto +++ b/generated_api_shadow/envoy/config/core/v4alpha/substitution_format_string.proto @@ -3,7 +3,9 @@ syntax = "proto3"; package envoy.config.core.v4alpha; import "envoy/config/core/v4alpha/base.proto"; +import "envoy/config/core/v4alpha/extension.proto"; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "udpa/annotations/status.proto"; @@ -19,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // Configuration to use multiple :ref:`command operators ` // to generate a new string in either plain text or JSON format. -// [#next-free-field: 6] +// [#next-free-field: 7] message SubstitutionFormatString { option (udpa.annotations.versioning).previous_message_type = "envoy.config.core.v3.SubstitutionFormatString"; @@ -107,4 +109,8 @@ message SubstitutionFormatString { // content_type: "text/html; charset=UTF-8" // string content_type = 4; + + // Specifies a collection of Formatter plugins that can be called from the access log configuration. + // See the formatters extensions documentation for details. + repeated TypedExtensionConfig formatters = 6; } diff --git a/include/envoy/formatter/substitution_formatter.h b/include/envoy/formatter/substitution_formatter.h index 0d78ce9aed0e..48541a0a98c2 100644 --- a/include/envoy/formatter/substitution_formatter.h +++ b/include/envoy/formatter/substitution_formatter.h @@ -4,6 +4,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/config/typed_config.h" #include "envoy/http/header_map.h" #include "envoy/stream_info/stream_info.h" @@ -78,5 +79,56 @@ class FormatterProvider { using FormatterProviderPtr = std::unique_ptr; +/** + * Interface for command parser. + * CommandParser returns a FormatterProviderPtr after successfully parsing an access log format + * token, nullptr otherwise. + */ +class CommandParser { +public: + virtual ~CommandParser() = default; + + /** + * Return a FormatterProviderPtr if this command is parsed from the token. + * @param token the token to parse + * @param pos current position in the entire format string + * @param command_end_position position at the end of the command token + * + * Given the following format line using an extension called %CMD()%: + * + * %CMD()% %START_TIME(%Y/%m/%d)% ... + * + * The call to parse() for that extension would look like this: + * + * parse("CMD()", 1, 5) + * + * @return FormattterProviderPtr substitution provider for the parsed command + */ + virtual FormatterProviderPtr parse(const std::string& token, size_t pos, + size_t command_end_position) const PURE; +}; + +using CommandParserPtr = std::unique_ptr; + +/** + * Implemented by each custom CommandParser and registered via Registry::registerFactory() + * or the convenience class RegisterFactory. + */ +class CommandParserFactory : public Config::TypedFactory { +public: + ~CommandParserFactory() override = default; + + /** + * Creates a particular CommandParser implementation. + * + * @param config supplies the configuration for the command parser. + * @return CommandParserPtr the CommandParser which will be used in + * SubstitutionFormatParser::parse() when evaluating an access log format string. + */ + virtual CommandParserPtr createCommandParserFromProto(const Protobuf::Message& config) PURE; + + std::string category() const override { return "envoy.formatter"; } +}; + } // namespace Formatter } // namespace Envoy diff --git a/source/common/formatter/BUILD b/source/common/formatter/BUILD index 5747f9d1e7f9..7b36bb0b7b28 100644 --- a/source/common/formatter/BUILD +++ b/source/common/formatter/BUILD @@ -35,6 +35,7 @@ envoy_cc_library( hdrs = ["substitution_format_string.h"], deps = [ ":substitution_formatter_lib", + "//source/common/config:utility_lib", "//source/common/protobuf", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/source/common/formatter/substitution_format_string.cc b/source/common/formatter/substitution_format_string.cc index 372b9fc6a612..bddaac039a7f 100644 --- a/source/common/formatter/substitution_format_string.cc +++ b/source/common/formatter/substitution_format_string.cc @@ -3,6 +3,7 @@ #include "envoy/api/api.h" #include "common/config/datasource.h" +#include "common/config/utility.h" #include "common/formatter/substitution_formatter.h" namespace Envoy { @@ -16,14 +17,26 @@ SubstitutionFormatStringUtils::createJsonFormatter(const ProtobufWkt::Struct& st FormatterPtr SubstitutionFormatStringUtils::fromProtoConfig( const envoy::config::core::v3::SubstitutionFormatString& config, Api::Api& api) { + // Instantiate formatter extensions. + std::vector commands; + for (const auto& formatter : config.formatters()) { + auto* factory = Envoy::Config::Utility::getFactory(formatter); + if (!factory) { + throw EnvoyException(absl::StrCat("Formatter not found: ", formatter.name())); + } + auto parser = factory->createCommandParserFromProto(formatter.typed_config()); + commands.push_back(std::move(parser)); + } + switch (config.format_case()) { case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormat: - return std::make_unique(config.text_format(), config.omit_empty_values()); + return std::make_unique(config.text_format(), config.omit_empty_values(), + commands); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kJsonFormat: { return createJsonFormatter(config.json_format(), true, config.omit_empty_values()); case envoy::config::core::v3::SubstitutionFormatString::FormatCase::kTextFormatSource: return std::make_unique( - Config::DataSource::read(config.text_format_source(), true, api)); + Config::DataSource::read(config.text_format_source(), true, api), false, commands); } default: NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index 1072ae132f6f..d7fca3cf7a49 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -113,6 +113,12 @@ FormatterImpl::FormatterImpl(const std::string& format, bool omit_empty_values) providers_ = SubstitutionFormatParser::parse(format); } +FormatterImpl::FormatterImpl(const std::string& format, bool omit_empty_values, + const std::vector& command_parsers) + : empty_value_string_(omit_empty_values ? EMPTY_STRING : DefaultUnspecifiedValueString) { + providers_ = SubstitutionFormatParser::parse(format, command_parsers); +} + std::string FormatterImpl::format(const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, const Http::ResponseTrailerMap& response_trailers, @@ -282,109 +288,134 @@ void SubstitutionFormatParser::parseCommand(const std::string& token, const size } } -// TODO(derekargueta): #2967 - Rewrite SubstitutionFormatter with parser library & formal grammar std::vector SubstitutionFormatParser::parse(const std::string& format) { - std::string current_token; - std::vector formatters; + return SubstitutionFormatParser::parse(format, {}); +} + +FormatterProviderPtr SubstitutionFormatParser::parseBuiltinCommand(const std::string& token) { static constexpr absl::string_view DYNAMIC_META_TOKEN{"DYNAMIC_METADATA("}; static constexpr absl::string_view FILTER_STATE_TOKEN{"FILTER_STATE("}; - const std::regex command_w_args_regex(R"EOF(^%([A-Z]|_)+(\([^\)]*\))?(:[0-9]+)?(%))EOF"); - static constexpr absl::string_view PLAIN_SERIALIZATION{"PLAIN"}; static constexpr absl::string_view TYPED_SERIALIZATION{"TYPED"}; + if (absl::StartsWith(token, "REQ(")) { + std::string main_header, alternative_header; + absl::optional max_length; + + parseCommandHeader(token, ReqParamStart, main_header, alternative_header, max_length); + + return std::make_unique(main_header, alternative_header, max_length); + } else if (absl::StartsWith(token, "RESP(")) { + std::string main_header, alternative_header; + absl::optional max_length; + + parseCommandHeader(token, RespParamStart, main_header, alternative_header, max_length); + + return std::make_unique(main_header, alternative_header, max_length); + } else if (absl::StartsWith(token, "TRAILER(")) { + std::string main_header, alternative_header; + absl::optional max_length; + + parseCommandHeader(token, TrailParamStart, main_header, alternative_header, max_length); + + return std::make_unique(main_header, alternative_header, max_length); + } else if (absl::StartsWith(token, "LOCAL_REPLY_BODY")) { + return std::make_unique(); + } else if (absl::StartsWith(token, DYNAMIC_META_TOKEN)) { + std::string filter_namespace; + absl::optional max_length; + std::vector path; + const size_t start = DYNAMIC_META_TOKEN.size(); + + parseCommand(token, start, ":", filter_namespace, path, max_length); + return std::make_unique(filter_namespace, path, max_length); + } else if (absl::StartsWith(token, FILTER_STATE_TOKEN)) { + std::string key; + absl::optional max_length; + std::vector path; + const size_t start = FILTER_STATE_TOKEN.size(); + + parseCommand(token, start, ":", key, path, max_length); + if (key.empty()) { + throw EnvoyException("Invalid filter state configuration, key cannot be empty."); + } + + const absl::string_view serialize_type = + !path.empty() ? path[path.size() - 1] : TYPED_SERIALIZATION; + + if (serialize_type != PLAIN_SERIALIZATION && serialize_type != TYPED_SERIALIZATION) { + throw EnvoyException("Invalid filter state serialize type, only support PLAIN/TYPED."); + } + const bool serialize_as_string = serialize_type == PLAIN_SERIALIZATION; + + return std::make_unique(key, max_length, serialize_as_string); + } else if (absl::StartsWith(token, "START_TIME")) { + return std::make_unique(token); + } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { + return std::make_unique(token); + } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { + return std::make_unique(token); + } else if (absl::StartsWith(token, "GRPC_STATUS")) { + return std::make_unique("grpc-status", "", absl::optional()); + } + + return nullptr; +} + +// TODO(derekargueta): #2967 - Rewrite SubstitutionFormatter with parser library & formal grammar +std::vector +SubstitutionFormatParser::parse(const std::string& format, + const std::vector& commands) { + std::string current_token; + std::vector formatters; + const std::regex command_w_args_regex(R"EOF(^%([A-Z]|_)+(\([^\)]*\))?(:[0-9]+)?(%))EOF"); + for (size_t pos = 0; pos < format.length(); ++pos) { - if (format[pos] == '%') { - if (!current_token.empty()) { - formatters.emplace_back(FormatterProviderPtr{new PlainStringFormatter(current_token)}); - current_token = ""; - } + if (format[pos] != '%') { + current_token += format[pos]; + continue; + } - std::smatch m; - const std::string search_space = format.substr(pos); - if (!std::regex_search(search_space, m, command_w_args_regex)) { - throw EnvoyException( - fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}", - format, pos)); - } + if (!current_token.empty()) { + formatters.emplace_back(FormatterProviderPtr{new PlainStringFormatter(current_token)}); + current_token = ""; + } - const std::string match = m.str(0); - const std::string token = match.substr(1, match.length() - 2); - pos += 1; - const int command_end_position = pos + token.length(); - - if (absl::StartsWith(token, "REQ(")) { - std::string main_header, alternative_header; - absl::optional max_length; - - parseCommandHeader(token, ReqParamStart, main_header, alternative_header, max_length); - - formatters.emplace_back(FormatterProviderPtr{ - new RequestHeaderFormatter(main_header, alternative_header, max_length)}); - } else if (absl::StartsWith(token, "RESP(")) { - std::string main_header, alternative_header; - absl::optional max_length; - - parseCommandHeader(token, RespParamStart, main_header, alternative_header, max_length); - - formatters.emplace_back(FormatterProviderPtr{ - new ResponseHeaderFormatter(main_header, alternative_header, max_length)}); - } else if (absl::StartsWith(token, "TRAILER(")) { - std::string main_header, alternative_header; - absl::optional max_length; - - parseCommandHeader(token, TrailParamStart, main_header, alternative_header, max_length); - - formatters.emplace_back(FormatterProviderPtr{ - new ResponseTrailerFormatter(main_header, alternative_header, max_length)}); - } else if (absl::StartsWith(token, "LOCAL_REPLY_BODY")) { - formatters.emplace_back(std::make_unique()); - } else if (absl::StartsWith(token, DYNAMIC_META_TOKEN)) { - std::string filter_namespace; - absl::optional max_length; - std::vector path; - const size_t start = DYNAMIC_META_TOKEN.size(); - - parseCommand(token, start, ":", filter_namespace, path, max_length); - formatters.emplace_back( - FormatterProviderPtr{new DynamicMetadataFormatter(filter_namespace, path, max_length)}); - } else if (absl::StartsWith(token, FILTER_STATE_TOKEN)) { - std::string key; - absl::optional max_length; - std::vector path; - const size_t start = FILTER_STATE_TOKEN.size(); - - parseCommand(token, start, ":", key, path, max_length); - if (key.empty()) { - throw EnvoyException("Invalid filter state configuration, key cannot be empty."); - } + std::smatch m; + const std::string search_space = format.substr(pos); + if (!std::regex_search(search_space, m, command_w_args_regex)) { + throw EnvoyException(fmt::format( + "Incorrect configuration: {}. Couldn't find valid command at position {}", format, pos)); + } - const absl::string_view serialize_type = - !path.empty() ? path[path.size() - 1] : TYPED_SERIALIZATION; + const std::string match = m.str(0); + const std::string token = match.substr(1, match.length() - 2); + pos += 1; + const size_t command_end_position = pos + token.length(); - if (serialize_type != PLAIN_SERIALIZATION && serialize_type != TYPED_SERIALIZATION) { - throw EnvoyException("Invalid filter state serialize type, only support PLAIN/TYPED."); + auto formatter = parseBuiltinCommand(token); + if (formatter) { + formatters.push_back(std::move(formatter)); + } else { + // Check formatter extensions. These are used for anything not provided by the built-in + // operators, e.g.: specialized formatting, computing stats from request/response headers + // or from stream info, etc. + bool added = false; + for (const auto& cmd : commands) { + auto formatter = cmd->parse(token, pos, command_end_position); + if (formatter) { + formatters.push_back(std::move(formatter)); + added = true; + break; } - const bool serialize_as_string = serialize_type == PLAIN_SERIALIZATION; - - formatters.push_back( - std::make_unique(key, max_length, serialize_as_string)); - } else if (absl::StartsWith(token, "START_TIME")) { - formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(token)}); - } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_START")) { - formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVStartFormatter(token)}); - } else if (absl::StartsWith(token, "DOWNSTREAM_PEER_CERT_V_END")) { - formatters.emplace_back(FormatterProviderPtr{new DownstreamPeerCertVEndFormatter(token)}); - } else if (absl::StartsWith(token, "GRPC_STATUS")) { - formatters.emplace_back(FormatterProviderPtr{ - new GrpcStatusFormatter("grpc-status", "", absl::optional())}); - } else { + } + + if (!added) { formatters.emplace_back(FormatterProviderPtr{new StreamInfoFormatter(token)}); } - pos = command_end_position; - } else { - current_token += format[pos]; } + + pos = command_end_position; } if (!current_token.empty()) { diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index dc0d39689e9b..2d1f121c4872 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -23,8 +23,9 @@ namespace Formatter { class SubstitutionFormatParser { public: static std::vector parse(const std::string& format); + static std::vector + parse(const std::string& format, const std::vector& command_parsers); -private: /** * Parse a header format rule of the form: %REQ(X?Y):Z% . * Will populate a main_header and an optional alternative header if specified. @@ -60,6 +61,19 @@ class SubstitutionFormatParser { const std::string& separator, std::string& main, std::vector& sub_items, absl::optional& max_length); + /** + * Return a FormatterProviderPtr if a built-in command is parsed from the token. This method + * handles mapping the command name to an appropriate formatter after parsing. + * + * TODO(rgs1): this can be refactored into a dispatch table using the command name as the key and + * the parsing parameters as the value. + * + * @param token the token to parse + * @return FormattterProviderPtr substitution provider for the parsed command or nullptr + */ + static FormatterProviderPtr parseBuiltinCommand(const std::string& token); + +private: // the indexes of where the parameters for each directive is expected to begin static const size_t ReqParamStart{sizeof("REQ(") - 1}; static const size_t RespParamStart{sizeof("RESP(") - 1}; @@ -93,6 +107,8 @@ class SubstitutionFormatUtils { class FormatterImpl : public Formatter { public: FormatterImpl(const std::string& format, bool omit_empty_values = false); + FormatterImpl(const std::string& format, bool omit_empty_values, + const std::vector& command_parsers); // Formatter::format std::string format(const Http::RequestHeaderMap& request_headers, diff --git a/test/common/formatter/BUILD b/test/common/formatter/BUILD index 9001f4820bdc..b50c253bb24c 100644 --- a/test/common/formatter/BUILD +++ b/test/common/formatter/BUILD @@ -4,6 +4,7 @@ load( "envoy_cc_benchmark_binary", "envoy_cc_fuzz_test", "envoy_cc_test", + "envoy_cc_test_library", "envoy_package", "envoy_proto_library", ) @@ -33,10 +34,21 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_test_library( + name = "command_extension_lib", + srcs = ["command_extension.cc"], + hdrs = ["command_extension.h"], + deps = [ + "//source/common/formatter:substitution_formatter_lib", + "//source/common/protobuf:utility_lib", + ], +) + envoy_cc_test( name = "substitution_formatter_test", srcs = ["substitution_formatter_test.cc"], deps = [ + ":command_extension_lib", "//source/common/common:utility_lib", "//source/common/formatter:substitution_formatter_lib", "//source/common/http:header_map_lib", @@ -57,10 +69,12 @@ envoy_cc_test( name = "substitution_format_string_test", srcs = ["substitution_format_string_test.cc"], deps = [ + ":command_extension_lib", "//source/common/formatter:substitution_format_string_lib", "//test/mocks/http:http_mocks", "//test/mocks/server:factory_context_mocks", "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:registry_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], diff --git a/test/common/formatter/command_extension.cc b/test/common/formatter/command_extension.cc new file mode 100644 index 000000000000..4dd2faa34625 --- /dev/null +++ b/test/common/formatter/command_extension.cc @@ -0,0 +1,45 @@ +#include "test/common/formatter/command_extension.h" + +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Formatter { + +absl::optional TestFormatter::format(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo&, + absl::string_view) const { + return "TestFormatter"; +} + +ProtobufWkt::Value TestFormatter::formatValue(const Http::RequestHeaderMap&, + const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, + const StreamInfo::StreamInfo&, + absl::string_view) const { + return ValueUtil::stringValue(""); +} + +FormatterProviderPtr TestCommandParser::parse(const std::string& token, size_t, size_t) const { + if (absl::StartsWith(token, "COMMAND_EXTENSION")) { + return std::make_unique(); + } + + return nullptr; +} + +CommandParserPtr TestCommandFactory::createCommandParserFromProto(const Protobuf::Message&) { + return std::make_unique(); +} + +std::string TestCommandFactory::configType() { return "google.protobuf.StringValue"; } + +ProtobufTypes::MessagePtr TestCommandFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +std::string TestCommandFactory::name() const { return "envoy.formatter.TestFormatter"; } + +} // namespace Formatter +} // namespace Envoy diff --git a/test/common/formatter/command_extension.h b/test/common/formatter/command_extension.h new file mode 100644 index 000000000000..64e875a90114 --- /dev/null +++ b/test/common/formatter/command_extension.h @@ -0,0 +1,37 @@ +#include + +#include "envoy/config/typed_config.h" +#include "envoy/registry/registry.h" + +#include "common/formatter/substitution_formatter.h" + +namespace Envoy { +namespace Formatter { + +class TestFormatter : public FormatterProvider { +public: + // FormatterProvider + absl::optional format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, + const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo&, + absl::string_view) const override; +}; + +class TestCommandParser : public CommandParser { +public: + TestCommandParser() = default; + FormatterProviderPtr parse(const std::string& token, size_t, size_t) const override; +}; + +class TestCommandFactory : public CommandParserFactory { +public: + CommandParserPtr createCommandParserFromProto(const Protobuf::Message&) override; + std::string configType() override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + std::string name() const override; +}; + +} // namespace Formatter +} // namespace Envoy diff --git a/test/common/formatter/substitution_format_string_test.cc b/test/common/formatter/substitution_format_string_test.cc index 105c18f0038b..3b7bd29abad8 100644 --- a/test/common/formatter/substitution_format_string_test.cc +++ b/test/common/formatter/substitution_format_string_test.cc @@ -2,9 +2,11 @@ #include "common/formatter/substitution_format_string.h" +#include "test/common/formatter/command_extension.h" #include "test/mocks/http/mocks.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/stream_info/mocks.h" +#include "test/test_common/registry.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -97,5 +99,40 @@ TEST_F(SubstitutionFormatStringUtilsTest, TestInvalidConfigs) { } } +TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtension) { + TestCommandFactory factory; + Registry::InjectFactory command_register(factory); + + const std::string yaml = R"EOF( + text_format_source: + inline_string: "plain text %COMMAND_EXTENSION()%" + formatters: + - name: envoy.formatter.TestFormatter + typed_config: + "@type": type.googleapis.com/google.protobuf.StringValue +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + auto formatter = SubstitutionFormatStringUtils::fromProtoConfig(config_, context_.api()); + EXPECT_EQ("plain text TestFormatter", formatter->format(request_headers_, response_headers_, + response_trailers_, stream_info_, body_)); +} + +TEST_F(SubstitutionFormatStringUtilsTest, TestFromProtoConfigFormatterExtensionUnknown) { + const std::string yaml = R"EOF( + text_format_source: + inline_string: "plain text" + formatters: + - name: envoy.formatter.TestFormatterUnknown + typed_config: + "@type": type.googleapis.com/google.protobuf.Any +)EOF"; + TestUtility::loadFromYaml(yaml, config_); + + EXPECT_THROW_WITH_MESSAGE(SubstitutionFormatStringUtils::fromProtoConfig(config_, context_.api()), + EnvoyException, + "Formatter not found: envoy.formatter.TestFormatterUnknown"); +} + } // namespace Formatter } // namespace Envoy diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 0dae4dfdc0c4..87d81342169b 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -14,6 +14,7 @@ #include "common/protobuf/utility.h" #include "common/router/string_accessor_impl.h" +#include "test/common/formatter/command_extension.h" #include "test/mocks/api/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/ssl/mocks.h" @@ -2449,6 +2450,23 @@ TEST(SubstitutionFormatterTest, ParserSuccesses) { } } +TEST(SubstitutionFormatterTest, FormatterExtension) { + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + StreamInfo::MockStreamInfo stream_info; + std::string body; + + std::vector commands; + commands.push_back(std::make_unique()); + + auto providers = SubstitutionFormatParser::parse("foo %COMMAND_EXTENSION(x)%", commands); + + EXPECT_EQ(providers.size(), 2); + EXPECT_EQ("TestFormatter", providers[1]->format(request_headers, response_headers, + response_trailers, stream_info, body)); +} + } // namespace } // namespace Formatter } // namespace Envoy diff --git a/test/config/utility.cc b/test/config/utility.cc index e0e6012e520c..7280fd5482c0 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -1025,7 +1025,9 @@ void ConfigHelper::addSslConfig(const ServerSslOptions& options) { filter_chain->mutable_transport_socket()->mutable_typed_config()->PackFrom(tls_context); } -bool ConfigHelper::setAccessLog(const std::string& filename, absl::string_view format) { +bool ConfigHelper::setAccessLog( + const std::string& filename, absl::string_view format, + std::vector formatters) { if (getFilterFromListener("http") == nullptr) { return false; } @@ -1035,8 +1037,15 @@ bool ConfigHelper::setAccessLog(const std::string& filename, absl::string_view f loadHttpConnectionManager(hcm_config); envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; if (!format.empty()) { - access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( - absl::StrCat(format, "\n")); + auto* log_format = access_log_config.mutable_log_format(); + log_format->mutable_text_format_source()->set_inline_string(absl::StrCat(format, "\n")); + if (!formatters.empty()) { + for (const auto& formatter : formatters) { + auto* added_formatter = log_format->add_formatters(); + added_formatter->set_name(formatter.name()); + added_formatter->mutable_typed_config()->PackFrom(formatter.typed_config()); + } + } } access_log_config.set_path(filename); hcm_config.mutable_access_log(0)->mutable_typed_config()->PackFrom(access_log_config); diff --git a/test/config/utility.h b/test/config/utility.h index b51843eb3341..8cae966a910e 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -209,7 +209,8 @@ class ConfigHelper { // Set the HTTP access log for the first HCM (if present) to a given file. The default is // the platform's null device. - bool setAccessLog(const std::string& filename, absl::string_view format = ""); + bool setAccessLog(const std::string& filename, absl::string_view format = "", + std::vector formatters = {}); // Set the listener access log for the first listener to a given file. bool setListenerAccessLog(const std::string& filename, absl::string_view format = ""); diff --git a/test/integration/BUILD b/test/integration/BUILD index 605d63e4f212..e3e1549221ca 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1585,6 +1585,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "command_formatter_extension_integration_test", + srcs = [ + "command_formatter_extension_integration_test.cc", + ], + deps = [ + ":http_integration_lib", + "//test/common/formatter:command_extension_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "health_check_integration_test", srcs = ["health_check_integration_test.cc"], diff --git a/test/integration/command_formatter_extension_integration_test.cc b/test/integration/command_formatter_extension_integration_test.cc new file mode 100644 index 000000000000..3cc1a21527dc --- /dev/null +++ b/test/integration/command_formatter_extension_integration_test.cc @@ -0,0 +1,40 @@ +#include "test/common/formatter/command_extension.h" +#include "test/integration/http_integration.h" +#include "test/test_common/registry.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::HasSubstr; + +namespace Envoy { +namespace Formatter { + +class CommandFormatterExtensionIntegrationTest : public testing::Test, public HttpIntegrationTest { +public: + CommandFormatterExtensionIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, Network::Address::IpVersion::v4) {} +}; + +TEST_F(CommandFormatterExtensionIntegrationTest, BasicExtension) { + TestCommandFactory factory; + Registry::InjectFactory command_register(factory); + std::vector formatters; + envoy::config::core::v3::TypedExtensionConfig typed_config; + ProtobufWkt::StringValue config; + + typed_config.set_name("envoy.formatter.TestFormatter"); + typed_config.mutable_typed_config()->PackFrom(config); + formatters.push_back(typed_config); + + useAccessLog("%COMMAND_EXTENSION()%", formatters); + initialize(); + std::string response; + sendRawHttpAndWaitForResponse(lookupPort("http"), "GET / HTTP/1.1\r\nHost: host\r\n\r\n", + &response, true); + std::string log = waitForAccessLog(access_log_name_); + EXPECT_THAT(log, HasSubstr("TestFormatter")); +} + +} // namespace Formatter +} // namespace Envoy diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 3e77ffef8e24..0553b16481a6 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -263,9 +263,11 @@ HttpIntegrationTest::HttpIntegrationTest(Http::CodecClient::Type downstream_prot config_helper_.setClientCodec(typeToCodecType(downstream_protocol_)); } -void HttpIntegrationTest::useAccessLog(absl::string_view format) { +void HttpIntegrationTest::useAccessLog( + absl::string_view format, + std::vector formatters) { access_log_name_ = TestEnvironment::temporaryPath(TestUtility::uniqueFilename()); - ASSERT_TRUE(config_helper_.setAccessLog(access_log_name_, format)); + ASSERT_TRUE(config_helper_.setAccessLog(access_log_name_, format, formatters)); } HttpIntegrationTest::~HttpIntegrationTest() { cleanupUpstreamAndDownstream(); } diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 457d81e666fe..13b91528e36c 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -106,7 +106,8 @@ class HttpIntegrationTest : public BaseIntegrationTest { ~HttpIntegrationTest() override; protected: - void useAccessLog(absl::string_view format = ""); + void useAccessLog(absl::string_view format = "", + std::vector formatters = {}); IntegrationCodecClientPtr makeHttpConnection(uint32_t port); // Makes a http connection object without checking its connected state.