diff --git a/BUILD b/BUILD index 662004eec..1b849c544 100644 --- a/BUILD +++ b/BUILD @@ -11,6 +11,7 @@ envoy_package() filegroup( name = "nighthawk", srcs = [ + ":nighthawk_adaptive_load_client", ":nighthawk_client", ":nighthawk_output_transform", ":nighthawk_service", @@ -18,6 +19,14 @@ filegroup( ], ) +envoy_cc_binary( + name = "nighthawk_adaptive_load_client", + repository = "@envoy", + deps = [ + "//source/exe:adaptive_load_client_entry_lib", + ], +) + envoy_cc_binary( name = "nighthawk_client", repository = "@envoy", diff --git a/api/adaptive_load/BUILD b/api/adaptive_load/BUILD index 009bbea75..6cfd818ec 100644 --- a/api/adaptive_load/BUILD +++ b/api/adaptive_load/BUILD @@ -15,7 +15,7 @@ api_cc_py_proto_library( ], visibility = ["//visibility:public"], deps = [ + "//api/client:base", "@envoy_api//envoy/config/core/v3:pkg", - "@nighthawk//api/client:base", ], ) diff --git a/source/adaptive_load/BUILD b/source/adaptive_load/BUILD index f6d8ccd8c..3d6428b05 100644 --- a/source/adaptive_load/BUILD +++ b/source/adaptive_load/BUILD @@ -32,6 +32,35 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "adaptive_load_client_main", + srcs = [ + "adaptive_load_client_main.cc", + ], + hdrs = [ + "adaptive_load_client_main.h", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":adaptive_load_controller_impl", + ":config_validator_impl", + ":input_variable_setter_impl", + ":metrics_evaluator_impl", + ":metrics_plugin_impl", + ":scoring_function_impl", + ":session_spec_proto_helper_impl", + ":step_controller_impl", + "//api/client:base_cc_proto", + "//api/client:grpc_service_lib", + "//source/common:nighthawk_common_lib", + "@envoy//source/common/common:assert_lib_with_external_headers", + "@envoy//source/common/common:minimal_logger_lib_with_external_headers", + "@envoy//source/common/event:real_time_system_lib_with_external_headers", + "@envoy//source/common/grpc:google_grpc_utils_lib_with_external_headers", + ], +) + envoy_cc_library( name = "config_validator_impl", srcs = [ diff --git a/source/adaptive_load/adaptive_load_client_main.cc b/source/adaptive_load/adaptive_load_client_main.cc new file mode 100644 index 000000000..b28c2ebf5 --- /dev/null +++ b/source/adaptive_load/adaptive_load_client_main.cc @@ -0,0 +1,137 @@ +#include "adaptive_load/adaptive_load_client_main.h" + +#include +#include +#include +#include + +#include "envoy/common/exception.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/common/exception.h" + +#include "external/envoy/source/common/grpc/google_grpc_utils.h" +#include "external/envoy/source/common/protobuf/protobuf.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/client/service.grpc.pb.h" +#include "api/client/service.pb.h" + +#include "common/utility.h" +#include "common/version_info.h" + +#include "fmt/ranges.h" +#include "google/rpc/status.pb.h" +#include "tclap/CmdLine.h" + +namespace Nighthawk { + +namespace { + +/** + * Writes a string to a file. + * + * @param filesystem Envoy abstraction around filesystem functions, to facilitate unit testing. + * @param path Relative or absolute path to the file to write. + * @param contents String to write to the file. + * + * @throw Nighthawk::NighthawkException For any filesystem error. + */ +void WriteFileOrThrow(Envoy::Filesystem::Instance& filesystem, absl::string_view path, + absl::string_view contents) { + Envoy::Filesystem::FilePtr file = filesystem.createFile(std::string(path)); + const Envoy::Api::IoCallBoolResult open_result = + file->open(((1 << Envoy::Filesystem::File::Operation::Write)) | + (1 << (Envoy::Filesystem::File::Operation::Create))); + if (!open_result.ok()) { + throw Nighthawk::NighthawkException(absl::StrCat("Unable to open output file \"", path, + "\": ", open_result.err_->getErrorDetails())); + } + const Envoy::Api::IoCallSizeResult write_result = file->write(contents); + if (!write_result.ok()) { + throw Nighthawk::NighthawkException(absl::StrCat("Unable to write to output file \"", path, + "\": ", write_result.err_->getErrorDetails())); + } + const Envoy::Api::IoCallBoolResult close_result = file->close(); + if (!close_result.ok()) { + throw Nighthawk::NighthawkException(absl::StrCat("Unable to close output file \"", path, + "\": ", close_result.err_->getErrorDetails())); + } +} + +} // namespace + +AdaptiveLoadClientMain::AdaptiveLoadClientMain(int argc, const char* const* argv, + AdaptiveLoadController& controller, + Envoy::Filesystem::Instance& filesystem) + : controller_{controller}, filesystem_{filesystem} { + TCLAP::CmdLine cmd("Adaptive Load tool that finds the optimal load on the target " + "through a series of Nighthawk Service benchmarks.", + /*delimiter=*/' ', VersionInfo::version()); + + TCLAP::ValueArg nighthawk_service_address( + /*flag_name=*/"", "nighthawk-service-address", + "host:port for Nighthawk Service. To enable TLS, set --use-tls.", + /*required=*/false, "localhost:8443", "string", cmd); + TCLAP::SwitchArg use_tls( + /*flag_name=*/"", "use-tls", + "Use TLS for the gRPC connection from this program to the Nighthawk Service. Set environment " + "variable GRPC_DEFAULT_SSL_ROOTS_FILE_PATH to override the default root certificates.", + cmd); + TCLAP::ValueArg spec_filename( + /*flag_name=*/"", "spec-file", + "Path to a textproto file describing the adaptive load session " + "(nighthawk::adaptive_load::AdaptiveLoadSessionSpec).", + /*required=*/true, /*default_value=*/"", "string", cmd); + TCLAP::ValueArg output_filename( + /*flag_name=*/"", "output-file", + "Path to write adaptive load session output textproto " + "(nighthawk::adaptive_load::AdaptiveLoadSessionOutput).", + /*required=*/true, /*default_value=*/"", "string", cmd); + + Nighthawk::Utility::parseCommand(cmd, argc, argv); + + nighthawk_service_address_ = nighthawk_service_address.getValue(); + use_tls_ = use_tls.getValue(); + spec_filename_ = spec_filename.getValue(); + output_filename_ = output_filename.getValue(); +} + +uint32_t AdaptiveLoadClientMain::Run() { + ENVOY_LOG(info, "Attempting adaptive load session: {}", DescribeInputs()); + std::string spec_textproto; + try { + spec_textproto = filesystem_.fileReadToEnd(spec_filename_); + } catch (const Envoy::EnvoyException& e) { + throw Nighthawk::NighthawkException("Failed to read spec textproto file \"" + spec_filename_ + + "\": " + e.what()); + } + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + if (!Envoy::Protobuf::TextFormat::ParseFromString(spec_textproto, &spec)) { + throw Nighthawk::NighthawkException("Unable to parse file \"" + spec_filename_ + + "\" as a text protobuf (type " + spec.GetTypeName() + ")"); + } + std::shared_ptr<::grpc::Channel> channel = grpc::CreateChannel( + nighthawk_service_address_, use_tls_ ? grpc::SslCredentials(grpc::SslCredentialsOptions()) + : grpc::InsecureChannelCredentials()); + std::unique_ptr stub( + nighthawk::client::NighthawkService::NewStub(channel)); + + absl::StatusOr output_or = + controller_.PerformAdaptiveLoadSession(stub.get(), spec); + if (!output_or.ok()) { + ENVOY_LOG(error, "Error in adaptive load session: {}", output_or.status().message()); + return 1; + } + nighthawk::adaptive_load::AdaptiveLoadSessionOutput output = output_or.value(); + WriteFileOrThrow(filesystem_, output_filename_, output.DebugString()); + return 0; +} + +std::string AdaptiveLoadClientMain::DescribeInputs() { + return "Nighthawk Service " + nighthawk_service_address_ + " using " + + (use_tls_ ? "TLS" : "insecure") + " connection, input file: " + spec_filename_ + + ", output file: " + output_filename_; +} + +} // namespace Nighthawk diff --git a/source/adaptive_load/adaptive_load_client_main.h b/source/adaptive_load/adaptive_load_client_main.h new file mode 100644 index 000000000..1e4dc4ab4 --- /dev/null +++ b/source/adaptive_load/adaptive_load_client_main.h @@ -0,0 +1,55 @@ +#pragma once + +#include "envoy/common/time.h" +#include "envoy/filesystem/filesystem.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" + +#include "external/envoy/source/common/common/logger.h" + +namespace Nighthawk { + +/** + * Main implementation of the CLI wrapper around the adaptive load controller library. + * Parses command line options, reads adaptive load session spec proto from a file, + * runs an adaptive load session, and writes the output proto to a file. + */ +class AdaptiveLoadClientMain : public Envoy::Logger::Loggable { +public: + /** + * Parses the command line arguments to class members. + * + * @param argc Standard argc passed through from the exe entry point. + * @param argv Standard argv passed through from the exe entry point. + * @param controller Adaptive load controller, passed in to allow unit testing of this class. + * @param filesystem Abstraction of the filesystem, passed in to allow unit testing of this + * class. + * + * @throw Nighthawk::Client::MalformedArgvException If command line constraints are violated. + */ + AdaptiveLoadClientMain(int argc, const char* const* argv, AdaptiveLoadController& controller, + Envoy::Filesystem::Instance& filesystem); + /** + * Loads the adaptive load session spec proto from a file, runs an adaptive load session, and + * writes the output proto to a file. File paths are taken from class members initialized in the + * constructor. + * + * @return Exit code for this process. + * @throw Nighthawk::NighthawkException If a file read or write error occurs. + */ + uint32_t Run(); + /** + * Describes the program inputs as parsed from the command line. + */ + std::string DescribeInputs(); + +private: + std::string nighthawk_service_address_; + bool use_tls_; + std::string spec_filename_; + std::string output_filename_; + AdaptiveLoadController& controller_; + Envoy::Filesystem::Instance& filesystem_; +}; + +} // namespace Nighthawk diff --git a/source/exe/BUILD b/source/exe/BUILD index d0385f278..18b672bad 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -8,6 +8,23 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "adaptive_load_client_entry_lib", + srcs = ["adaptive_load_client_main_entry.cc"], + external_deps = [ + "abseil_symbolize", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + "//source/adaptive_load:adaptive_load_client_main", + "//source/common:nighthawk_service_client_impl", + "//source/common:version_linkstamp", + "@envoy//source/exe:platform_header_lib_with_external_headers", + "@envoy//source/exe:platform_impl_lib", + ], +) + envoy_cc_library( name = "nighthawk_client_entry_lib", srcs = ["client_main_entry.cc"], diff --git a/source/exe/adaptive_load_client_main_entry.cc b/source/exe/adaptive_load_client_main_entry.cc new file mode 100644 index 000000000..67d45b434 --- /dev/null +++ b/source/exe/adaptive_load_client_main_entry.cc @@ -0,0 +1,45 @@ +// Command line adaptive load controller driving a Nighthawk Service. +#include + +#include "nighthawk/common/exception.h" + +#include "external/envoy/source/common/event/real_time_system.h" +#include "external/envoy/source/exe/platform_impl.h" + +#include "common/nighthawk_service_client_impl.h" + +#include "absl/debugging/symbolize.h" +#include "adaptive_load/adaptive_load_client_main.h" +#include "adaptive_load/adaptive_load_controller_impl.h" +#include "adaptive_load/metrics_evaluator_impl.h" +#include "adaptive_load/session_spec_proto_helper_impl.h" + +// NOLINT(namespace-nighthawk) + +int main(int argc, char* argv[]) { +#ifndef __APPLE__ + // absl::Symbolize mostly works without this, but this improves corner case + // handling, such as running in a chroot jail. + absl::InitializeSymbolizer(argv[0]); +#endif + Nighthawk::NighthawkServiceClientImpl nighthawk_service_client; + Nighthawk::MetricsEvaluatorImpl metrics_evaluator; + Nighthawk::AdaptiveLoadSessionSpecProtoHelperImpl spec_proto_helper; + Envoy::Event::RealTimeSystem time_system; // NO_CHECK_FORMAT(real_time) + Nighthawk::AdaptiveLoadControllerImpl controller(nighthawk_service_client, metrics_evaluator, + spec_proto_helper, time_system); + Envoy::PlatformImpl platform_impl; + try { + Nighthawk::AdaptiveLoadClientMain program(argc, argv, controller, platform_impl.fileSystem()); + return program.Run(); + } catch (const Nighthawk::Client::NoServingException& e) { + return EXIT_SUCCESS; + } catch (const Nighthawk::Client::MalformedArgvException& e) { + std::cerr << "Invalid args: " << e.what() << std::endl; + return EXIT_FAILURE; + } catch (const Nighthawk::NighthawkException& e) { + std::cerr << "Failure: " << e.what() << std::endl; + return EXIT_FAILURE; + } + return 0; +} diff --git a/test/adaptive_load/BUILD b/test/adaptive_load/BUILD index b831726cc..902e744b0 100644 --- a/test/adaptive_load/BUILD +++ b/test/adaptive_load/BUILD @@ -21,6 +21,28 @@ envoy_cc_test_library( ], ) +envoy_cc_test( + name = "adaptive_load_client_main_test", + srcs = ["adaptive_load_client_main_test.cc"], + data = [ + "test_data/golden_output.textproto", + "test_data/invalid_session_spec.textproto", + "test_data/valid_session_spec.textproto", + ], + repository = "@envoy", + deps = [ + ":minimal_output", + "//include/nighthawk/adaptive_load:adaptive_load_controller", + "//source/adaptive_load:adaptive_load_client_main", + "//test/mocks/adaptive_load:mock_adaptive_load_controller", + "//test/test_common:environment_lib", + "@com_github_grpc_grpc//:grpc++_test", + "@envoy//source/common/protobuf:utility_lib_with_external_headers", + "@envoy//test/mocks/filesystem:filesystem_mocks", + "@envoy//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "adaptive_load_controller_test", srcs = ["adaptive_load_controller_test.cc"], diff --git a/test/adaptive_load/adaptive_load_client_main_test.cc b/test/adaptive_load/adaptive_load_client_main_test.cc new file mode 100644 index 000000000..49fee0928 --- /dev/null +++ b/test/adaptive_load/adaptive_load_client_main_test.cc @@ -0,0 +1,379 @@ +#include "envoy/api/io_error.h" +#include "envoy/filesystem/filesystem.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/common/exception.h" + +#include "external/envoy/test/mocks/filesystem/mocks.h" +#include "external/envoy/test/test_common/file_system_for_test.h" +#include "external/envoy/test/test_common/utility.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/adaptive_load/benchmark_result.pb.h" + +#include "test/adaptive_load/minimal_output.h" +#include "test/mocks/adaptive_load/mock_adaptive_load_controller.h" +#include "test/test_common/environment.h" + +#include "absl/strings/string_view.h" +#include "adaptive_load/adaptive_load_client_main.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Nighthawk { + +namespace { + +using ::testing::_; +using ::testing::ByMove; +using ::testing::DoAll; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SetArgPointee; + +/** + * Envoy IO error value to simulate filesystem errors. + */ +class UnknownIoError : public Envoy::Api::IoError { +public: + IoErrorCode getErrorCode() const override { + return Envoy::Api::IoError::IoErrorCode::UnknownError; + } + std::string getErrorDetails() const override { return "unknown error details"; } +}; + +/** + * Creates a minimal valid output that matches test/adaptive_load/test_data/golden_output.textproto. + * + * @return AdaptiveLoadSessionOutput + */ +nighthawk::adaptive_load::AdaptiveLoadSessionOutput MakeBasicAdaptiveLoadSessionOutput() { + nighthawk::adaptive_load::AdaptiveLoadSessionOutput output; + nighthawk::adaptive_load::MetricEvaluation* evaluation = + output.mutable_adjusting_stage_results()->Add()->add_metric_evaluations(); + evaluation->set_metric_id("com.a/b"); + evaluation->set_metric_value(123); + return output; +} + +TEST(AdaptiveLoadClientMainTest, FailsWithNoInputs) { + const char* const argv[] = { + "executable-name-here", + }; + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + EXPECT_THROW_WITH_REGEX(AdaptiveLoadClientMain(1, argv, controller, filesystem), + Nighthawk::Client::MalformedArgvException, "Required arguments missing"); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfSpecFileNotSet) { + std::string outfile = Nighthawk::TestEnvironment::runfilesPath("unused.textproto"); + const char* const argv[] = { + "executable-name-here", + "--output-file", + outfile.c_str(), + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + EXPECT_THROW_WITH_REGEX(AdaptiveLoadClientMain(3, argv, controller, filesystem), + Nighthawk::Client::MalformedArgvException, + "Required argument missing: spec-file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfOutputFileNotSet) { + std::string infile = Nighthawk::TestEnvironment::runfilesPath("unused.textproto"); + const char* const argv[] = { + "executable-name-here", + "--spec-file", + infile.c_str(), + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + EXPECT_THROW_WITH_REGEX(AdaptiveLoadClientMain main(3, argv, controller, filesystem), + Nighthawk::Client::MalformedArgvException, + "Required argument missing: output-file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsWithNonexistentInputFile) { + std::string infile = Nighthawk::TestEnvironment::runfilesPath("nonexistent.textproto"); + std::string outfile = Nighthawk::TestEnvironment::runfilesPath("unused.textproto"); + const char* const argv[] = { + "executable-name-here", "--spec-file", infile.c_str(), "--output-file", outfile.c_str(), + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, + "Failed to read spec textproto file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsWithUnparseableInputFile) { + std::string infile = Nighthawk::TestEnvironment::runfilesPath( + "test/adaptive_load/test_data/invalid_session_spec.textproto"); + std::string outfile = Nighthawk::TestEnvironment::runfilesPath("unused.textproto"); + const char* const argv[] = { + "executable-name-here", "--spec-file", infile.c_str(), "--output-file", outfile.c_str(), + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, "Unable to parse file"); +} + +TEST(AdaptiveLoadClientMainTest, ExitsProcessWithNonzeroStatusOnAdaptiveControllerError) { + std::string infile = Nighthawk::TestEnvironment::runfilesPath( + "test/adaptive_load/test_data/valid_session_spec.textproto"); + std::string outfile = Nighthawk::TestEnvironment::runfilesPath( + "test/adaptive_load/test_data/nonexistent-dir/out.textproto"); + const char* const argv[] = { + "executable-name-here", "--spec-file", infile.c_str(), "--output-file", outfile.c_str(), + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(absl::DataLossError("error message"))); + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_EQ(main.Run(), 1); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfCreatingOutputFileFails) { + std::string infile = Nighthawk::TestEnvironment::runfilesPath( + "test/adaptive_load/test_data/valid_session_spec.textproto"); + std::string outfile = Nighthawk::TestEnvironment::runfilesPath( + "test/adaptive_load/test_data/nonexistent-dir/out.textproto"); + const char* const argv[] = { + "executable-name-here", "--spec-file", infile.c_str(), "--output-file", outfile.c_str(), + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(MakeBasicAdaptiveLoadSessionOutput())); + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, "Unable to open output file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfOpeningOutputFileFails) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "in-dummy.textproto", + "--output-file", "out-dummy.textproto", + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(MakeBasicAdaptiveLoadSessionOutput())); + + NiceMock filesystem; + + std::string infile_contents = + Envoy::Filesystem::fileSystemForTest().fileReadToEnd(Nighthawk::TestEnvironment::runfilesPath( + std::string("test/adaptive_load/test_data/valid_session_spec.textproto"))); + EXPECT_CALL(filesystem, fileReadToEnd(_)).WillOnce(Return(infile_contents)); + + NiceMock* mock_file = new NiceMock; + EXPECT_CALL(filesystem, createFile(_)) + .WillOnce(Return(ByMove(std::unique_ptr>(mock_file)))); + + EXPECT_CALL(*mock_file, open_(_)) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + false, Envoy::Api::IoErrorPtr(new UnknownIoError(), + [](Envoy::Api::IoError* err) { delete err; }))))); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, "Unable to open output file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfWritingOutputFileFails) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "in-dummy.textproto", + "--output-file", "out-dummy.textproto", + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(MakeBasicAdaptiveLoadSessionOutput())); + + NiceMock filesystem; + + std::string infile_contents = + Envoy::Filesystem::fileSystemForTest().fileReadToEnd(Nighthawk::TestEnvironment::runfilesPath( + std::string("test/adaptive_load/test_data/valid_session_spec.textproto"))); + EXPECT_CALL(filesystem, fileReadToEnd(_)).WillOnce(Return(infile_contents)); + + NiceMock* mock_file = new NiceMock; + EXPECT_CALL(filesystem, createFile(_)) + .WillOnce(Return(ByMove(std::unique_ptr>(mock_file)))); + + EXPECT_CALL(*mock_file, open_(_)) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + true, Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {}))))); + EXPECT_CALL(*mock_file, write_(_)) + .WillOnce(Return(ByMove(Envoy::Api::IoCallSizeResult( + -1, Envoy::Api::IoErrorPtr(new UnknownIoError(), + [](Envoy::Api::IoError* err) { delete err; }))))); + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, + "Unable to write to output file"); +} + +TEST(AdaptiveLoadClientMainTest, FailsIfClosingOutputFileFails) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "in-dummy.textproto", + "--output-file", "out-dummy.textproto", + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(MakeBasicAdaptiveLoadSessionOutput())); + + NiceMock filesystem; + + std::string infile_contents = + Envoy::Filesystem::fileSystemForTest().fileReadToEnd(Nighthawk::TestEnvironment::runfilesPath( + std::string("test/adaptive_load/test_data/valid_session_spec.textproto"))); + EXPECT_CALL(filesystem, fileReadToEnd(_)).WillOnce(Return(infile_contents)); + + NiceMock* mock_file = new NiceMock; + EXPECT_CALL(filesystem, createFile(_)) + .WillOnce(Return(ByMove(std::unique_ptr>(mock_file)))); + + EXPECT_CALL(*mock_file, open_(_)) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + true, Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {}))))); + EXPECT_CALL(*mock_file, write_(_)) + .WillRepeatedly(Invoke([](absl::string_view data) -> Envoy::Api::IoCallSizeResult { + return Envoy::Api::IoCallSizeResult( + static_cast(data.length()), + Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {})); + })); + EXPECT_CALL(*mock_file, close_()) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + false, Envoy::Api::IoErrorPtr(new UnknownIoError(), + [](Envoy::Api::IoError* err) { delete err; }))))); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + EXPECT_THROW_WITH_REGEX(main.Run(), Nighthawk::NighthawkException, "Unable to close output file"); +} + +TEST(AdaptiveLoadClientMainTest, WritesOutputProtoToFile) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "in-dummy.textproto", + "--output-file", "out-dummy.textproto", + }; + + MockAdaptiveLoadController controller; + EXPECT_CALL(controller, PerformAdaptiveLoadSession(_, _)) + .WillOnce(Return(MakeBasicAdaptiveLoadSessionOutput())); + + NiceMock filesystem; + + std::string infile_contents = + Envoy::Filesystem::fileSystemForTest().fileReadToEnd(Nighthawk::TestEnvironment::runfilesPath( + std::string("test/adaptive_load/test_data/valid_session_spec.textproto"))); + EXPECT_CALL(filesystem, fileReadToEnd(_)).WillOnce(Return(infile_contents)); + + std::string actual_outfile_contents; + NiceMock* mock_file = new NiceMock; + EXPECT_CALL(filesystem, createFile(_)) + .WillOnce(Return(ByMove(std::unique_ptr>(mock_file)))); + + EXPECT_CALL(*mock_file, open_(_)) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + true, Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {}))))); + EXPECT_CALL(*mock_file, write_(_)) + .WillRepeatedly(Invoke( + [&actual_outfile_contents](absl::string_view data) -> Envoy::Api::IoCallSizeResult { + actual_outfile_contents += data; + return Envoy::Api::IoCallSizeResult( + static_cast(data.length()), + Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {})); + })); + + EXPECT_CALL(*mock_file, close_()) + .WillOnce(Return(ByMove(Envoy::Api::IoCallBoolResult( + true, Envoy::Api::IoErrorPtr(nullptr, [](Envoy::Api::IoError*) {}))))); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + main.Run(); + + std::string golden_output = + Envoy::Filesystem::fileSystemForTest().fileReadToEnd(Nighthawk::TestEnvironment::runfilesPath( + std::string("test/adaptive_load/test_data/golden_output.textproto"))); + EXPECT_EQ(actual_outfile_contents, golden_output); +} + +TEST(AdaptiveLoadClientMainTest, DefaultsToInsecureConnection) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "a", "--output-file", "b", + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + + EXPECT_THAT(main.DescribeInputs(), HasSubstr("insecure")); +} + +TEST(AdaptiveLoadClientMainTest, UsesTlsConnectionWhenSpecified) { + const char* const argv[] = { + "executable-name-here", "--use-tls", "--spec-file", "a", "--output-file", "b", + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(6, argv, controller, filesystem); + + EXPECT_THAT(main.DescribeInputs(), HasSubstr("TLS")); +} + +TEST(AdaptiveLoadClientMainTest, UsesDefaultNighthawkServiceAddress) { + const char* const argv[] = { + "executable-name-here", "--spec-file", "a", "--output-file", "b", + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(5, argv, controller, filesystem); + + EXPECT_THAT(main.DescribeInputs(), HasSubstr("localhost:8443")); +} + +TEST(AdaptiveLoadClientMainTest, UsesCustomNighthawkServiceAddress) { + const char* const argv[] = { + "executable-name-here", + "--nighthawk-service-address", + "1.2.3.4:5678", + "--spec-file", + "a", + "--output-file", + "b", + }; + + NiceMock controller; + Envoy::Filesystem::Instance& filesystem = Envoy::Filesystem::fileSystemForTest(); + + AdaptiveLoadClientMain main(7, argv, controller, filesystem); + + EXPECT_THAT(main.DescribeInputs(), HasSubstr("1.2.3.4:5678")); +} + +} // namespace + +} // namespace Nighthawk diff --git a/test/adaptive_load/test_data/golden_output.textproto b/test/adaptive_load/test_data/golden_output.textproto new file mode 100644 index 000000000..af90ac058 --- /dev/null +++ b/test/adaptive_load/test_data/golden_output.textproto @@ -0,0 +1,6 @@ +adjusting_stage_results { + metric_evaluations { + metric_id: "com.a/b" + metric_value: 123 + } +} diff --git a/test/adaptive_load/test_data/invalid_session_spec.textproto b/test/adaptive_load/test_data/invalid_session_spec.textproto new file mode 100644 index 000000000..5f82458f7 --- /dev/null +++ b/test/adaptive_load/test_data/invalid_session_spec.textproto @@ -0,0 +1 @@ +Bogus text will not parse as a textproto. diff --git a/test/adaptive_load/test_data/valid_session_spec.textproto b/test/adaptive_load/test_data/valid_session_spec.textproto new file mode 100644 index 000000000..9896f3593 --- /dev/null +++ b/test/adaptive_load/test_data/valid_session_spec.textproto @@ -0,0 +1,98 @@ +nighthawk_traffic_template { + # This is a full Nighthawk CommandLineOptions proto, input to Nighthawk Service. + # The adaptive load controller will send this proto as-is to the Nighthawk Service, + # except with a dynamic requests_per_second value inserted. + # + # See https://github.com/envoyproxy/nighthawk/blob/main/api/client/options.proto + # for full details on available settings. + # + # Note that the address of the system under test is set here. + # + # Add any customizations here such as headers. + uri { + value: "http://127.0.0.1:12345/" + } +} + +# 3 seconds per benchmark is good for demos. +# 60 seconds per benchmark has been found to reduce noise. +measuring_period { + seconds: 3 +} + +convergence_deadline { + seconds: 1000 +} + +# Confirm the final qps with a longer benchmark. +testing_stage_duration { + seconds: 10 +} + +# Back off if the latency statistic is over a threshold. +# See https://github.com/envoyproxy/nighthawk/blob/9ade1a58c787e4d0e165cabbb42f6a410a56a865/source/adaptive_load/metrics_plugin_impl.cc#L148 +# for the full list of supported metrics. +metric_thresholds { + # metric_spec { metric_name: "latency-ns-mean-plus-2stdev" } + metric_spec { metric_name: "latency-ns-max" } + threshold_spec { + scoring_function { + name: "nighthawk.binary_scoring" + typed_config { + [type.googleapis.com/ + nighthawk.adaptive_load.BinaryScoringFunctionConfig] { + # 1,000,000 ns = 1 ms + upper_threshold { value: 1000000 } + } + } + } + } +} + +# Back off if Nighthawk internal limitations prevented sending >5% of requests. +metric_thresholds { + metric_spec { + metric_name: "send-rate" + } + threshold_spec { + scoring_function { + name: "nighthawk.binary_scoring" + typed_config { + [type.googleapis.com/nighthawk.adaptive_load.BinaryScoringFunctionConfig] { + lower_threshold { + value: 0.95 + } + } + } + } + } +} + +# Back off if less than 95% of received responses were 2xx. +metric_thresholds { + metric_spec { + metric_name: "success-rate" + } + threshold_spec { + scoring_function { + name: "nighthawk.binary_scoring" + typed_config { + [type.googleapis.com/nighthawk.adaptive_load.BinaryScoringFunctionConfig] { + lower_threshold { + value: 0.95 + } + } + } + } + } +} + +# Exponential search starting with 10 qps +step_controller_config { + name: "nighthawk.exponential_search" + typed_config { + [type.googleapis.com/nighthawk.adaptive_load.ExponentialSearchStepControllerConfig] { + initial_value: 10.0 + } + } +}