[salvo] Initial commit for benchmark abstraction framework

.gitignore

@@ -0,0 +1,5 @@
+**/*.pyc
+**/*.swp
+**/.vscode/*
+**/__pycache__/*
+bazel-*

README.md

@@ -21,3 +21,5 @@ Performance benchmarking can take multiple forms:
 2. [siege/](siege/README.md) contains an initial attempt at a simple test to run
    iteratively during development to get a view of the time/space impact of the
    changes under configuration.
+2. [salvo/](salvo/README.md) contains a framework that abstracts nighthawk 
+   benchmark execution. This is still under active development

salvo/BUILD

@@ -0,0 +1,27 @@
+licenses(["notice"])
+
+py_binary(
+  name = "salvo",
+  srcs = [ "salvo.py" ],
+  srcs_version = "PY3",
+  deps = [
+    ":api",
+    ":lib",
+  ],
+)
+
+py_library(
+    name = "api",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//src/lib/api:schema_proto",
+    ],
+)
+
+py_library(
+    name = "lib",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//src/lib:helper_library",
+    ],
+)

salvo/README.md

@@ -0,0 +1,73 @@
+# salvo
+
+This is a framework that abstracts executing multiple benchmarks of the Envoy Proxy using [NightHawk](https://github.com/envoyproxy/nighthawk).
+
+## Example Control Documents
+
+The control document defines the data needed to excute a benchmark.  At the moment the dockerized scavenging benchmark is the only one supported.  To run the benchmark, create a file with the following example contents:
+
+JSON Example:
+
+```json
+{
+  "remote": false,
+  "dockerizedBenchmark": true,
+  "images": {
+    "reuseNhImages": true,
+    "nighthawkBenchmarkImage": "envoyproxy/nighthawk-benchmark-dev:latest",
+    "nighthawkBinaryImage": "envoyproxy/nighthawk-dev:latest",
+    "envoyImage": "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f"
+  },
+  "environment": {
+    "v4only": true,
+    "envoyPath": "envoy",
+    "outputDir": "/home/ubuntu/nighthawk_output",
+    "testDir": "/home/ubuntu/nighthawk_tests"
+  }
+}
+```
+
+YAML Example:
+
+```yaml
+remote: false
+dockerizedBenchmark: true
+environment:
+  envoyPath: 'envoy'
+  outputDir: '/home/ubuntu/nighthawk_output'
+  testDir: '/home/ubuntu/nighthawk_tests'
+  v4only: true
+images:
+  reuseNhImages: true
+  nighthawkBenchmarkImage: 'envoyproxy/nighthawk-benchmark-dev:latest'
+  nighthawkBinaryImage: 'envoyproxy/nighthawk-dev:latest'
+  envoyImage: "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f"
+```
+
+In both examples, the envoy image being tested is a specific hash.  This hash can be replaced with "latest" to test the most recently created image against the previous image built from the prior Envoys master commit.
+
+
+## Building Salvo
+
+```bash
+bazel build //:salvo
+```
+
+## Running Salvo
+
+```bash
+bazel-bin/salvo --job ~/test_data/demo_jobcontrol.yaml
+```
+
+## Testing Salvo
+
+```bash
+ bazel test //test:*
+```
+
+## Dependencies
+
+* python 3.6+
+* git
+* docker
+* tuned/tunedadm (eventually)

salvo/WORKSPACE

@@ -0,0 +1,16 @@
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+git_repository(
+    name = "com_google_protobuf",
+    remote = "https://github.com/protocolbuffers/protobuf",
+    tag = "v3.10.0",
+)
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+
+protobuf_deps()
+
+local_repository(
+    name = "salvo_build_config",
+    path = ".",
+)

salvo/salvo.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import site
+import sys
+
+# Run in the actual bazel directory so that the sys.path
+# is setup correctly
+if os.path.islink(sys.argv[0]):
+    real_exec_dir = os.path.dirname(sys.argv[0])
+    os.chdir(real_exec_dir)
+
+site.addsitedir("src")
+
+from lib.message_helper import load_control_doc
+from lib.run_benchmark import Benchmark
+
+LOGFORMAT = "%(asctime)s: %(process)d [ %(levelname)-5s] [%(module)-5s] %(message)s"
+
+log = logging.getLogger()
+
+def setup_logging(loglevel=logging.DEBUG):
+    """Basic logging configuration """
+
+    logging.basicConfig(format=LOGFORMAT, level=loglevel)
+
+def setup_options():
+    """Parse command line arguments required for operation"""
+
+    parser = argparse.ArgumentParser(description="Salvo Benchmark Runner")
+    parser.add_argument('--job', dest='jobcontrol',
+                        help='specify the location for the job control json document')
+    # FIXME: Add an option to generate a default job Control JSON/YAML
+
+    return parser.parse_args()
+
+def main():
+    """Driver module for benchmark """
+
+    args = setup_options()
+    setup_logging()
+
+    if not args.jobcontrol:
+        print("No job control document specified.  Use \"--help\" for usage")
+        return 1
+
+    job_control = load_control_doc(args.jobcontrol)
+
+    log.debug("Job definition:\n%s\n%s\n%s\n", '='*20, job_control, '='*20)
+
+    benchmark = Benchmark(job_control)
+    try:
+        benchmark.validate()
+    # TODO: Create a different class for these exceptions
+    except Exception as validation_exception:
+        log.error("Unable to validate data needed for benchmark run: %s", validation_exception)
+        return 1
+
+    benchmark.execute()
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
+#  vim: set ts=4 sw=4 tw=0 et :

salvo/src/lib/BUILD

@@ -0,0 +1,12 @@
+py_library(
+  name = "helper_library",
+  data = glob([
+      '*.py',
+      'benchmark/*.py',
+      'common/*.py',
+  ], allow_empty=False) + 
+  [
+    "//:api",
+  ],
+  visibility = ["//visibility:public"],
+)

salvo/src/lib/api/BUILD

@@ -0,0 +1,7 @@
+load("@com_google_protobuf//:protobuf.bzl", "py_proto_library")
+
+py_proto_library(
+  name = "schema_proto",
+  srcs = glob(['*.proto'], allow_empty=False),
+  visibility = ["//visibility:public"],
+)

salvo/src/lib/api/control.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package salvo;
+
+import "src/lib/api/image.proto";
+import "src/lib/api/source.proto";
+import "src/lib/api/env.proto";
+
+
+message JobControl {
+  // Specify whether the benchmark runs locally or in a service
+  bool remote = 1;
+
+  // Specify the benchmark to execute
+  oneof benchmark {
+    bool scavenging_benchmark = 2;
+    bool dockerized_benchmark = 3;
+    bool binary_benchmark = 4;
+  }
+
+  // Define where we find all required sources
+  repeated SourceRepository source = 6;
+
+  // Define the names of all required docker images
+  DockerImages images = 7;
+
+  // Define the environment variables needed for the test
+  EnvironmentVars environment = 8;
+}
+

salvo/src/lib/api/docker_volume.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+
+package salvo;
+
+// This message defines the properties for a given mount.  It is used
+// to generate the dictionary specifying volumes for the Python Docker
+// SDK
+message VolumeProperties {
+  // Defines a list of properties governing the mount in the container
+  string bind = 1;
+
+  // Define whether the mount point is read-write or read-only
+  string mode = 2;
+}
+
+// This message defines the volume structure consumed by the command
+// to run a docker image. 
+message Volume {
+  // Specify a map of volumes and their mount points for use in a container
+  map<string, VolumeProperties> volumes = 1;
+}
+
+

salvo/src/lib/api/env.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package salvo;
+
+// Capture all Environment variables required for the benchmark
+message EnvironmentVars {
+  // Specify the IP version for tests
+  oneof envoy_ip_test_versions {
+    bool v4only = 1;
+    bool v6only = 2;
+    bool all = 3;
+  }
+
+  // Controls whether envoy is placed between the nighthawk client and server
+  string envoy_path = 4;
+
+  // Specify the output directory for nighthawk artifacts
+  string output_dir = 5;
+
+  // Specify the directory where external tests are located
+  string test_dir = 6;
+
+  // Additional environment variables that may be needed for operation
+  map<string, string> variables = 7;
+}
+

salvo/src/lib/api/image.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package salvo;
+
+// Capture all docker images required for the benchmark. This object is examined
+// first before evaluating the state of specified source locations
+message DockerImages {
+  // Determines whether required docker are used if they already exist.  The
+  // benchmark image and binary image must be specified. If this is set to false
+  // there must be a specified source location from which we build the image
+
+  // This should be implicit. Images are preferred over building from source. We
+  // build only if the image pull is not successful and sources are present
+  bool reuse_nh_images = 1;
+
+  // Specifies the name of the docker image containing the benchmark framework
+  // If populated we will attempt to pull this image
+  string nighthawk_benchmark_image = 2;
+
+  // Specifies the name of the docker image containing nighthawk binaries
+  string nighthawk_binary_image = 3;
+
+  // Specifies the envoy image from which Envoy is injected
+  string envoy_image = 4;
+}
+

salvo/src/lib/api/source.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package salvo;
+
+// Capture the location of sources needed for the benchmark
+message SourceRepository {
+  // Specify whether this source location is Envoy or NightHawk
+  oneof identity {
+    bool envoy = 1;
+    bool nighthawk = 2;
+  }
+
+  // Specify the location of the source repository on disk. If specified
+  // this location is used to determine the origin url, branch, and commit
+  // hash.  If not specified, the remaining fields must be populated
+  string location = 3;
+
+  // Specify the remote location of the repository.  This is ignored if
+  // the source location is specified.
+  string url = 4;
+
+  // Specify the local working branch.This is ignored if the source 
+  // location is specified.
+  string branch = 5;
+
+  // Specify a commit hash if applicable.  If not identified we will 
+  // determine this from the source tree.  We will also use this field
+  // to identify the corresponding NightHawk or Envoy image used for
+  // the benchmark
+  string hash = 6;
+
+  // Internal use only. This field is used to specify the location of
+  // the patch generated from changes in the local environment
+  string patch = 7;
+}