diff --git a/.circleci/config.yml b/.circleci/config.yml index 03cae01a..5a09560b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ references: envoy-build-image: &envoy-build-image - envoyproxy/envoy-build:latest + envoyproxy/envoy-build-ubuntu:e7ea4e81bbd5028abb9d3a2f2c0afe063d9b62c0 version: 2 jobs: diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b1fbef32 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +**/*.pyc +**/*.swp +**/.vscode/* +**/__pycache__/* +bazel-* +**/venv/* diff --git a/README.md b/README.md index 89f71543..2d50c3a4 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,5 @@ Performance benchmarking can take multiple forms: 2. [siege/](siege/README.md) contains an initial attempt at a simple test to run iteratively during development to get a view of the time/space impact of the changes under configuration. +2. [salvo/](salvo/README.md) contains a framework that abstracts nighthawk + benchmark execution. This is still under active development diff --git a/ci/do_ci.sh b/ci/do_ci.sh index a49a6175..09c36959 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -1,4 +1,44 @@ #!/bin/bash -# This is a dummy, populate with real CI. -echo "Hello world" +# Run a CI build/test target + +# Avoid using "-x" here so that we do not leak credentials +# in build logs +set -e + +# Build the salvo framework +function build_salvo() { + echo "Building Salvo" + pushd salvo + bazel build //... + popd +} + +# Test the salvo framework +function test_salvo() { + echo "Running Salvo unit tests" + pushd salvo + ./install_deps.sh + + bazel test //... + + popd +} + + +# Set the build target. If no parameters are specified +# we default to "build" +build_target=${1:-build} + +case $build_target in + "build") + build_salvo + ;; + "test") + test_salvo + ;; + *) + ;; +esac + +exit 0 diff --git a/salvo/BUILD b/salvo/BUILD new file mode 100644 index 00000000..6d0d9c28 --- /dev/null +++ b/salvo/BUILD @@ -0,0 +1,14 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +licenses(["notice"]) # Apache 2 + +py_binary( + name = "salvo", + srcs = ["salvo.py"], + srcs_version = "PY3", + deps = [ + "//api:schema_proto", + "//src/lib:job_control", + ], +) + diff --git a/salvo/README.md b/salvo/README.md new file mode 100644 index 00000000..057d5064 --- /dev/null +++ b/salvo/README.md @@ -0,0 +1,78 @@ +# Salvo + +This is a framework that abstracts executing multiple benchmarks of the Envoy Proxy using [NightHawk](https://github.com/envoyproxy/nighthawk). + +## Example Control Documents + +The control document defines the data needed to execute a benchmark. At the moment, the fully dockerized benchmark is the only one supported. This benchmark discoveres user supplied tests for execution and uses docker images to run the tests. In the example below, the user supplied tests files are located in `/home/ubuntu/nighthawk_tests` and are mapped to a volume in the docker container. + +To run the benchmark, create a file with the following example contents: + +JSON Example: + +```json +{ + "remote": false, + "dockerizedBenchmark": true, + "images": { + "reuseNhImages": true, + "nighthawkBenchmarkImage": "envoyproxy/nighthawk-benchmark-dev:latest", + "nighthawkBinaryImage": "envoyproxy/nighthawk-dev:latest", + "envoyImage": "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + }, + "environment": { + "testVersions": V4ONLY, + "envoyPath": "envoy", + "outputDir": "/home/ubuntu/nighthawk_output", + "testDir": "/home/ubuntu/nighthawk_tests" + } +} +``` + +YAML Example: + +```yaml +remote: false +dockerizedBenchmark: true +environment: + envoyPath: 'envoy' + outputDir: '/home/ubuntu/nighthawk_output' + testDir: '/home/ubuntu/nighthawk_tests' + testVersions: V4ONLY +images: + reuseNhImages: true + nighthawkBenchmarkImage: 'envoyproxy/nighthawk-benchmark-dev:latest' + nighthawkBinaryImage: 'envoyproxy/nighthawk-dev:latest' + envoyImage: "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" +``` + +In both examples, the envoy image being tested is a specific hash. This hash can be replaced with "latest" to test the most recently created image against the previous image built from the prior Envoys master commit. + +## Building Salvo + +```bash +bazel build //:salvo +``` + +## Running Salvo + +```bash +bazel-bin/salvo --job ~/test_data/demo_jobcontrol.yaml +``` + +## Testing Salvo + + +From the root package directory, run the do_ci.sh script with the "test" argument. Since this installs packages packages, it will need to be run as root. +```bash +bazel test //src/... +``` + +## Dependencies + +The `install_deps.sh` script will install any dependencies needed to run salvo. + +* python 3.6+ +* git +* docker +* tuned/tunedadm (eventually) diff --git a/salvo/WORKSPACE b/salvo/WORKSPACE new file mode 100644 index 00000000..89a380e7 --- /dev/null +++ b/salvo/WORKSPACE @@ -0,0 +1,18 @@ +workspace(name = "salvo") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "com_google_protobuf", + remote = "https://github.com/protocolbuffers/protobuf", + tag = "v3.10.0", +) + +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + +local_repository( + name = "salvo_build_config", + path = ".", +) diff --git a/salvo/api/BUILD b/salvo/api/BUILD new file mode 100644 index 00000000..63251cae --- /dev/null +++ b/salvo/api/BUILD @@ -0,0 +1,15 @@ +load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") + +licenses(["notice"]) # Apache 2 + +py_proto_library( + name = "schema_proto", + srcs = [ + "control.proto", + "docker_volume.proto", + "env.proto", + "image.proto", + "source.proto", + ], + visibility = ["//visibility:public"], +) diff --git a/salvo/api/control.proto b/salvo/api/control.proto new file mode 100644 index 00000000..ae67450f --- /dev/null +++ b/salvo/api/control.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package salvo; + +import "api/image.proto"; +import "api/source.proto"; +import "api/env.proto"; + +// This message type defines the schema for the consumed data file +// controlling the benchmark being executed. In it a user will +// define whether the benchmark uses images only, builds images +// from source, and whether the benchark executes locally or remotely. +message JobControl { + // Specify whether the benchmark runs locally or in a service + bool remote = 1; + + // Specify the benchmark to execute + oneof benchmark { + bool scavenging_benchmark = 2; + bool dockerized_benchmark = 3; + bool binary_benchmark = 4; + } + + // Define where we find all required sources + repeated SourceRepository source = 6; + + // Define the names of all required docker images + DockerImages images = 7; + + // Define the environment variables needed for the test + EnvironmentVars environment = 8; +} diff --git a/salvo/api/docker_volume.proto b/salvo/api/docker_volume.proto new file mode 100644 index 00000000..d15f4849 --- /dev/null +++ b/salvo/api/docker_volume.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package salvo; + +// This message defines the properties for a given mount. It is used +// to generate the dictionary specifying volumes for the Python Docker +// SDK. This message is not constructed by the user +message VolumeProperties { + // Define whether the mount point is read-write or read-only + enum Mode { + RO = 0; + RW = 1; + } + + // Specified the destination mount path in the benchmark container + string bind = 1; + + // Specify whether the volume is read-write or read-only + Mode mode = 2; +} + +// This message defines the volume structure consumed by the command +// to run a docker image. +message Volume { + // Specify a map of local paths and their mount points in the container + map volumes = 1; +} diff --git a/salvo/api/env.proto b/salvo/api/env.proto new file mode 100644 index 00000000..f0eb9e2a --- /dev/null +++ b/salvo/api/env.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package salvo; + +// Capture all Environment variables required for the benchmark +message EnvironmentVars { + // Specify the IP version for tests + enum TestIpVersion { + V4ONLY = 0; + V6ONLY = 1; + ALL = 2; + } + + TestIpVersion test_version = 1; + + // Controls whether envoy is placed between the nighthawk client and server + string envoy_path = 2; + + // Specify the output directory for nighthawk artifacts + // eg: "/home/user/test_output_path" + string output_dir = 3; + + // Specify the directory where external tests are located + // eg: "/home/user/nighthawk_external_tests" + string test_dir = 4; + + // Additional environment variables that may be needed for operation + map variables = 5; +} diff --git a/salvo/api/image.proto b/salvo/api/image.proto new file mode 100644 index 00000000..71fe7eeb --- /dev/null +++ b/salvo/api/image.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package salvo; + +// Capture all docker images required for the benchmark. This object is examined +// first before evaluating the state of specified source locations +message DockerImages { + // Determines whether required docker are used if they already exist. The + // benchmark image and binary image must be specified. If this is set to false + // there must be a specified source location from which we build the image + + // This should be implicit. Images are preferred over building from source. We + // build only if the image pull is not successful and sources are present + bool reuse_nh_images = 1; + + // Specifies the name of the docker image containing the benchmark framework + // If populated we will attempt to pull this image + // eg: "envoyproxy/nighthawk-benchmark-dev:latest" + string nighthawk_benchmark_image = 2; + + // Specifies the name of the docker image containing nighthawk binaries + // eg: "envoyproxy/nighthawk-dev:latest" + string nighthawk_binary_image = 3; + + // Specifies the envoy image from which Envoy is injected. This supports + // using a commit hash or a tag to identify a specific image + // eg: "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + string envoy_image = 4; +} diff --git a/salvo/api/source.proto b/salvo/api/source.proto new file mode 100644 index 00000000..10434eb9 --- /dev/null +++ b/salvo/api/source.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package salvo; + +// Capture the location of sources needed for the benchmark +message SourceRepository { + + // Specify whether this source location is Envoy or NightHawk + enum SourceIdentity { + ENVOY = 0; + NIGHTHAWK = 1; + } + + SourceIdentity identity = 1; + + oneof source_location { + // Specify the location of the source repository on disk. If specified + // this location is used to determine the origin url, branch, and commit + // hash. If not specified, the remaining fields must be populated + // eg: "/home/user/code/envoy" + string source_path = 2; + + // Specify the remote location of the repository. + // eg: "https://github.com/envoyproxy/envoy.git" + string source_url = 3; + } + + // Specify the local working branch.This is ignored if the source + // location is specified. + string branch = 4; + + // Specify a commit hash if applicable. If not identified we will + // determine this from the source tree. We will also use this field + // to identify the corresponding NightHawk or Envoy image used for + // the benchmark + string commit_hash = 5; +} diff --git a/salvo/install_deps.sh b/salvo/install_deps.sh new file mode 100755 index 00000000..45c36eb7 --- /dev/null +++ b/salvo/install_deps.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ ${UID} -ne 0 ] +then + echo "This script needs root priviliges to install dependencies. Continuing may elicit failures from tests" + exit 0 +fi + +/usr/bin/apt update +/usr/bin/apt -y install \ + docker.io \ + python3-pytest \ + python3-docker diff --git a/salvo/salvo.py b/salvo/salvo.py new file mode 100644 index 00000000..3e4a6e69 --- /dev/null +++ b/salvo/salvo.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import os +import site +import sys + +from src.lib.job_control_loader import load_control_doc + + +LOGFORMAT = "%(asctime)s: %(process)d [ %(levelname)-5s] [%(module)-5s] %(message)s" + +log = logging.getLogger() + + +def setup_logging(loglevel=logging.DEBUG): + """Basic logging configuration """ + + logging.basicConfig(format=LOGFORMAT, level=loglevel) + + +def setup_options(): + """Parse command line arguments required for operation""" + + parser = argparse.ArgumentParser(description="Salvo Benchmark Runner") + parser.add_argument('--job', + dest='jobcontrol', + help='specify the location for the job control json document') + # FIXME: Add an option to generate a default job Control JSON/YAML + + return parser.parse_args() + + +def main(): + """Driver module for benchmark """ + + args = setup_options() + setup_logging() + + if not args.jobcontrol: + print("No job control document specified. Use \"--help\" for usage") + return 1 + + job_control = load_control_doc(args.jobcontrol) + + log.debug("Job definition:\n%s\n%s\n%s\n", '=' * 20, job_control, '=' * 20) + + # Execute the benchmark given the contents of the job control file + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/salvo/src/lib/BUILD b/salvo/src/lib/BUILD new file mode 100644 index 00000000..a575010f --- /dev/null +++ b/salvo/src/lib/BUILD @@ -0,0 +1,53 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +licenses(["notice"]) # Apache 2 + +py_library( + name = "docker_image", + data = [ + "docker_image.py", + ], +) + +py_library( + name = "docker_volume", + data = [ + "docker_volume.py", + ], +) + +py_library( + name = "job_control", + data = [ + "job_control_loader.py", + ], + visibility = ["//visibility:public"], +) + +py_library( + name = "shell", + data = [ + "cmd_exec.py", + ], +) + +py_test( + name = "test_docker_image", + srcs = ["test_docker_image.py"], + srcs_version = "PY3", + deps = [ + ":docker_image", + ":docker_volume", + "//api:schema_proto", + ], +) + +py_test( + name = "test_job_control_loader", + srcs = ["test_job_control_loader.py"], + srcs_version = "PY3", + deps = [ + ":job_control", + "//api:schema_proto", + ], +) diff --git a/salvo/src/lib/cmd_exec.py b/salvo/src/lib/cmd_exec.py new file mode 100644 index 00000000..8096e7e6 --- /dev/null +++ b/salvo/src/lib/cmd_exec.py @@ -0,0 +1,31 @@ +""" +Module to execute a command and return the output generated. Returns both +stdout and stderr in the buffer. We also convert a byte array to a string +so callers manipulate one type of object +""" +import shlex +import subprocess +import logging + +log = logging.getLogger(__name__) + + +def run_command(cmd, **kwargs): + """ + Run the specified command returning its string output to the caller + """ + output = '' + try: + log.debug(f"Executing command: {cmd} with args {kwargs}") + cmd_array = shlex.split(cmd) + output = subprocess.check_output(cmd_array, stderr=subprocess.STDOUT, **kwargs) + + if isinstance(output, bytes): + output = output.decode('utf-8').strip() + + log.debug(f"Returning output: [{output}]") + except subprocess.CalledProcessError as process_error: + log.error(f"Unable to execute {cmd}: {process_error}") + raise + + return output diff --git a/salvo/src/lib/docker_image.py b/salvo/src/lib/docker_image.py new file mode 100644 index 00000000..3d06d8b2 --- /dev/null +++ b/salvo/src/lib/docker_image.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +This module contains abstracts running a docker image +""" + +import json +import logging + +# Ref: https://docker-py.readthedocs.io/en/stable/index.html +import docker + +from google.protobuf.json_format import (Error, MessageToJson) + +from api.docker_volume_pb2 import Volume, VolumeProperties + +log = logging.getLogger(__name__) + + +class DockerImage(): + """ + This class is a wrapper to encapsulate docker operations + + It uses an available docker python module which handles the + heavy lifting for manipulating images. + """ + + def __init__(self): + self._client = docker.from_env() + + def pull_image(self, image_name): + """Pull the identified docker image""" + return self._client.images.pull(image_name) + + def list_images(self): + """List all available docker images""" + return self._client.images.list() + + def run_image(self, image_name, **kwargs): + """ + Execute the identified docker image + + The user must specify the command to run and any environment + variables required + """ + return self._client.containers.run(image_name, stdout=True, stderr=True, detach=False, **kwargs) diff --git a/salvo/src/lib/docker_volume.py b/salvo/src/lib/docker_volume.py new file mode 100644 index 00000000..56feb4ea --- /dev/null +++ b/salvo/src/lib/docker_volume.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +This module builds the volume mapping structure passed to a docker image +""" + +import json +import logging + +# Ref: https://docker-py.readthedocs.io/en/stable/index.html +import docker + +from google.protobuf.json_format import (Error, MessageToJson) + +from api.docker_volume_pb2 import Volume, VolumeProperties + +log = logging.getLogger(__name__) + +def generate_volume_config(output_dir, test_dir=None): + """ + Generates the volumes config necessary for a container to run. + The docker path is hardcoded at the moment. The output directory + is mounted read-write and the test directory if specified is mounted + read-only + """ + volume_cfg = Volume() + + # Setup the docker socket + properties = VolumeProperties() + properties.bind = '/var/run/docker.sock' + properties.mode = 'rw' + volume_cfg.volumes['/var/run/docker.sock'].CopyFrom(properties) + + # Setup the output directory + properties = VolumeProperties() + properties.bind = output_dir + properties.mode = 'rw' + volume_cfg.volumes[output_dir].CopyFrom(properties) + + # Setup the test directory + if test_dir: + properties = VolumeProperties() + properties.bind = '/usr/local/bin/benchmarks/benchmarks.runfiles/nighthawk/benchmarks/external_tests/' + properties.mode = 'ro' + volume_cfg.volumes[test_dir].CopyFrom(properties) + + volume_json = {} + try: + volume_json = json.loads(MessageToJson(volume_cfg)) + except Error as serialize_error: + log.exception(f"Could not build volume json object: {serialize_error}") + raise + + return volume_json["volumes"] diff --git a/salvo/src/lib/job_control_loader.py b/salvo/src/lib/job_control_loader.py new file mode 100644 index 00000000..49056410 --- /dev/null +++ b/salvo/src/lib/job_control_loader.py @@ -0,0 +1,77 @@ +""" +This object abstracts the loading of json strings into protobuf objects +""" +import json +import re +import logging +import yaml + +from api.control_pb2 import JobControl +from google.protobuf.json_format import (Error, Parse) + +log = logging.getLogger(__name__) + + +def _load_json_doc(filename): + """ + Load a disk file as JSON + """ + contents = None + log.debug(f"Opening JSON file {filename}") + try: + with open(filename, 'r') as json_doc: + contents = Parse(json_doc.read(), JobControl()) + except FileNotFoundError as file_not_found: + log.exception(f"Unable to load {filename}: {file_not_found}") + except Error as json_parse_error: + log.exception(f"Unable to parse JSON contents {filename}: {json_parse_error}") + + return contents + + +def _load_yaml_doc(filename): + """ + Load a disk file as YAML + """ + log.debug(f"Opening YAML file {filename}") + contents = None + try: + with open(filename, 'r') as yaml_doc: + contents = yaml.load(yaml_doc.read()) + contents = Parse(json.dumps(contents), JobControl()) + except FileNotFoundError as file_not_found: + log.exception(f"Unable to load {filename}: {file_not_found}") + except Error as yaml_parse_error: + log.exception(f"Unable to parse YAML contents {filename}: {yaml_parse_error}") + + return contents + + +def load_control_doc(filename): + """ + Return a JobControl object from the identified filename + """ + contents = None + + # Try loading the contents based on the file extension + if filename.endswith('.json'): + log.debug(f"Loading JSON file {filename}") + return _load_json_doc(filename) + elif filename.endswith('.yaml'): + log.debug(f"Loading YAML file {filename}") + return _load_yaml_doc(filename) + else: + log.debug(f"Auto-detecting contents of {filename}") + # Attempt to autodetect the contents + try: + contents = _load_json_doc(filename) + except Error: + log.info(f"Parsing {filename} as JSON failed. Trying YAML") + + if not contents: + try: + contents = _load_yaml_doc(filename) + except Error: + log.info(f"Parsing {filename} as YAML failed. The data is in an unsupported format") + + return contents diff --git a/salvo/src/lib/test_docker_image.py b/salvo/src/lib/test_docker_image.py new file mode 100644 index 00000000..a02f2f06 --- /dev/null +++ b/salvo/src/lib/test_docker_image.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +Test Docker interactions +""" + +import os +import re +import site +import pytest + +site.addsitedir("src") + +from lib.docker_image import DockerImage + + +def test_pull_image(): + """Test retrieving an image""" + + if not os.path.exists("/var/run/docker.sock"): + pytest.skip("Skipping docker test since no socket is available") + + docker_image = DockerImage() + container = docker_image.pull_image("amazonlinux:2") + assert container is not None + + +def test_run_image(): + """Test executing a command in an image""" + + if not os.path.exists("/var/run/docker.sock"): + pytest.skip("Skipping docker test since no socket is available") + + env = ['key1=val1', 'key2=val2'] + cmd = ['uname', '-r'] + image_name = 'amazonlinux:2' + + docker_image = DockerImage() + kwargs = {} + kwargs['environment'] = env + kwargs['command'] = cmd + result = docker_image.run_image(image_name, **kwargs) + + assert result is not None + assert re.match(r'[0-9\-a-z]', result.decode('utf-8')) is not None + + +def test_list_images(): + """Test listing available images""" + + if not os.path.exists("/var/run/docker.sock"): + pytest.skip("Skipping docker test since no socket is available") + + docker_image = DockerImage() + images = docker_image.list_images() + assert images != [] + + +if __name__ == '__main__': + raise SystemExit(pytest.main(['-s', '-v', __file__])) diff --git a/salvo/src/lib/test_job_control_loader.py b/salvo/src/lib/test_job_control_loader.py new file mode 100644 index 00000000..9e0bb573 --- /dev/null +++ b/salvo/src/lib/test_job_control_loader.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +""" +Test module to validate parsing of the job control document +""" +import os +import json +import site +import tempfile +import pytest + +from google.protobuf.json_format import (MessageToJson) + +site.addsitedir("src") + +from job_control_loader import load_control_doc +from api.control_pb2 import JobControl +from api.source_pb2 import SourceRepository +from api.docker_volume_pb2 import (Volume, VolumeProperties) + + +def _write_object_to_disk(pb_obj, path): + """ + Store a formatted json document to disk + """ + json_obj = MessageToJson(pb_obj, indent=2) + with open(path, 'w') as json_doc: + json_doc.write(json_obj) + + print("\n===== BEGIN ======") + print(json_obj) + print("====== END ========\n") + + +def _serialize_and_read_object(pb_obj): + """ + Serialize a protobuf object to disk and verify we can re-read it as JSON + """ + with tempfile.NamedTemporaryFile(mode='w', delete=True) as tmp: + _write_object_to_disk(pb_obj, tmp.name) + + with open(tmp.name, 'r') as json_file: + json_object = json.loads(json_file.read()) + assert json_object is not None + assert json_object != {} + + +def _validate_job_control_object(job_control): + """ + Common verification function for a job control object + """ + assert job_control is not None + + # Verify execution location + assert job_control.remote + + # Verify configured benchmark + assert job_control.scavenging_benchmark + assert not job_control.dockerized_benchmark + assert not job_control.binary_benchmark + + # Verify sources + assert job_control.source is not None or job_control.source != [] + assert len(job_control.source) == 2 + + saw_envoy = False + saw_nighthawk = False + for source in job_control.source: + if source.identity == SourceRepository.SourceIdentity.NIGHTHAWK: + assert not source.source_path + assert source.source_url == "https://github.com/envoyproxy/nighthawk.git" + assert source.branch == "master" + assert not source.commit_hash + saw_nighthawk = True + + elif source.identity == SourceRepository.SourceIdentity.ENVOY: + assert source.source_path == "/home/ubuntu/envoy" + assert not source.source_url + assert source.branch == "master" + assert source.commit_hash == "random_commit_hash_string" + saw_envoy = True + + assert saw_envoy + assert saw_nighthawk + + # Verify images + assert job_control.images is not None + assert job_control.images.reuse_nh_images + assert job_control.images.nighthawk_benchmark_image == \ + "envoyproxy/nighthawk-benchmark-dev:latest" + assert job_control.images.nighthawk_binary_image == \ + "envoyproxy/nighthawk-dev:latest" + assert job_control.images.envoy_image == \ + "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + + # Verify environment + assert job_control.environment is not None + assert job_control.environment.test_version == job_control.environment.V4ONLY + assert job_control.environment.variables is not None + assert 'TMP_DIR' in job_control.environment.variables + assert job_control.environment.output_dir is not None + assert job_control.environment.output_dir == '/home/ubuntu/nighthawk_output' + assert job_control.environment.test_dir is not None + assert job_control.environment.test_dir == '/home/ubuntu/nighthawk_tests' + + assert job_control.environment.variables['TMP_DIR'] == "/home/ubuntu/nighthawk_output" + + +def test_control_doc_parse_yaml(): + """ + Verify that we can consume a yaml formatted control document + """ + control_yaml = """ + remote: true + scavengingBenchmark: true + source: + - identity: NIGHTHAWK + source_url: "https://github.com/envoyproxy/nighthawk.git" + branch: "master" + - identity: ENVOY + source_path: "/home/ubuntu/envoy" + branch: "master" + commit_hash: "random_commit_hash_string" + images: + reuseNhImages: true + nighthawkBenchmarkImage: "envoyproxy/nighthawk-benchmark-dev:latest" + nighthawkBinaryImage: "envoyproxy/nighthawk-dev:latest" + envoyImage: "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + environment: + testVersion: V4ONLY + envoyPath: "envoy" + outputDir: "/home/ubuntu/nighthawk_output" + testDir: "/home/ubuntu/nighthawk_tests" + variables: + TMP_DIR: "/home/ubuntu/nighthawk_output" + """ + + # Write YAML contents to a temporary file that we clean up once + # the object is parsed + job_control = None + with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp: + tmp.write(control_yaml) + tmp.close() + job_control = load_control_doc(tmp.name) + os.unlink(tmp.name) + + _validate_job_control_object(job_control) + + +def test_control_doc_parse(): + """ + Verify that we can consume a JSON formatted control document + """ + + control_json = """ + { + "remote": true, + "scavengingBenchmark": true, + "source": [ + { + "identity": NIGHTHAWK, + "source_url": "https://github.com/envoyproxy/nighthawk.git", + "branch": "master" + }, + { + "identity": ENVOY, + "source_path": "/home/ubuntu/envoy", + "branch": "master", + "commit_hash": "random_commit_hash_string" + } + ], + "images": { + "reuseNhImages": true, + "nighthawkBenchmarkImage": "envoyproxy/nighthawk-benchmark-dev:latest", + "nighthawkBinaryImage": "envoyproxy/nighthawk-dev:latest", + "envoyImage": "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + }, + "environment": { + testVersion: V4ONLY, + "envoyPath": "envoy", + "outputDir": "/home/ubuntu/nighthawk_output", + "testDir": "/home/ubuntu/nighthawk_tests", + "variables": { + "TMP_DIR": "/home/ubuntu/nighthawk_output" + } + } + } + """ + + # Write JSON contents to a temporary file that we clean up once + # the object is parsed + job_control = None + with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp: + tmp.write(control_json) + tmp.close() + job_control = load_control_doc(tmp.name) + os.unlink(tmp.name) + + _validate_job_control_object(job_control) + + +def test_generate_control_doc(): + """ + Verify that we can serialize an object to a file in JSON format + """ + job_control = JobControl() + job_control.remote = True + job_control.scavenging_benchmark = True + + nighthawk_source = job_control.source.add() + nighthawk_source.identity == SourceRepository.SourceIdentity.NIGHTHAWK + nighthawk_source.source_url = "https://github.com/envoyproxy/nighthawk.git" + nighthawk_source.branch = "master" + + envoy_source = job_control.source.add() + envoy_source.identity = SourceRepository.SourceIdentity.ENVOY + envoy_source.source_path = "/home/ubuntu/envoy" + envoy_source.branch = "master" + envoy_source.commit_hash = "random_commit_hash_string" + + job_control.images.reuse_nh_images = True + job_control.images.nighthawk_benchmark_image = "envoyproxy/nighthawk-benchmark-dev:latest" + job_control.images.nighthawk_binary_image = "envoyproxy/nighthawk-dev:latest" + job_control.images.envoy_image = "envoyproxy/envoy-dev:f61b096f6a2dd3a9c74b9a9369a6ea398dbe1f0f" + + job_control.environment.variables["TMP_DIR"] = "/home/ubuntu/nighthawk_output" + job_control.environment.test_version = job_control.environment.V4ONLY + job_control.environment.envoy_path = "envoy" + job_control.environment.output_dir = '/home/ubuntu/nighthawk_output' + job_control.environment.test_dir = '/home/ubuntu/nighthawk_tests' + + # Verify that we the serialized data is json consumable + _serialize_and_read_object(job_control) + + +def _test_docker_volume_generation(): + """ + Verify construction of the volume mount map that we provide to a docker container + """ + volume_cfg = Volume() + + props = VolumeProperties() + props.bind = '/var/run/docker.sock' + props.mode = VolumeProperties.RW + volume_cfg.volumes['/var/run/docker.sock'].CopyFrom(props) + + props = VolumeProperties() + props.bind = '/home/ubuntu/nighthawk_output' + props.mode = VolumeProperties.RW + volume_cfg.volumes['/home/ubuntu/nighthawk_output'].CopyFrom(props) + + props = VolumeProperties() + props.bind = '/usr/local/bin/benchmarks/benchmarks.runfiles/nighthawk/benchmarks/external_tests/' + props.mode = VolumeProperites.RW + volume_cfg.volumes['/home/ubuntu/nighthawk_tests'].CopyFrom(props) + + # Verify that we the serialized data is json consumable + _serialize_and_read_object(volume_cfg) + + +if __name__ == '__main__': + raise SystemExit(pytest.main(['-s', '-v', __file__])) diff --git a/salvo/tools/.style.yapf b/salvo/tools/.style.yapf new file mode 100644 index 00000000..b07cb797 --- /dev/null +++ b/salvo/tools/.style.yapf @@ -0,0 +1,6 @@ +# The Google Python styles can be found here: https://github.com/google/styleguide/blob/gh-pages/pyguide.md +# TODO: Look into enforcing single vs double quote. +[style] +based_on_style=Google +indent_width=2 +column_limit=100 diff --git a/salvo/tools/format_python_tools.py b/salvo/tools/format_python_tools.py new file mode 100755 index 00000000..f4dc8cd3 --- /dev/null +++ b/salvo/tools/format_python_tools.py @@ -0,0 +1,75 @@ +import argparse +import fnmatch +import os +import sys + +from yapf.yapflib.yapf_api import FormatFile + +EXCLUDE_LIST = ['generated', 'venv', ".cache"] + + +def collectFiles(): + """Collect all Python files in the tools directory. + + Returns: A collection of python files in the tools directory excluding + any directories in the EXCLUDE_LIST constant. + """ + # TODO: Add ability to collect a specific file or files. + matches = [] + path_parts = os.getcwd().split('/') + dirname = '.' + if path_parts[-1] == 'tools': + dirname = '/'.join(path_parts[:-1]) + for root, dirnames, filenames in os.walk(dirname): + dirnames[:] = [d for d in dirnames if d not in EXCLUDE_LIST] + for filename in fnmatch.filter(filenames, '*.py'): + matches.append(os.path.join(root, filename)) + return matches + + +def validateFormat(fix=False): + """Check the format of python files in the tools directory. + + Arguments: + fix: a flag to indicate if fixes should be applied. + """ + fixes_required = False + failed_update_files = set() + successful_update_files = set() + for python_file in collectFiles(): + reformatted_source, encoding, changed = FormatFile(python_file, + style_config='.style.yapf', + in_place=fix, + print_diff=not fix) + if not fix: + fixes_required = True if changed else fixes_required + if reformatted_source: + print(reformatted_source) + continue + file_list = failed_update_files if reformatted_source else successful_update_files + file_list.add(python_file) + if fix: + displayFixResults(successful_update_files, failed_update_files) + fixes_required = len(failed_update_files) > 0 + return not fixes_required + + +def displayFixResults(successful_files, failed_files): + if successful_files: + print('Successfully fixed {} files'.format(len(successful_files))) + + if failed_files: + print('The following files failed to fix inline:') + for failed_file in failed_files: + print(' - {}'.format(failed_file)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Tool to format python files.') + parser.add_argument('action', + choices=['check', 'fix'], + default='check', + help='Fix invalid syntax in files.') + args = parser.parse_args() + is_valid = validateFormat(args.action == 'fix') + sys.exit(0 if is_valid else 1) diff --git a/salvo/tools/format_python_tools.sh b/salvo/tools/format_python_tools.sh new file mode 100755 index 00000000..4182a4b4 --- /dev/null +++ b/salvo/tools/format_python_tools.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This script runs the style and formatting checks + +set -e + +VENV_DIR="pyformat" +SCRIPTPATH=$(realpath "$(dirname $0)") +. $SCRIPTPATH/shell_utils.sh +cd "$SCRIPTPATH" + +source_venv "$VENV_DIR" +echo "Installing requirements..." +pip install -r requirements.txt + +echo "Running Python format check..." +python format_python_tools.py $1 + +echo "Running Python3 flake8 check..." +cd .. +EXCLUDE="--exclude=benchmarks/tmp/*,.cache/*,*/venv/*,tools/format_python_tools.py,tools/gen_compilation_database.py,bazel-*" + + +# Because of conflict with the automatic fix format script, we ignore: +# E111 Indentation is not a multiple of four +# E114 Indentation is not a multiple of four (comment) +# E501 Line too long (82 > 79 characters) +# E124 Closing bracket does not match visual indentation +# E125 Continuation line with same indent as next logical line +# E126 Continuation line over-indented for hanging indent + +# We ignore false positives because of what look like pytest peculiarities +# F401 Module imported but unused +# F811 Redefinition of unused name from line n +flake8 . ${EXCLUDE} --ignore=E114,E111,E501,F401,F811,E124,E125,E126,D --count --show-source --statistics +# D = Doc comment related checks (We check both p257 AND google conventions). +flake8 . ${EXCLUDE} --docstring-convention pep257 --select=D --count --show-source --statistics +flake8 . ${EXCLUDE} --docstring-convention google --select=D --count --show-source --statistics + diff --git a/salvo/tools/requirements.txt b/salvo/tools/requirements.txt new file mode 100644 index 00000000..f1d27c32 --- /dev/null +++ b/salvo/tools/requirements.txt @@ -0,0 +1,3 @@ +flake8==3.8.3 +yapf==0.30.0 +flake8-docstrings==1.5.0 diff --git a/salvo/tools/shell_utils.sh b/salvo/tools/shell_utils.sh new file mode 100755 index 00000000..8d71a7ab --- /dev/null +++ b/salvo/tools/shell_utils.sh @@ -0,0 +1,31 @@ +# This script contains functions to create a virtual environment +# with the dependencies necessary to execute style and formatting +# checks. This is sourced from format_python_tools.sh + +# Active the environment if it exists, create it if it does not +source_venv() { + VENV_DIR=$1 + if [[ "${VIRTUAL_ENV}" == "" ]]; then + if [[ ! -d "${VENV_DIR}"/venv ]]; then + virtualenv "${VENV_DIR}"/venv --python=python3 + fi + source "${VENV_DIR}"/venv/bin/activate + else + echo "Found existing virtualenv" + fi +} + +# Install python dependencies into the virtual environment +python_venv() { + SCRIPT_DIR=$(realpath "$(dirname "$0")") + + BUILD_DIR=build_tools + PY_NAME="$1" + VENV_DIR="${BUILD_DIR}/${PY_NAME}" + + source_venv "${VENV_DIR}" + pip install -r "${SCRIPT_DIR}"/requirements.txt + + shift + python3 "${SCRIPT_DIR}/${PY_NAME}.py" $* +}