diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 55291c4b6..3897f2d67 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -24,6 +24,13 @@ This document will walk you through on what's needed to start contributing code - **Pyenv** : Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation). + Install the following modules to continue with the next steps: + ``` + sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev git + ``` + - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters. JDK version 17 is used to build the current version of OpenSearch. Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites). Note that the `javadoc` executable should be available in the JDK installation. An earlier version of the JDK can be used, but not all the integration tests will pass. @@ -38,7 +45,9 @@ This document will walk you through on what's needed to start contributing code ### Setup -To develop OSB properly, it is recommended that you fork the official OpenSearch Benchmark repository. +To develop OSB properly, it is recommended that you fork the official OpenSearch Benchmark repository. + +For those working on WSL2, it is recommended to clone the repository and set up the working environment within the Linux subsystem. Refer to the guide for setting up WSL2 on [Visual Studio Code](https://code.visualstudio.com/docs/remote/wsl) or [PyCharm](https://www.jetbrains.com/help/pycharm/using-wsl-as-a-remote-interpreter.html#create-wsl-interpreter). After you git cloned the forked copy of OpenSearch Benchmark, use the following command-line instructions to set up OpenSearch Benchmark for development: ``` @@ -74,6 +83,74 @@ This is typically created in PyCharm IDE by visiting the `Python Interpreter`, s ` In order to run tests within the PyCharm IDE, ensure the `Python Integrated Tools` / `Testing` / `Default Test Runner` is set to `pytest`. +## Running Workloads + +### Installation + +Download the latest release of OpenSearch from https://opensearch.org/downloads.html. If you are using WSL, make sure to download it into your `/home/` directory instead of `/mnt/c`. +``` +wget https://artifacts.opensearch.org/releases/bundle/opensearch//opensearch--linux-x64.tar.gz +tar -xf opensearch-x.x.x-linux-x64.tar.gz +cd opensearch-x.x.x +``` +NOTE: Have Docker running in the background for the next steps. Refer to the installation instructions [here](https://docs.docker.com/compose/install/). + +### Setup + +Add the following settings to the `opensearch.yml` file under the config directory +``` +vim config/opensearch.yml +``` +``` +# +discovery.type: single-node +plugins.security.disabled: true +# +``` +Run the opensearch-tar-install.sh script to install and setup a cluster for our use. +``` +bash opensearch-tar-install.sh +``` +Check the output of `curl.exe "http://localhost:9200/_cluster/health?pretty"`. Output should be similar to this: +``` +{ + "cluster_name" : "", + "status" : "green", + "timed_out" : false, + "number_of_nodes" : 1, + "number_of_data_nodes" : 1, + "discovered_master" : true, + "discovered_cluster_manager" : true, + "active_primary_shards" : 3, + "active_shards" : 3, + "relocating_shards" : 0, + "initializing_shards" : 0, + "unassigned_shards" : 0, + "delayed_unassigned_shards" : 0, + "number_of_pending_tasks" : 0, + "number_of_in_flight_fetch" : 0, + "task_max_waiting_in_queue_millis" : 0, + "active_shards_percent_as_number" : 100.0 +} +``` +Now, you have a local cluster running! You can connect to this and run the workload for the next step. + +### Running the workload + +Here's a sample executation of the geonames benchmark which can be found from the [workloads](https://github.com/opensearch-project/opensearch-benchmark-workloads) repo. +``` +opensearch-benchmark execute-test --pipeline=benchmark-only --workload=geonames --target-host=127.0.0.1:9200 --test-mode --workload-params '{"number_of_shards":"1","number_of_replicas":"0"}' +``` + +And we're done! You should be seeing the performance metrics soon enough! + +### Debugging + +**If you are not seeing any results, it should be an indicator that there is an issue with your cluster setup or the way the manager is accessing it**. Use the command below to view the logs. +``` +tail -f ~/.bencmark/logs/bechmark.log +``` + ## Executing tests Once setup is complete, you may run the unit and integration tests. @@ -87,10 +164,10 @@ make test ### Integration Tests -Integration tests can be run on the following operating systems: +Integration tests are expected to run for approximately **20-30 mins** and can be run on the following operating systems: * RedHat * CentOS - * Ubuntu + * Ubuntu (and WSL) * Amazon Linux 2 * MacOS @@ -100,7 +177,6 @@ Invoke integration tests by running the following command within the root direct make it ``` -Integration tests are expected to run for approximately 20-30 mins. ## Submitting your changes for a pull request diff --git a/it/generate_test.py b/it/generate_test.py deleted file mode 100644 index d7fca6c85..000000000 --- a/it/generate_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os - -import it - - -@it.benchmark_in_mem -def test_workload_info_with_test_procedure(cfg, tmp_path): - cwd = os.path.dirname(__file__) - chart_spec_path = os.path.join(cwd, "resources", "sample-test-execution-config.json") - output_path = os.path.join(tmp_path, "nightly-charts.ndjson") - assert it.osbenchmark(cfg, f"generate charts " - f"--chart-spec-path={chart_spec_path} " - f"--chart-type=time-series " - f"--output-path={output_path}") == 0 diff --git a/osbenchmark/__init__.py b/osbenchmark/__init__.py index a4163ec97..9dc0d4493 100644 --- a/osbenchmark/__init__.py +++ b/osbenchmark/__init__.py @@ -26,10 +26,6 @@ import sys import urllib -import pkg_resources - -__version__ = pkg_resources.require("opensearch-benchmark")[0].version - # Allow an alternative program name be set in case Benchmark is invoked a wrapper script PROGRAM_NAME = os.getenv("BENCHMARK_ALTERNATIVE_BINARY_NAME", os.path.basename(sys.argv[0])) diff --git a/osbenchmark/benchmark.py b/osbenchmark/benchmark.py index 31f3f1934..61a7d8590 100644 --- a/osbenchmark/benchmark.py +++ b/osbenchmark/benchmark.py @@ -36,7 +36,7 @@ from osbenchmark import PROGRAM_NAME, BANNER, FORUM_LINK, SKULL, check_python_version, doc_link, telemetry from osbenchmark import version, actor, config, paths, \ test_execution_orchestrator, results_publisher, \ - metrics, workload, chart_generator, exceptions, log + metrics, workload, exceptions, log from osbenchmark.builder import provision_config, builder from osbenchmark.workload_generator import workload_generator from osbenchmark.utils import io, convert, process, console, net, opts, versions @@ -188,30 +188,6 @@ def add_workload_source(subparser): help="Map of index name and integer doc count to extract. Ensure that index name also exists in --indices parameter. " + "To specify several indices and doc counts, use format: : : ...") - generate_parser = subparsers.add_parser("generate", help="Generate artifacts") - generate_parser.add_argument( - "artifact", - metavar="artifact", - help="The artifact to create. Possible values are: charts", - choices=["charts"]) - # We allow to either have a chart-spec-path *or* define a chart-spec on the fly - # with workload, test_procedure and provision_config_instance. Convincing - # argparse to validate that everything is correct *might* be doable but it is - # simpler to just do this manually. - generate_parser.add_argument( - "--chart-spec-path", - required=True, - help="Path to a JSON file(s) containing all combinations of charts to generate. Wildcard patterns can be used to specify " - "multiple files.") - generate_parser.add_argument( - "--chart-type", - help="Chart type to generate (default: time-series).", - choices=["time-series", "bar"], - default="time-series") - generate_parser.add_argument( - "--output-path", - help="Output file name (default: stdout).", - default=None) compare_parser = subparsers.add_parser("compare", help="Compare two test_executions") compare_parser.add_argument( @@ -600,7 +576,7 @@ def add_workload_source(subparser): default=False) for p in [list_parser, test_execution_parser, compare_parser, download_parser, install_parser, - start_parser, stop_parser, info_parser, generate_parser, create_workload_parser]: + start_parser, stop_parser, info_parser, create_workload_parser]: # This option is needed to support a separate configuration for the integration tests on the same machine p.add_argument( "--configuration-name", @@ -742,11 +718,6 @@ def with_actor_system(runnable, cfg): console.warn("Could not terminate all internal processes within timeout. Please check and force-terminate " "all Benchmark processes.") - -def generate(cfg): - chart_generator.generate(cfg) - - def configure_telemetry_params(args, cfg): cfg.add(config.Scope.applicationOverride, "telemetry", "devices", opts.csv_to_list(args.telemetry)) cfg.add(config.Scope.applicationOverride, "telemetry", "params", opts.to_dict(args.telemetry_params)) @@ -906,11 +877,6 @@ def dispatch_sub_command(arg_parser, args, cfg): configure_results_publishing_params(args, cfg) execute_test(cfg, args.kill_running_processes) - elif sub_command == "generate": - cfg.add(config.Scope.applicationOverride, "generator", "chart.spec.path", args.chart_spec_path) - cfg.add(config.Scope.applicationOverride, "generator", "chart.type", args.chart_type) - cfg.add(config.Scope.applicationOverride, "generator", "output.path", args.output_path) - generate(cfg) elif sub_command == "create-workload": cfg.add(config.Scope.applicationOverride, "generator", "indices", args.indices) cfg.add(config.Scope.applicationOverride, "generator", "number_of_docs", args.number_of_docs) diff --git a/osbenchmark/chart_generator.py b/osbenchmark/chart_generator.py deleted file mode 100644 index a191abfae..000000000 --- a/osbenchmark/chart_generator.py +++ /dev/null @@ -1,1954 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import glob -import json -import logging -import uuid - -from osbenchmark import workload, config, exceptions -from osbenchmark.utils import io, console - -color_scheme_rgba = [ - # #00BFB3 - "rgba(0,191,179,1)", - # #00A9E0 - "rgba(0,169,224,1)", - # #F04E98 - "rgba(240,78,152,1)", - # #FFCD00 - "rgba(255,205,0,1)", - # #0076A8 - "rgba(0,118,168,1)", - # #93C90E - "rgba(147,201,14,1)", - # #646464 - "rgba(100,100,100,1)", -] - - -def index_label(test_execution_config): - if test_execution_config.label: - return test_execution_config.label - - label = "%s-%s" % (test_execution_config.test_procedure, test_execution_config.provision_config_instance) - if test_execution_config.plugins: - label += "-%s" % test_execution_config.plugins.replace(":", "-").replace(",", "+") - if test_execution_config.node_count > 1: - label += " (%d nodes)" % test_execution_config.node_count - return label - - -class BarCharts: - UI_STATE_JSON = json.dumps( - { - "vis": { - "colors": dict( - zip(["bare-oss", "bare-basic", "bare-trial-security", "docker-basic", "ear-basic"], color_scheme_rgba)) - } - }) - - @staticmethod - # flavor's unused but we need the same signature used by the corresponding method in TimeSeriesCharts - def format_title(environment, workload_name, flavor=None, os_license=None, suffix=None): - title = f"{environment}-{workload_name}" - - if suffix: - title += f"-{suffix}" - - return title - - @staticmethod - def filter_string(environment, test_ex_config): - if test_ex_config.name: - return f"environment:\"{environment}\" AND active:true AND user-tags.name:\"{test_ex_config.name}\"" - else: - return f"environment:\"{environment}\" AND active:true AND workload:\"{test_ex_config.workload}\""\ - f" AND test_procedure:\"{test_ex_config.test_procedure}\""\ - f" AND provision_config_instance:\"{test_ex_config.provision_config_instance}\""\ - f" AND node-count:{test_ex_config.node_count}" - - @staticmethod - def gc(title, environment, test_execution_config): - vis_state = { - "title": title, - "type": "histogram", - "params": { - "addLegend": True, - "addTimeMarker": False, - "addTooltip": True, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": True, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "filters" - }, - "type": "category" - } - ], - "defaultYExtents": False, - "drawLinesBetweenPoints": True, - "grid": { - "categoryLines": False, - "style": { - "color": "#eee" - } - }, - "interpolate": "linear", - "legendPosition": "right", - "radiusRatio": 9, - "scale": "linear", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "Total GC Duration [ms]" - }, - "drawLinesBetweenPoints": True, - "mode": "normal", - "show": "True", - "showCircles": True, - "type": "histogram", - "valueAxis": "ValueAxis-1" - } - ], - "setYExtents": False, - "showCircles": True, - "times": [], - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": False, - "rotate": 0, - "show": True, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "Total GC Duration [ms]" - }, - "type": "value" - } - ] - }, - "aggs": [ - { - "id": "1", - "enabled": True, - "type": "median", - "schema": "metric", - "params": { - "field": "value.single", - "percents": [ - 50 - ], - "customLabel": "Total GC Duration [ms]" - } - }, - { - "id": "2", - "enabled": True, - "type": "filters", - "schema": "segment", - "params": { - "filters": [ - { - "input": { - "query": { - "query_string": { - "query": "name:young_gc_time", - "analyze_wildcard": True - } - } - }, - "label": "Young GC" - }, - { - "input": { - "query": { - "query_string": { - "query": "name:old_gc_time", - "analyze_wildcard": True - } - } - }, - "label": "Old GC" - } - ] - } - }, - { - "id": "3", - "enabled": True, - "type": "terms", - "schema": "split", - "params": { - "field": "distribution-version", - "size": 10, - "order": "asc", - "orderBy": "_term", - "row": False - } - }, - { - "id": "4", - "enabled": True, - "type": "terms", - "schema": "group", - "params": { - "field": "user-tags.setup", - "size": 5, - "order": "desc", - "orderBy": "_term" - } - } - ], - "listeners": {} - } - - search_source = { - "index": "benchmark-results-*", - "query": { - "query_string": { - "query": BarCharts.filter_string(environment, test_execution_config), - "analyze_wildcard": True - } - }, - "filter": [] - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": BarCharts.UI_STATE_JSON, - "description": "gc", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps(search_source) - } - } - } - - @staticmethod - def io(title, environment, test_execution_config): - vis_state = { - "title": title, - "type": "histogram", - "params": { - "addLegend": True, - "addTimeMarker": False, - "addTooltip": True, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": True, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "filters" - }, - "type": "category" - } - ], - "defaultYExtents": False, - "drawLinesBetweenPoints": True, - "grid": { - "categoryLines": False, - "style": { - "color": "#eee" - } - }, - "interpolate": "linear", - "legendPosition": "right", - "radiusRatio": 9, - "scale": "linear", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "[Bytes]" - }, - "drawLinesBetweenPoints": True, - "mode": "normal", - "show": "True", - "showCircles": True, - "type": "histogram", - "valueAxis": "ValueAxis-1" - } - ], - "setYExtents": False, - "showCircles": True, - "times": [], - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": False, - "rotate": 0, - "show": True, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "[Bytes]" - }, - "type": "value" - } - ] - }, - "aggs": [ - { - "id": "1", - "enabled": True, - "type": "sum", - "schema": "metric", - "params": { - "field": "value.single", - "customLabel": "[Bytes]" - } - }, - { - "id": "2", - "enabled": True, - "type": "filters", - "schema": "segment", - "params": { - "filters": [ - { - "input": { - "query": { - "query_string": { - "analyze_wildcard": True, - "query": "name:index_size" - } - } - }, - "label": "Index size" - }, - { - "input": { - "query": { - "query_string": { - "analyze_wildcard": True, - "query": "name:bytes_written" - } - } - }, - "label": "Bytes written" - } - ] - } - }, - { - "id": "3", - "enabled": True, - "type": "terms", - "schema": "split", - "params": { - "field": "distribution-version", - "size": 10, - "order": "asc", - "orderBy": "_term", - "row": False - } - }, - { - "id": "4", - "enabled": True, - "type": "terms", - "schema": "group", - "params": { - "field": "user-tags.setup", - "size": 5, - "order": "desc", - "orderBy": "_term" - } - } - ], - "listeners": {} - } - - search_source = { - "index": "benchmark-results-*", - "query": { - "query_string": { - "query": BarCharts.filter_string(environment, test_execution_config), - "analyze_wildcard": True - } - }, - "filter": [] - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": BarCharts.UI_STATE_JSON, - "description": "io", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps(search_source) - } - } - } - - @staticmethod - def segment_memory(title, environment, test_execution_config): - # don't generate segment memory charts for releases - return None - - @staticmethod - def query(environment, test_execution_config, q): - metric = "service_time" - title = BarCharts.format_title( - environment, - test_execution_config.workload, - suffix="%s-%s-p99-%s" % (test_execution_config.label, - q, - metric)) - label = "Query Service Time [ms]" - - vis_state = { - "title": title, - "type": "histogram", - "params": { - "addLegend": True, - "addTimeMarker": False, - "addTooltip": True, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": True, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "distribution-version: Ascending" - }, - "type": "category" - } - ], - "defaultYExtents": False, - "drawLinesBetweenPoints": True, - "grid": { - "categoryLines": False, - "style": { - "color": "#eee" - } - }, - "interpolate": "linear", - "legendPosition": "right", - "radiusRatio": 9, - "scale": "linear", - "seriesParams": [ - { - "data": { - "id": "1", - "label": label - }, - "drawLinesBetweenPoints": True, - "mode": "normal", - "show": "True", - "showCircles": True, - "type": "histogram", - "valueAxis": "ValueAxis-1" - } - ], - "setYExtents": False, - "showCircles": True, - "times": [], - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": False, - "rotate": 0, - "show": True, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": label - }, - "type": "value" - } - ] - }, - "aggs": [ - { - "id": "1", - "enabled": True, - "type": "median", - "schema": "metric", - "params": { - "field": "value.99_0", - "percents": [ - 50 - ], - "customLabel": label - } - }, - { - "id": "2", - "enabled": True, - "type": "terms", - "schema": "segment", - "params": { - "field": "distribution-version", - "size": 10, - "order": "asc", - "orderBy": "_term" - } - }, - { - "id": "3", - "enabled": True, - "type": "terms", - "schema": "group", - "params": { - "field": "user-tags.setup", - "size": 10, - "order": "desc", - "orderBy": "_term" - } - } - ], - "listeners": {} - } - - search_source = { - "index": "benchmark-results-*", - "query": { - "query_string": { - "query": "name:\"%s\" AND task:\"%s\" AND %s" % ( - metric, - q, - BarCharts.filter_string( - environment, - test_execution_config)), - "analyze_wildcard": True - } - }, - "filter": [] - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": BarCharts.UI_STATE_JSON, - "description": "query", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps(search_source) - } - } - } - - @staticmethod - def index(environment, test_execution_configs, title): - filters = [] - for test_execution_config in test_execution_configs: - label = index_label(test_execution_config) - # the assumption is that we only have one bulk task - for bulk_task in test_execution_config.bulk_tasks: - filters.append({ - "input": { - "query": { - "query_string": { - "analyze_wildcard": True, - "query": "task:\"%s\" AND %s" % (bulk_task, BarCharts.filter_string(environment, test_execution_config)) - } - } - }, - "label": label - }) - - vis_state = { - "aggs": [ - { - "enabled": True, - "id": "1", - "params": { - "customLabel": "Median Indexing Throughput [docs/s]", - "field": "value.median", - "percents": [ - 50 - ] - }, - "schema": "metric", - "type": "median" - }, - { - "enabled": True, - "id": "2", - "params": { - "field": "distribution-version", - "order": "asc", - "orderBy": "_term", - "size": 10 - }, - "schema": "segment", - "type": "terms" - }, - { - "enabled": True, - "id": "3", - "params": { - "field": "user-tags.setup", - "order": "desc", - "orderBy": "_term", - "size": 10 - }, - "schema": "group", - "type": "terms" - }, - { - "enabled": True, - "id": "4", - "params": { - "filters": filters - }, - "schema": "split", - "type": "filters" - } - ], - "listeners": {}, - "params": { - "addLegend": True, - "addTimeMarker": False, - "addTooltip": True, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": True, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "distribution-version: Ascending" - }, - "type": "category" - } - ], - "defaultYExtents": False, - "drawLinesBetweenPoints": True, - "grid": { - "categoryLines": False, - "style": { - "color": "#eee" - } - }, - "interpolate": "linear", - "legendPosition": "right", - "radiusRatio": 9, - "scale": "linear", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "Median Indexing Throughput [docs/s]" - }, - "drawLinesBetweenPoints": True, - "mode": "normal", - "show": "True", - "showCircles": True, - "type": "histogram", - "valueAxis": "ValueAxis-1" - } - ], - "setYExtents": False, - "showCircles": True, - "times": [], - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": False, - "rotate": 0, - "show": True, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": True, - "style": {}, - "title": { - "text": "Median Indexing Throughput [docs/s]" - }, - "type": "value" - } - ], - "row": True - }, - "title": title, - "type": "histogram" - } - - search_source = { - "index": "benchmark-results-*", - "query": { - "query_string": { - "analyze_wildcard": True, - "query": "environment:\"%s\" AND active:true AND name:\"throughput\"" % environment - } - }, - "filter": [] - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": BarCharts.UI_STATE_JSON, - "description": "index", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps(search_source) - } - } - } - - -class TimeSeriesCharts: - @staticmethod - def format_title(environment, workload_name, flavor=None, os_license=None, suffix=None): - if flavor: - title = [environment, flavor, str(workload_name)] - elif os_license: - title = [environment, os_license, str(workload_name)] - elif flavor and os_license: - raise exceptions.BenchmarkAssertionError( - f"Specify either flavor [{flavor}] or license [{os_license}] but not both") - else: - title = [environment, str(workload_name)] - if suffix: - title.append(suffix) - - return "-".join(title) - - @staticmethod - def filter_string(environment, test_ex_config): - nightly_extra_filter = "" - if test_ex_config.os_license: - # Time series charts need to support different licenses and produce customized titles. - nightly_extra_filter = f" AND user-tags.license:\"{test_ex_config.os_license}\"" - if test_ex_config.name: - return f"environment:\"{environment}\" AND active:true AND user-tags.name:\"{test_ex_config.name}\"{nightly_extra_filter}" - else: - return f"environment:\"{environment}\" AND active:true AND workload:\"{test_ex_config.workload}\""\ - f" AND test_procedure:\"{test_ex_config.test_procedure}\""\ - f" AND provision_config_instance:\"{test_ex_config.provision_config_instance}\""\ - f" AND node-count:{test_ex_config.node_count}" - - @staticmethod - def gc(title, environment, test_ex_config): - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "number", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.single" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": "", - "split_filters": [ - { - "filter": "young_gc_time", - "label": "Young Gen GC time", - "color": "rgba(0,191,179,1)", - "id": str(uuid.uuid4()) - }, - { - "filter": "old_gc_time", - "label": "Old Gen GC time", - "color": "rgba(254,209,10,1)", - "id": str(uuid.uuid4()) - } - ], - "label": "GC Times", - "value_template": "{{value}} ms", - "steps": 0 - } - ], - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": TimeSeriesCharts.filter_string(environment, test_ex_config), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_ex_config.workload}\") "\ - f"AND ((NOT _exists_:chart) OR chart:gc) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "axis_min": "0" - }, - "aggs": [], - "listeners": {} - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "gc", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - @staticmethod - def merge_time(title, environment, test_execution_config): - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "number", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.single" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": "", - "split_filters": [ - { - "filter": "merge_time", - "label": "Cumulative merge time", - "color": "rgba(0,191,179,1)", - "id": str(uuid.uuid4()) - }, - { - "filter": "merge_throttle_time", - "label": "Cumulative merge throttle time", - "color": "rgba(254,209,10,1)", - "id": str(uuid.uuid4()) - } - ], - "label": "Merge Times", - "value_template": "{{value}} ms", - "steps": 0 - } - ], - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": TimeSeriesCharts.filter_string(environment, test_execution_config), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_execution_config.workload}\") " - f"AND ((NOT _exists_:chart) OR chart:merge_times) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "axis_min": "0" - }, - "aggs": [], - "listeners": {} - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "merge_times", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - @staticmethod - def merge_count(title, environment, test_execution_config): - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "number", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.single" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": "", - "split_filters": [ - { - "filter": "merge_count", - "label": "Cumulative merge count", - "color": "rgba(0,191,179,1)", - "id": str(uuid.uuid4()) - } - ], - "label": "Merge Count", - "value_template": "{{value}}", - "steps": 0 - } - ], - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": TimeSeriesCharts.filter_string(environment, test_execution_config), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_execution_config.workload}\") " - f"AND ((NOT _exists_:chart) OR chart:merge_count) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "axis_min": "0" - }, - "aggs": [], - "listeners": {} - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "merge_count", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - @staticmethod - def io(title, environment, test_ex_config): - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "bytes", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "sum", - "field": "value.single" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": "", - "split_filters": [ - { - "filter": "name:index_size", - "label": "Index Size", - "color": "rgba(0,191,179,1)", - "id": str(uuid.uuid4()) - }, - { - "filter": "name:bytes_written", - "label": "Written", - "color": "rgba(254,209,10,1)", - "id": str(uuid.uuid4()) - } - ], - "label": "Disk IO", - "value_template": "{{value}}", - "steps": 0 - } - ], - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": TimeSeriesCharts.filter_string(environment, test_ex_config), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_ex_config.workload}\") "\ - f"AND ((NOT _exists_:chart) OR chart:io) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "axis_min": "0" - }, - "aggs": [], - "listeners": {} - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "io", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - @staticmethod - def segment_memory(title, environment, test_ex_config): - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "bytes", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.single" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": f"environment:{environment} AND workload:\"{test_ex_config.workload}\"", - "split_filters": [ - { - "filter": "memory_segments", - "label": "Segments", - "color": color_scheme_rgba[0], - "id": str(uuid.uuid4()) - }, - { - "filter": "memory_doc_values", - "label": "Doc Values", - "color": color_scheme_rgba[1], - "id": str(uuid.uuid4()) - }, - { - "filter": "memory_terms", - "label": "Terms", - "color": color_scheme_rgba[2], - "id": str(uuid.uuid4()) - }, - { - "filter": "memory_norms", - "label": "Norms", - "color": color_scheme_rgba[3], - "id": str(uuid.uuid4()) - }, - { - "filter": "memory_points", - "label": "Points", - "color": color_scheme_rgba[4], - "id": str(uuid.uuid4()) - }, - { - "filter": "memory_stored_fields", - "label": "Stored Fields", - "color": color_scheme_rgba[5], - "id": str(uuid.uuid4()) - } - ], - "label": "Segment Memory", - "value_template": "{{value}}", - "steps": 0 - } - ], - "show_legend": 1, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": TimeSeriesCharts.filter_string(environment, test_ex_config), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_ex_config.workload}\") " - f"AND ((NOT _exists_:chart) OR chart:segment_memory) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "show_grid": 1, - "drop_last_bucket": 0, - "axis_min": "0" - }, - "aggs": [] - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "segment_memory", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"*\",\"language\":\"lucene\"},\"filter\":[]}" - } - } - } - - @staticmethod - def query(environment, test_ex_config, q): - metric = "latency" - title = TimeSeriesCharts.format_title(environment, test_ex_config.workload, os_license=test_ex_config.os_license, - suffix="%s-%s-%s" % (test_ex_config.label, q, metric)) - - vis_state = { - "title": title, - "type": "metrics", - "params": { - "id": str(uuid.uuid4()), - "type": "timeseries", - "series": [ - { - "id": str(uuid.uuid4()), - "color": color_scheme_rgba[0], - "split_mode": "everything", - "label": "50th percentile", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.50_0" - } - ], - "seperate_axis": 0, - "axis_position": "right", - "formatter": "number", - "chart_type": "line", - "line_width": 1, - "point_size": 1, - "fill": "0.6", - "stacked": "none", - "split_color_mode": "gradient", - "series_drop_last_bucket": 0, - "value_template": "{{value}} ms", - }, - { - "id": str(uuid.uuid4()), - "color": color_scheme_rgba[1], - "split_mode": "everything", - "label": "90th percentile", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.90_0" - } - ], - "seperate_axis": 0, - "axis_position": "right", - "formatter": "number", - "chart_type": "line", - "line_width": 1, - "point_size": 1, - "fill": "0.4", - "stacked": "none", - "split_color_mode": "gradient", - "series_drop_last_bucket": 0, - "value_template": "{{value}} ms", - }, - { - "id": str(uuid.uuid4()), - "color": color_scheme_rgba[2], - "split_mode": "everything", - "label": "99th percentile", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.99_0" - } - ], - "seperate_axis": 0, - "axis_position": "right", - "formatter": "number", - "chart_type": "line", - "line_width": 1, - "point_size": 1, - "fill": "0.2", - "stacked": "none", - "split_color_mode": "gradient", - "series_drop_last_bucket": 0, - "value_template": "{{value}} ms", - }, - { - "id": str(uuid.uuid4()), - "color": color_scheme_rgba[3], - "split_mode": "everything", - "label": "100th percentile", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.100_0" - } - ], - "seperate_axis": 0, - "axis_position": "right", - "formatter": "number", - "chart_type": "line", - "line_width": 1, - "point_size": 1, - "fill": "0.1", - "stacked": "none", - "split_color_mode": "gradient", - "series_drop_last_bucket": 0, - "value_template": "{{value}} ms", - } - ], - "time_field": "test-execution-timestamp", - "index_pattern": "benchmark-results-*", - "interval": "1d", - "axis_position": "left", - "axis_formatter": "number", - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "background_color_rules": [ - { - "id": str(uuid.uuid4()) - } - ], - "filter": "task:\"%s\" AND name:\"%s\" AND %s" % (q, metric, TimeSeriesCharts.filter_string( - environment, test_ex_config)), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{test_ex_config.workload}\") " - f"AND ((NOT _exists_:chart) OR chart:query) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ] - }, - "aggs": [], - "listeners": {} - } - - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "query", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - @staticmethod - def index(environment, test_execution_configs, title): - filters = [] - # any test_execution_config will do - they all belong to the same workload - t = test_execution_configs[0].workload - for idx, test_execution_config in enumerate(test_execution_configs): - label = index_label(test_execution_config) - for bulk_task in test_execution_config.bulk_tasks: - filters.append( - { - "filter": "task:\"%s\" AND %s" % (bulk_task, TimeSeriesCharts.filter_string(environment, test_execution_config)), - "label": label, - "color": color_scheme_rgba[idx % len(color_scheme_rgba)], - "id": str(uuid.uuid4()) - } - ) - - vis_state = { - "title": title, - "type": "metrics", - "params": { - "axis_formatter": "number", - "axis_position": "left", - "id": str(uuid.uuid4()), - "index_pattern": "benchmark-results-*", - "interval": "1d", - "series": [ - { - "axis_position": "left", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "number", - "id": str(uuid.uuid4()), - "line_width": "1", - "metrics": [ - { - "id": str(uuid.uuid4()), - "type": "avg", - "field": "value.median" - } - ], - "point_size": "3", - "seperate_axis": 1, - "split_mode": "filters", - "stacked": "none", - "filter": "environment:\"%s\" AND workload:\"%s\"" % (environment, t), - "split_filters": filters, - "label": "Indexing Throughput", - "value_template": "{{value}} docs/s", - "steps": 0 - } - ], - "show_legend": 1, - "show_grid": 1, - "drop_last_bucket": 0, - "time_field": "test-execution-timestamp", - "type": "timeseries", - "filter": "environment:\"%s\" AND workload:\"%s\" AND name:\"throughput\" AND active:true" % (environment, t), - "annotations": [ - { - "fields": "message", - "template": "{{message}}", - "index_pattern": "benchmark-annotations", - "query_string": f"((NOT _exists_:workload) OR workload:\"{t}\") " - f"AND ((NOT _exists_:chart) OR chart:indexing) " - f"AND ((NOT _exists_:chart-name) OR chart-name:\"{title}\") AND environment:\"{environment}\"", - "id": str(uuid.uuid4()), - "color": "rgba(102,102,102,1)", - "time_field": "test-execution-timestamp", - "icon": "fa-tag", - "ignore_panel_filters": 1 - } - ], - "axis_min": "0" - }, - "aggs": [], - "listeners": {} - } - return { - "id": str(uuid.uuid4()), - "type": "visualization", - "attributes": { - "title": title, - "visState": json.dumps(vis_state), - "uiStateJSON": "{}", - "description": "index", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":\"*\",\"filter\":[]}" - } - } - } - - -class TestExecutionConfigWorkload: - def __init__(self, cfg, repository, name=None): - self.repository = repository - self.cached_workload = self.load_workload(cfg, name=name) - - def load_workload(self, cfg, name=None, params=None, excluded_tasks=None): - if not params: - params = {} - # required in case a previous workload using a different repository has specified the revision - if cfg.opts("workload", "repository.name", mandatory=False) != self.repository: - cfg.add(config.Scope.applicationOverride, "workload", "repository.revision", None) - # hack to make this work with multiple workloads (Benchmark core is usually not meant to be used this way) - if name: - cfg.add(config.Scope.applicationOverride, "workload", "repository.name", self.repository) - cfg.add(config.Scope.applicationOverride, "workload", "workload.name", name) - # another hack to ensure any workload-params in the test_execution config are used by Benchmark's workload loader - cfg.add(config.Scope.applicationOverride, "workload", "params", params) - if excluded_tasks: - cfg.add(config.Scope.application, "workload", "exclude.tasks", excluded_tasks) - return workload.load_workload(cfg) - - def get_workload(self, cfg, name=None, params=None, excluded_tasks=None): - if params or excluded_tasks: - return self.load_workload(cfg, name, params, excluded_tasks) - # if no params specified, return the initially cached, (non-parametrized) workload - return self.cached_workload - - -def generate_index_ops(chart_type, test_execution_configs, environment, logger): - idx_test_execution_configs = list(filter(lambda c: "indexing" in c.charts, test_execution_configs)) - for test_execution_conf in idx_test_execution_configs: - logger.debug("Gen index visualization for test_execution config with name:[%s] / label:[%s] / flavor: [%s] / license: [%s]", - test_execution_conf.name, - test_execution_conf.label, - test_execution_conf.flavor, - test_execution_conf.os_license) - charts = [] - - if idx_test_execution_configs: - title = chart_type.format_title( - environment, - test_execution_configs[0].workload, - flavor=test_execution_configs[0].flavor, - suffix="indexing-throughput") - charts = [chart_type.index(environment, idx_test_execution_configs, title)] - return charts - - -def generate_queries(chart_type, test_execution_configs, environment): - # output JSON structures - structures = [] - - for test_execution_config in test_execution_configs: - if "query" in test_execution_config.charts: - for q in test_execution_config.throttled_tasks: - structures.append(chart_type.query(environment, test_execution_config, q)) - return structures - - -def generate_io(chart_type, test_execution_configs, environment): - # output JSON structures - structures = [] - for test_execution_config in test_execution_configs: - if "io" in test_execution_config.charts: - title = chart_type.format_title(environment, test_execution_config.workload, os_license=test_execution_config.os_license, - suffix="%s-io" % test_execution_config.label) - structures.append(chart_type.io(title, environment, test_execution_config)) - - return structures - - -def generate_gc(chart_type, test_execution_configs, environment): - structures = [] - for test_execution_config in test_execution_configs: - if "gc" in test_execution_config.charts: - title = chart_type.format_title(environment, test_execution_config.workload, os_license=test_execution_config.os_license, - suffix="%s-gc" % test_execution_config.label) - structures.append(chart_type.gc(title, environment, test_execution_config)) - - return structures - -def generate_merge_time(chart_type, test_execution_configs, environment): - structures = [] - for test_execution_config in test_execution_configs: - if "merge_times" in test_execution_config.charts: - title = chart_type.format_title(environment, test_execution_config.workload, os_license=test_execution_config.os_license, - suffix=f"{test_execution_config.label}-merge-times") - structures.append(chart_type.merge_time(title, environment, test_execution_config)) - - return structures - -def generate_merge_count(chart_type, test_execution_configs, environment): - structures = [] - for test_execution_config in test_execution_configs: - if "merge_count" in test_execution_config.charts: - title = chart_type.format_title(environment, test_execution_config.workload, os_license=test_execution_config.os_license, - suffix=f"{test_execution_config.label}-merge-count") - structures.append(chart_type.merge_count(title, environment, test_execution_config)) - - return structures - - -def generate_segment_memory(chart_type, test_execution_configs, environment): - structures = [] - for test_execution_config in test_execution_configs: - if "segment_memory" in test_execution_config.charts: - title = chart_type.format_title(environment, test_execution_config.workload, os_license=test_execution_config.os_license, - suffix="%s-segment-memory" % test_execution_config.label) - chart = chart_type.segment_memory(title, environment, test_execution_config) - if chart: - structures.append(chart) - return structures - - -def generate_dashboard(chart_type, environment, workload, charts, flavor=None): - panels = [] - - width = 24 - height = 32 - - row = 0 - col = 0 - - for idx, chart in enumerate(charts): - panelIndex = idx + 1 - # make index charts wider - if chart["attributes"]["description"] == "index": - chart_width = 2 * width - # force one panel per row - next_col = 0 - else: - chart_width = width - # two rows per panel - next_col = (col + 1) % 2 - - panel = { - "id": chart["id"], - "panelIndex": panelIndex, - "gridData": { - "x": (col * chart_width), - "y": (row * height), - "w": chart_width, - "h": height, - "i": str(panelIndex) - }, - "type": "visualization", - "version": "7.10.2" - } - panels.append(panel) - col = next_col - if col == 0: - row += 1 - - return { - "id": str(uuid.uuid4()), - "type": "dashboard", - "attributes": { - "title": chart_type.format_title(environment, workload.name, flavor=flavor), - "hits": 0, - "description": "", - "panelsJSON": json.dumps(panels), - "optionsJSON": "{\"darkTheme\":false}", - "uiStateJSON": "{}", - "version": 1, - "timeRestore": False, - "kibanaSavedObjectMeta": { - "searchSourceJSON": json.dumps( - { - "filter": [ - { - "query": { - "query_string": { - "analyze_wildcard": True, - "query": "*" - } - } - } - ], - "highlightAll": True, - "version": True - } - ) - } - } - } - - -class TestExecutionConfig: - def __init__(self, workload, cfg=None, flavor=None, os_license=None, \ - test_procedure=None, provision_config_instance=None, node_count=None,\ - charts=None): - self.workload = workload - if cfg: - self.configuration = cfg - self.configuration["flavor"] = flavor - self.configuration["os_license"] = os_license - else: - self.configuration = { - "charts": charts, - "test_procedure": test_procedure, - "provision-config-instance": provision_config_instance, - "node-count": node_count - } - - @property - def name(self): - return self.configuration.get("name") - - @property - def flavor(self): - return self.configuration.get("flavor") - - @property - def os_license(self): - return self.configuration.get("os_license") - - @property - def label(self): - return self.configuration.get("label") - - @property - def charts(self): - return self.configuration["charts"] - - @property - def node_count(self): - return self.configuration.get("node-count", 1) - - @property - def test_procedure(self): - return self.configuration["test_procedure"] - - @property - def provision_config_instance(self): - return self.configuration["provision-config-instance"] - - @property - def plugins(self): - return self.configuration.get("plugins", "") - - @property - def bulk_tasks(self): - task_names = [] - for task in self.workload.find_test_procedure_or_default(self.test_procedure).schedule: - for sub_task in task: - # We are looking for type bulk operations to add to indexing throughput chart. - # For the observability workload, the index operation is of type raw-bulk, instead of type bulk. - # Doing a lenient match to allow for that. - if workload.OperationType.Bulk.to_hyphenated_string() in sub_task.operation.type: - if workload.OperationType.Bulk.to_hyphenated_string() != sub_task.operation.type: - console.info(f"Found [{sub_task.name}] of type [{sub_task.operation.type}] in "\ - f"[{self.test_procedure}], adding it to indexing dashboard.\n", flush=True) - task_names.append(sub_task.name) - return task_names - - @property - def throttled_tasks(self): - task_names = [] - for task in self.workload.find_test_procedure_or_default(self.test_procedure).schedule: - for sub_task in task: - # We are assuming here that each task with a target throughput or target interval is interesting for latency charts. - # We should refactor the chart generator to make this classification logic more flexible so the user can specify - # which tasks / or types of operations should be used for which chart types. - if "target-throughput" in sub_task.params or "target-interval" in sub_task.params: - task_names.append(sub_task.name) - return task_names - - -def load_test_execution_configs(cfg, chart_type, chart_spec_path=None): - def add_configs(test_execution_configs_per_lic, flavor_name="oss", lic="oss", workload_name=None): - configs_per_lic = [] - for test_execution_config in test_execution_configs_per_lic: - excluded_tasks = None - if "exclude-tasks" in test_execution_config: - excluded_tasks = test_execution_config.get("exclude-tasks").split(",") - configs_per_lic.append( - TestExecutionConfig(workload=test_execution_config_workload.get_workload(cfg, name=workload_name, - params=test_execution_config.get("workload-params", {}), - excluded_tasks=excluded_tasks), - cfg=test_execution_config, - flavor=flavor_name, - os_license=lic) - ) - return configs_per_lic - - def add_test_execution_configs(license_configs, flavor_name, workload_name): - if chart_type == BarCharts: - # Only one license config, "basic", is present in bar charts - _lic_conf = [license_config["configurations"] for license_config in license_configs if license_config["name"] == "basic"] - if _lic_conf: - test_execution_configs_per_workload.extend(add_configs(_lic_conf[0], workload_name=workload_name)) - else: - for lic_config in license_configs: - test_execution_configs_per_workload.extend(add_configs(lic_config["configurations"], - flavor_name, - lic_config["name"], - workload_name)) - - test_execution_configs = {"oss": [], "default": []} - if chart_type == BarCharts: - test_execution_configs = [] - - for _workload_file in glob.glob(io.normalize_path(chart_spec_path)): - with open(_workload_file, mode="rt", encoding="utf-8") as f: - for item in json.load(f): - _workload_repository = item.get("workload-repository", "default") - test_execution_config_workload = TestExecutionConfigWorkload(cfg, _workload_repository, name=item["workload"]) - for flavor in item["flavors"]: - test_execution_configs_per_workload = [] - _flavor_name = flavor["name"] - _workload_name = item["workload"] - add_test_execution_configs(flavor["licenses"], _flavor_name, _workload_name) - - if test_execution_configs_per_workload: - if chart_type == BarCharts: - test_execution_configs.append(test_execution_configs_per_workload) - else: - test_execution_configs[_flavor_name].append(test_execution_configs_per_workload) - return test_execution_configs - - -def gen_charts_per_workload_configs(test_execution_configs, chart_type, env, flavor=None, logger=None): - charts = generate_index_ops(chart_type, test_execution_configs, env, logger) + \ - generate_io(chart_type, test_execution_configs, env) + \ - generate_gc(chart_type, test_execution_configs, env) + \ - generate_merge_time(chart_type, test_execution_configs, env) + \ - generate_merge_count(chart_type, test_execution_configs, env) + \ - generate_segment_memory(chart_type, test_execution_configs, env) + \ - generate_queries(chart_type, test_execution_configs, env) - - dashboard = generate_dashboard(chart_type, env, test_execution_configs[0].workload, charts, flavor) - - return charts, dashboard - - -def gen_charts_per_workload(test_execution_configs, chart_type, env, flavor=None, logger=None): - structures = [] - for test_execution_configs_per_workload in test_execution_configs: - charts, dashboard = gen_charts_per_workload_configs(test_execution_configs_per_workload, chart_type, env, flavor, logger) - structures.extend(charts) - structures.append(dashboard) - - return structures - - -def gen_charts_from_workload_combinations(test_execution_configs, chart_type, env, logger): - structures = [] - for flavor, test_execution_configs_per_flavor in test_execution_configs.items(): - for test_execution_configs_per_workload in test_execution_configs_per_flavor: - logger.debug("Generating charts for test_execution_configs with name:[%s]/flavor:[%s]", - test_execution_configs_per_workload[0].name, flavor) - charts, dashboard = gen_charts_per_workload_configs(test_execution_configs_per_workload, chart_type, env, flavor, logger) - - structures.extend(charts) - structures.append(dashboard) - - return structures - - -def generate(cfg): - logger = logging.getLogger(__name__) - - chart_spec_path = cfg.opts("generator", "chart.spec.path") - if cfg.opts("generator", "chart.type") == "time-series": - chart_type = TimeSeriesCharts - else: - chart_type = BarCharts - - console.info("Loading workload data...", flush=True) - test_execution_configs = load_test_execution_configs(cfg, chart_type, chart_spec_path) - env = cfg.opts("system", "env.name") - - structures = [] - console.info("Generating charts...", flush=True) - - if chart_type == BarCharts: - # bar charts are flavor agnostic and split results based on a separate `user.setup` field - structures = gen_charts_per_workload(test_execution_configs, chart_type, env, logger=logger) - elif chart_type == TimeSeriesCharts: - structures = gen_charts_from_workload_combinations(test_execution_configs, chart_type, env, logger) - - output_path = cfg.opts("generator", "output.path") - if output_path: - with open(io.normalize_path(output_path), mode="wt", encoding="utf-8") as f: - for record in structures: - print(json.dumps(record), file=f) - else: - for record in structures: - print(json.dumps(record)) diff --git a/setup.py b/setup.py index 108e4343d..f93e9d000 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def str_from_file(name): # transitive dependencies: # urllib3: MIT # aiohttp: Apache 2.0 - "opensearch-py[async]==2.2.0", + "opensearch-py[async]==2.3.2", # License: BSD "psutil==5.8.0", # License: MIT @@ -97,7 +97,7 @@ def str_from_file(name): # botocore: Apache 2.0 # jmespath: MIT # s3transfer: Apache 2.0 - "boto3==1.10.32", + "boto3==1.28.62", ] tests_require = [