From cd2321d8e5d020d33f3ac380d0a4e1617e063c70 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Dec 2024 18:52:25 +0000 Subject: [PATCH] introduce common operations directory for OSB workloads (#514) * restructure common operations folder based on discussion with ian Signed-off-by: Michael Oviedo * refactor common ops changes based on discussion with ian Signed-off-by: Michael Oviedo * rework common ops configuration Signed-off-by: Michael Oviedo * add missing params and update geonames workload Signed-off-by: Michael Oviedo * parameterize request-timeout and warmup-time-period Signed-off-by: Michael Oviedo --------- Signed-off-by: Michael Oviedo (cherry picked from commit 511939335611e1cd7cea8b54582f20fcdd543c08) Signed-off-by: github-actions[bot] --- common_operations/check_cluster_health.json | 12 + common_operations/create_index.json | 6 + common_operations/delete_index.json | 3 + common_operations/force_merge.json | 25 ++ common_operations/index_append.json | 10 + common_operations/workload_setup.json | 5 + geonames/test_procedures/default.json | 266 ++------------------ 7 files changed, 83 insertions(+), 244 deletions(-) create mode 100644 common_operations/check_cluster_health.json create mode 100644 common_operations/create_index.json create mode 100644 common_operations/delete_index.json create mode 100644 common_operations/force_merge.json create mode 100644 common_operations/index_append.json create mode 100644 common_operations/workload_setup.json diff --git a/common_operations/check_cluster_health.json b/common_operations/check_cluster_health.json new file mode 100644 index 00000000..4f39e419 --- /dev/null +++ b/common_operations/check_cluster_health.json @@ -0,0 +1,12 @@ +{ + "name": "check-cluster-health", + "operation": { + "operation-type": "cluster-health", + "index": "{{ index_name }}", + "request-params": { + "wait_for_status": "{{cluster_health | default('green')}}", + "wait_for_no_relocating_shards": "true" + }, + "retry-until-success": true + } +} \ No newline at end of file diff --git a/common_operations/create_index.json b/common_operations/create_index.json new file mode 100644 index 00000000..07fae35e --- /dev/null +++ b/common_operations/create_index.json @@ -0,0 +1,6 @@ +{ + "operation": { + "operation-type": "create-index", + "settings": {{index_settings | default(default_index_settings | default({})) | tojson}} + } +} \ No newline at end of file diff --git a/common_operations/delete_index.json b/common_operations/delete_index.json new file mode 100644 index 00000000..99b6b631 --- /dev/null +++ b/common_operations/delete_index.json @@ -0,0 +1,3 @@ +{ + "operation": "delete-index" +} \ No newline at end of file diff --git a/common_operations/force_merge.json b/common_operations/force_merge.json new file mode 100644 index 00000000..e0ddfb41 --- /dev/null +++ b/common_operations/force_merge.json @@ -0,0 +1,25 @@ +{ + "operation": { + "operation-type": "force-merge", + "request-timeout": {{ request_timeout | default(60) | tojson }}{%- if max_num_segments is defined %}, + "max-num-segments": {{ max_num_segments | tojson }} + {%- endif %} + } +}, +{ + "name": "refresh-after-force-merge", + "operation": "refresh" +}, +{ + "name": "wait-until-merges-finish", + "operation": { + "operation-type": "index-stats", + "index": "_all", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } +} \ No newline at end of file diff --git a/common_operations/index_append.json b/common_operations/index_append.json new file mode 100644 index 00000000..a0f0b288 --- /dev/null +++ b/common_operations/index_append.json @@ -0,0 +1,10 @@ +{ + "operation": "index-append", + "warmup-time-period": {{ warmup_time_period | default(120) | tojson }}, + "clients": {{bulk_indexing_clients | default(8)}}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}" +}, +{ + "name": "refresh-after-index", + "operation": "refresh" +} \ No newline at end of file diff --git a/common_operations/workload_setup.json b/common_operations/workload_setup.json new file mode 100644 index 00000000..17ce8c68 --- /dev/null +++ b/common_operations/workload_setup.json @@ -0,0 +1,5 @@ +{{ benchmark.collect(parts="delete_index.json") }}, +{{ benchmark.collect(parts="create_index.json") }}, +{{ benchmark.collect(parts="check_cluster_health.json") }}, +{{ benchmark.collect(parts="index_append.json") }}, +{{ benchmark.collect(parts="force_merge.json") }} \ No newline at end of file diff --git a/geonames/test_procedures/default.json b/geonames/test_procedures/default.json index 7faf0c03..8227eec7 100644 --- a/geonames/test_procedures/default.json +++ b/geonames/test_procedures/default.json @@ -3,60 +3,9 @@ "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. After that a couple of queries are run.", "default": true, "schedule": [ - { - "operation": "delete-index" - }, - { - "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } - }, - { - "name": "check-cluster-health", - "operation": { - "operation-type": "cluster-health", - "index": "geonames", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": "index-append", - "warmup-time-period": 120, - "clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - { - "name": "refresh-after-index", - "operation": "refresh" - }, - { - "operation": { - "operation-type": "force-merge", - "request-timeout": 7200 - } - }, - { - "name": "refresh-after-force-merge", - "operation": "refresh" - }, - { - "name": "wait-until-merges-finish", - "operation": { - "operation-type": "index-stats", - "index": "_all", - "condition": { - "path": "_all.total.merges.current", - "expected-value": 0 - }, - "retry-until-success": true, - "include-in-reporting": false - } - }, + {% with default_index_settings={} %} + {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}, + {% endwith %} { "operation": "index-stats", "warmup-iterations": {{ index_stats_warmup_iterations or warmup_iterations | default(500) | tojson }}, @@ -239,214 +188,43 @@ "name": "append-no-conflicts-index-only", "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.", "schedule": [ - { - "operation": "delete-index" - }, - { - "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } - }, - { - "name": "check-cluster-health", - "operation": { - "operation-type": "cluster-health", - "index": "geonames", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": "index-append", - "warmup-time-period": 120, - "clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - { - "operation": { - "operation-type": "force-merge", - "request-timeout": 7200 - } - }, - { - "name": "wait-until-merges-finish", - "operation": { - "operation-type": "index-stats", - "index": "_all", - "condition": { - "path": "_all.total.merges.current", - "expected-value": 0 - }, - "retry-until-success": true, - "include-in-reporting": false - } - } + {% with default_index_settings={} %} + {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }} + {% endwith %} ] }, { "name": "append-sorted-no-conflicts", "description": "Indexes the whole document corpus in an index sorted by country_code field in ascending order. Document ids are unique so all index operations are append only.", "schedule": [ - { - "operation": "delete-index" - }, - { - "operation": { - "operation-type": "create-index", - "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} { - "index.sort.field": ["country_code.raw", "admin1_code.raw"], - "index.sort.order": ["asc", "asc"] - }{%- endif %} - } - }, - { - "name": "check-cluster-health", - "operation": { - "operation-type": "cluster-health", - "index": "geonames", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": "index-append", - "warmup-time-period": 120, - "clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - { - "operation": { - "operation-type": "force-merge", - "request-timeout": 7200 - } - }, - { - "name": "wait-until-merges-finish", - "operation": { - "operation-type": "index-stats", - "index": "_all", - "condition": { - "path": "_all.total.merges.current", - "expected-value": 0 - }, - "retry-until-success": true, - "include-in-reporting": false - } - } + {% with default_index_settings={ + "index.sort.field": ["country_code.raw", "admin1_code.raw"], + "index.sort.order": ["asc", "asc"] + } %} + {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }} + {% endwith %} ] }, { "name": "append-fast-with-conflicts", "description": "Indexes the whole document corpus using a setup that will lead to a larger indexing throughput than the default settings. Benchmark will produce duplicate ids in 25% of all documents (not configurable) so we can simulate a scenario with appends most of the time and some updates in between.", "schedule": [ - { - "operation": "delete-index" - }, - { - "operation": { - "operation-type": "create-index", - "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} { - "index.refresh_interval": "30s", - "index.number_of_shards": {{number_of_shards | default(6)}}, - "index.translog.flush_threshold_size": "4g" - }{%- endif %} - } - }, - { - "name": "check-cluster-health", - "operation": { - "operation-type": "cluster-health", - "index": "geonames", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": "index-update", - "warmup-time-period": 45, - "clients": {{bulk_indexing_clients | default(8)}}, - "ignore-response-error-level": "{{error_level | default('non-fatal')}}" - }, - { - "operation": { - "operation-type": "force-merge", - "request-timeout": 7200 - } - }, - { - "name": "wait-until-merges-finish", - "operation": { - "operation-type": "index-stats", - "index": "_all", - "condition": { - "path": "_all.total.merges.current", - "expected-value": 0 - }, - "retry-until-success": true, - "include-in-reporting": false - } - } + {% with default_index_settings={ + "index.refresh_interval": "30s", + "index.number_of_shards": number_of_shards | default(6), + "index.translog.flush_threshold_size": "4g" + } %} + {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }} + {% endwith %} ] }, { "name": "significant-text", "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.", "schedule": [ - { - "operation": "delete-index" - }, - { - "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } - }, - { - "name": "check-cluster-health", - "operation": { - "operation-type": "cluster-health", - "index": "geonames", - "request-params": { - "wait_for_status": "{{cluster_health | default('green')}}", - "wait_for_no_relocating_shards": "true" - }, - "retry-until-success": true - } - }, - { - "operation": "index-append", - "warmup-time-period": 120, - "clients": {{bulk_indexing_clients | default(8)}} - }, - { - "operation": { - "operation-type": "force-merge", - "request-timeout": 7200 - } - }, - { - "name": "wait-until-merges-finish", - "operation": { - "operation-type": "index-stats", - "index": "_all", - "condition": { - "path": "_all.total.merges.current", - "expected-value": 0 - }, - "retry-until-success": true, - "include-in-reporting": false - } - }, + {% with default_index_settings={} %} + {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}, + {% endwith %} { "operation": "significant_text_selective", "warmup-iterations": {{ significant_text_selective_warmup_iterations or warmup_iterations | default(200) | tojson }},