introduce common operations directory for OSB workloads (#514) (#528)

* restructure common operations folder based on discussion with ian * refactor common ops changes based on discussion with ian * rework common ops configuration * add missing params and update geonames workload * parameterize request-timeout and warmup-time-period --------- (cherry picked from commit 5119393) Signed-off-by: Michael Oviedo <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Dec 12, 2024 · af2910c · af2910c
1 parent a135b49
commit af2910c
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 244 deletions.
diff --git a/common_operations/check_cluster_health.json b/common_operations/check_cluster_health.json
@@ -0,0 +1,12 @@
+{
+    "name": "check-cluster-health",
+    "operation": {
+        "operation-type": "cluster-health",
+        "index": "{{ index_name }}",
+        "request-params": {
+            "wait_for_status": "{{cluster_health | default('green')}}",
+            "wait_for_no_relocating_shards": "true"
+        },
+        "retry-until-success": true
+    }
+}
diff --git a/common_operations/create_index.json b/common_operations/create_index.json
@@ -0,0 +1,6 @@
+{
+    "operation": {
+        "operation-type": "create-index",
+        "settings": {{index_settings | default(default_index_settings | default({})) | tojson}}
+    }
+}
diff --git a/common_operations/delete_index.json b/common_operations/delete_index.json
@@ -0,0 +1,3 @@
+{
+    "operation": "delete-index"
+}
diff --git a/common_operations/force_merge.json b/common_operations/force_merge.json
@@ -0,0 +1,25 @@
+{
+    "operation": {
+        "operation-type": "force-merge",
+        "request-timeout": {{ request_timeout | default(60) | tojson }}{%- if max_num_segments is defined %},
+        "max-num-segments": {{ max_num_segments | tojson }}
+        {%- endif %}
+    }
+},
+{
+    "name": "refresh-after-force-merge",
+    "operation": "refresh"
+},
+{
+    "name": "wait-until-merges-finish",
+    "operation": {
+        "operation-type": "index-stats",
+        "index": "_all",
+        "condition": {
+            "path": "_all.total.merges.current",
+            "expected-value": 0
+        },
+        "retry-until-success": true,
+        "include-in-reporting": false
+    }
+}
diff --git a/common_operations/index_append.json b/common_operations/index_append.json
@@ -0,0 +1,10 @@
+{
+    "operation": "index-append",
+    "warmup-time-period": {{ warmup_time_period | default(120) | tojson }},
+    "clients": {{bulk_indexing_clients | default(8)}},
+    "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
+},
+{
+    "name": "refresh-after-index",
+    "operation": "refresh"
+}
diff --git a/common_operations/workload_setup.json b/common_operations/workload_setup.json
@@ -0,0 +1,5 @@
+{{ benchmark.collect(parts="delete_index.json") }},
+{{ benchmark.collect(parts="create_index.json") }},
+{{ benchmark.collect(parts="check_cluster_health.json") }},
+{{ benchmark.collect(parts="index_append.json") }},
+{{ benchmark.collect(parts="force_merge.json") }}
diff --git a/geonames/test_procedures/default.json b/geonames/test_procedures/default.json
@@ -3,60 +3,9 @@
       "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. After that a couple of queries are run.",
       "default": true,
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {{index_settings | default({}) | tojson}}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "geonames",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "name": "refresh-after-index",
-          "operation": "refresh"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "refresh-after-force-merge",
-          "operation": "refresh"
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        },
+        {% with default_index_settings={} %}
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }},
+        {% endwith %}
         {
           "operation": "index-stats",
           "warmup-iterations": {{ index_stats_warmup_iterations or warmup_iterations | default(500) | tojson }},
@@ -239,214 +188,43 @@
       "name": "append-no-conflicts-index-only",
       "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.",
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {{index_settings | default({}) | tojson}}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "geonames",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        }
+        {% with default_index_settings={} %}
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}
+        {% endwith %}
       ]
     },
     {
       "name": "append-sorted-no-conflicts",
       "description": "Indexes the whole document corpus in an index sorted by country_code field in ascending order. Document ids are unique so all index operations are append only.",
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} {
-              "index.sort.field": ["country_code.raw", "admin1_code.raw"],
-              "index.sort.order": ["asc", "asc"]
-            }{%- endif %}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "geonames",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        }
+        {% with default_index_settings={
+          "index.sort.field": ["country_code.raw", "admin1_code.raw"],
+          "index.sort.order": ["asc", "asc"]
+        } %}
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}
+        {% endwith %}
       ]
     },
     {
       "name": "append-fast-with-conflicts",
       "description": "Indexes the whole document corpus using a setup that will lead to a larger indexing throughput than the default settings. Benchmark will produce duplicate ids in 25% of all documents (not configurable) so we can simulate a scenario with appends most of the time and some updates in between.",
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} {
-              "index.refresh_interval": "30s",
-              "index.number_of_shards": {{number_of_shards | default(6)}},
-              "index.translog.flush_threshold_size": "4g"
-            }{%- endif %}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "geonames",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-update",
-          "warmup-time-period": 45,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        }
+        {% with default_index_settings={
+          "index.refresh_interval": "30s",
+          "index.number_of_shards": number_of_shards | default(6),
+          "index.translog.flush_threshold_size": "4g"
+        } %}
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}
+        {% endwith %}
       ]
     },
     {
       "name": "significant-text",
       "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.",
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {{index_settings | default({}) | tojson}}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "geonames",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}}
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        },
+        {% with default_index_settings={} %}
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }},
+        {% endwith %}
         {
           "operation": "significant_text_selective",
           "warmup-iterations": 200,