refactor big5, eventdata, http_logs workloads

Signed-off-by: Michael Oviedo <[email protected]>
opensearch-project · Dec 18, 2024 · 43a92de · 43a92de
1 parent 429382c
commit 43a92de
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 553 deletions.
diff --git a/big5/test_procedures/common/big5-schedule.json b/big5/test_procedures/common/big5-schedule.json
@@ -1,59 +1,4 @@
-{
-  "operation": "delete-index"
-},
-{
-  "operation": {
-    "operation-type": "create-index",
-    "settings": {{index_settings | default({}) | tojson}}
-  }
-},
-{
-  "name": "check-cluster-health",
-  "operation": {
-    "operation-type": "cluster-health",
-    "index": "{{index_name | default('big5')}}",
-    "request-params": {
-      "wait_for_status": "{{cluster_health | default('green')}}",
-      "wait_for_no_relocating_shards": "true"
-    },
-    "retry-until-success": true
-  }
-},
-{
-  "operation": "index-append",
-  "warmup-time-period": 240,
-  "clients": {{bulk_indexing_clients | default(8)}},
-  "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-},
-{
-  "name": "refresh-after-index",
-  "operation": "refresh"
-},
-{
-  "operation": {
-    "operation-type": "force-merge",
-    "request-timeout": 7200{%- if max_num_segments is defined %},
-    "max-num-segments": {{ max_num_segments | tojson }}
-    {%- endif %}
-  }
-},
-{
-  "name": "refresh-after-force-merge",
-  "operation": "refresh"
-},
-{
-  "name": "wait-until-merges-finish",
-  "operation": {
-    "operation-type": "index-stats",
-    "index": "_all",
-    "condition": {
-      "path": "_all.total.merges.current",
-      "expected-value": 0
-    },
-    "retry-until-success": true,
-    "include-in-reporting": false
-  }
-},
+{{ benchmark.collect(parts="../../../common_operations/workload_setup.json") }},
 {
   "operation": "default",
   "warmup-iterations": {{ warmup_iterations | default(200) | tojson }},

diff --git a/eventdata/test_procedures/default.json b/eventdata/test_procedures/default.json
@@ -3,105 +3,15 @@
       "description": "Indexes the whole document corpus using OpenSearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Benchmark will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.",
       "default": true,
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {{index_settings | default({}) | tojson}}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "eventdata",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        }
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }}
       ]
     },
     {
       "name": "transform",
       "description": "Indexes the whole document corpus using OpenSearch default settings and run transforms to pivot data by terms, date and geo tiles.",
       "default": false,
       "schedule": [
-        {
-          "operation": "delete-index"
-        },
-        {
-          "operation": {
-            "operation-type": "create-index",
-            "settings": {{index_settings | default({}) | tojson}}
-          }
-        },
-        {
-          "name": "check-cluster-health",
-          "operation": {
-            "operation-type": "cluster-health",
-            "index": "eventdata",
-            "request-params": {
-              "wait_for_status": "{{cluster_health | default('green')}}",
-              "wait_for_no_relocating_shards": "true"
-            },
-            "retry-until-success": true
-          }
-        },
-        {
-          "operation": "index-append",
-          "warmup-time-period": 120,
-          "clients": {{bulk_indexing_clients | default(8)}},
-          "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
-        },
-        {
-          "operation": {
-            "operation-type": "force-merge",
-            "request-timeout": 7200
-          }
-        },
-        {
-          "name": "wait-until-merges-finish",
-          "operation": {
-            "operation-type": "index-stats",
-            "index": "_all",
-            "condition": {
-              "path": "_all.total.merges.current",
-              "expected-value": 0
-            },
-            "retry-until-success": true,
-            "include-in-reporting": false
-          }
-        },
+        {{ benchmark.collect(parts="../../common_operations/workload_setup.json") }},
         {
           "name": "delete-transform-group-by-terms",
           "operation": {