AD model performance benchmark (#718)

This PR adds an AD model performance benchmark so that we can compare model performance across versions. We run the benchmark in separate github workflows since they can be time consuming. For example, it takes 25+ minutes to run HCAD benchmarking alone in 1.1. Also, we print bench-marking results in standard output for recording purpose. For HCAD, we randomly generated synthetic data with known anomalies inserted throughout the signal. In particular, these are one/two/four dimensional data where each dimension is a noisy cosine wave. Anomalies are inserted into one dimension with 0.003 probability. Anomalies across each dimension can be independent or dependent. We have approximately 5000 observations per data set. The data set is generated using the same random seed so the result is comparable across versions. For single stream detectors, we use a curated data set with known anomaly windows. We also backported #600 so that we can capture the performance data in CI output. Testing done: 1. added unit tests to run the benchmark. Signed-off-by: Kaituo Li <[email protected]>
opensearch-project · Nov 17, 2022 · 9805c92 · 9805c92
1 parent c54e39e
commit 9805c92
Show file tree

Hide file tree

Showing 11 changed files with 957 additions and 188 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -33,16 +33,16 @@ jobs:
         with:
           repository: 'opensearch-project/OpenSearch'
           path: OpenSearch
-          ref: '1.1'
+          ref: '1.1.0'
       - name: Build OpenSearch
         working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal
+        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
 
       # dependencies: common-utils
       - name: Checkout common-utils
         uses: actions/checkout@v2
         with:
-          ref: 'main'
+          ref: '1.1'
           repository: 'opensearch-project/common-utils'
           path: common-utils
       - name: Build common-utils
@@ -53,7 +53,7 @@ jobs:
       - name: Checkout job-scheduler
         uses: actions/checkout@v2
         with:
-          ref: 'main'
+          ref: '1.1'
           repository: 'opensearch-project/job-scheduler'
           path: job-scheduler
 

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,86 @@
+name: Run AD benchmark
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+
+  Build-ad:
+    strategy:
+      matrix:
+        java: [14]
+
+    name: Run Anomaly detection model performance benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      # dependencies: OpenSearch
+      - name: Checkout OpenSearch
+        uses: actions/checkout@v2
+        with:
+          repository: 'opensearch-project/OpenSearch'
+          path: OpenSearch
+          ref: '1.1.0'
+      - name: Build OpenSearch
+        working-directory: ./OpenSearch
+        run: ./gradlew publishToMavenLocal
+
+      # dependencies: common-utils
+      - name: Checkout common-utils
+        uses: actions/checkout@v2
+        with:
+          ref: '1.1'
+          repository: 'opensearch-project/common-utils'
+          path: common-utils
+      - name: Build common-utils
+        working-directory: ./common-utils
+        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
+
+      # dependencies: job-scheduler
+      - name: Checkout job-scheduler
+        uses: actions/checkout@v2
+        with:
+          ref: '1.1'
+          repository: 'opensearch-project/job-scheduler'
+          path: job-scheduler
+
+      - name: Build job-scheduler
+        working-directory: ./job-scheduler
+        run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
+      - name: Assemble job-scheduler
+        working-directory: ./job-scheduler
+        run: |
+          ./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
+          echo "Creating ../src/test/resources/job-scheduler ..."
+          mkdir -p ../src/test/resources/job-scheduler
+          pwd
+          echo "Copying ./build/distributions/*.zip to ../src/test/resources/job-scheduler ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ../src/test/resources/job-scheduler
+          echo "Copied ./build/distributions/*.zip to ../src/test/resources/job-scheduler ..."
+          ls ../src/test/resources/job-scheduler
+          echo "Creating ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          mkdir -p ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+          echo "Copying ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+          echo "Copied ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
+          ls ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true
+          ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -45,6 +45,8 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and
 8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
 9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
 10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
+11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output
+12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 

diff --git a/build.gradle b/build.gradle
@@ -36,13 +36,17 @@ buildscript {
         opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
         job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build)
+        // gradle build won't print logs during test by default unless there is a failure.
+        // It is useful to record intermediately information like prediction precision and recall.
+        // This option turn on log printing during tests. 
+        printLogs = "true" == System.getProperty("test.logs", "false")
     }
 
     repositories {
         mavenLocal()
+        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
         mavenCentral()
         maven { url "https://plugins.gradle.org/m2/" }
-        jcenter()
     }
 
     dependencies {
@@ -69,9 +73,10 @@ tasks.withType(Javadoc) {
 
 repositories {
     mavenLocal()
+    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
     mavenCentral()
     maven { url "https://plugins.gradle.org/m2/" }
-    jcenter()
+    maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" }
 }
 
 apply plugin: 'java'
@@ -168,6 +173,12 @@ opensearch_tmp_dir.mkdirs()
 test {
     include '**/*Tests.class'
     systemProperty 'tests.security.manager', 'false'
+
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.ml.HCADModelPerfTests"
+        }
+    }
 }
 
 task integTest(type: RestIntegTestTask) {
@@ -206,6 +217,12 @@ integTest {
         }
     }
 
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.e2e.SingleStreamModelPerfIT"
+        }
+    }
+
     // The 'doFirst' delays till execution time.
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
@@ -226,6 +243,12 @@ integTest {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
     }
 
+    if (printLogs) {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
 }
 
 testClusters.integTest {
@@ -699,3 +722,13 @@ validateNebulaPom.enabled = false
 tasks.withType(licenseHeaders.class) {
     additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
 }
+
+// show test results so that we can record information like precion/recall results of correctness testing.
+if (printLogs) {
+    test {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
+}
diff --git a/src/test/java/org/opensearch/ad/TestHelpers.java b/src/test/java/org/opensearch/ad/TestHelpers.java
@@ -51,6 +51,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -1150,4 +1151,133 @@ public static Map<String, Object> parseStatsResult(String statsResult) throws IO
         }
         return adStats;
     }
+
+    public static class AnomalyDetectorBuilder {
+        private String detectorId = randomAlphaOfLength(10);
+        private Long version = randomLong();
+        private String name = randomAlphaOfLength(20);
+        private String description = randomAlphaOfLength(30);
+        private String timeField = randomAlphaOfLength(5);
+        private List<String> indices = ImmutableList.of(randomAlphaOfLength(10).toLowerCase(Locale.ROOT));
+        private List<Feature> featureAttributes = ImmutableList.of(randomFeature(true));
+        private QueryBuilder filterQuery;
+        private TimeConfiguration detectionInterval = randomIntervalTimeConfiguration();
+        private TimeConfiguration windowDelay = randomIntervalTimeConfiguration();
+        private Integer shingleSize = randomIntBetween(1, AnomalyDetectorSettings.MAX_SHINGLE_SIZE);
+        private Map<String, Object> uiMetadata = null;
+        private Integer schemaVersion = randomInt();
+        private Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        private List<String> categoryFields = null;
+        private User user = randomUser();
+        private String resultIndex = null;
+
+        public static AnomalyDetectorBuilder newInstance() throws IOException {
+            return new AnomalyDetectorBuilder();
+        }
+
+        private AnomalyDetectorBuilder() throws IOException {
+            filterQuery = randomQuery();
+        }
+
+        public AnomalyDetectorBuilder setDetectorId(String detectorId) {
+            this.detectorId = detectorId;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setVersion(Long version) {
+            this.version = version;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setName(String name) {
+            this.name = name;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setTimeField(String timeField) {
+            this.timeField = timeField;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setIndices(List<String> indices) {
+            this.indices = indices;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setFeatureAttributes(List<Feature> featureAttributes) {
+            this.featureAttributes = featureAttributes;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setFilterQuery(QueryBuilder filterQuery) {
+            this.filterQuery = filterQuery;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setDetectionInterval(TimeConfiguration detectionInterval) {
+            this.detectionInterval = detectionInterval;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setWindowDelay(TimeConfiguration windowDelay) {
+            this.windowDelay = windowDelay;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setShingleSize(Integer shingleSize) {
+            this.shingleSize = shingleSize;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setUiMetadata(Map<String, Object> uiMetadata) {
+            this.uiMetadata = uiMetadata;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setSchemaVersion(Integer schemaVersion) {
+            this.schemaVersion = schemaVersion;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setLastUpdateTime(Instant lastUpdateTime) {
+            this.lastUpdateTime = lastUpdateTime;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setCategoryFields(List<String> categoryFields) {
+            this.categoryFields = categoryFields;
+            return this;
+        }
+
+        public AnomalyDetectorBuilder setUser(User user) {
+            this.user = user;
+            return this;
+        }
+
+        public AnomalyDetector build() {
+            return new AnomalyDetector(
+                detectorId,
+                version,
+                name,
+                description,
+                timeField,
+                indices,
+                featureAttributes,
+                filterQuery,
+                detectionInterval,
+                windowDelay,
+                shingleSize,
+                uiMetadata,
+                schemaVersion,
+                lastUpdateTime,
+                categoryFields,
+                user
+            );
+        }
+    }
 }
diff --git a/.../ad/e2e/DetectionResultEvalutationIT.java → ...earch/ad/e2e/SingleStreamModelPerfIT.java b/.../ad/e2e/DetectionResultEvalutationIT.java → ...earch/ad/e2e/SingleStreamModelPerfIT.java
@@ -64,8 +64,8 @@
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 
-public class DetectionResultEvalutationIT extends ODFERestTestCase {
-    protected static final Logger LOG = (Logger) LogManager.getLogger(DetectionResultEvalutationIT.class);
+public class SingleStreamModelPerfIT extends ODFERestTestCase {
+    protected static final Logger LOG = (Logger) LogManager.getLogger(SingleStreamModelPerfIT.class);
 
     // TODO: fix flaky test, sometimes this assert will fail "assertTrue(precision >= minPrecision);"
     public void testDataset() throws Exception {
@@ -123,6 +123,7 @@ private void verifyTestResults(
         assertTrue(recall >= minRecall);
 
         assertTrue(errors <= maxError);
+        LOG.info("Precision: {}, Window recall: {}", precision, recall);
     }
 
     private int isAnomaly(Instant time, List<Entry<Instant, Instant>> labels) {