Expose model accuracy metrics in tests (#600)

* Expose model accuracy metrics in tests

This PR adds an option flag to print logs during tests and turn on the flag in CI workflow. The flag is disabled by default. By doing this, we can record model accuracy metrics in git workflows and later retrieve it for analysis.

Testing done:
1. We can turn on/off logs during tests.
2. The accuracy logs are recorded.

Signed-off-by: Kaituo Li <[email protected]>

.github/workflows/CI.yml

@@ -41,7 +41,7 @@ jobs:
           ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
       - name: Build and Run Tests
         run: |
-          ./gradlew build
+          ./gradlew build -Dtest.logs=true
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal

build.gradle

@@ -42,6 +42,10 @@ buildscript {
                 'opendistro-anomaly-detection/opendistro-anomaly-detection-1.13.0.0.zip'
         bwcOpenDistroJSDownload = 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/' +
                 'opendistro-job-scheduler/opendistro-job-scheduler-1.13.0.0.zip'
+        // gradle build won't print logs during test by default unless there is a failure.
+        // It is useful to record intermediately information like prediction precision and recall.
+        // This option turn on log printing during tests. 
+        printLogs = "true" == System.getProperty("test.logs", "false")
     }
 
     repositories {
@@ -269,6 +273,12 @@ integTest {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
     }
 
+    if (printLogs) {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
 }
 
 testClusters.integTest {
@@ -779,3 +789,13 @@ validateNebulaPom.enabled = false
 tasks.withType(licenseHeaders.class) {
     additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
 }
+
+// show test results so that we can record information like precion/recall results of correctness testing.
+if (printLogs) {
+    test {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
+}

release-notes/opensearch-anomaly-detection.release-notes-2.1.0.0.md

@@ -7,6 +7,7 @@ Compatible with OpenSearch 2.1.0
 
 * bump rcf to 3.0-rc3 ([#568](https://github.com/opensearch-project/anomaly-detection/pull/568))
 * Disable interpolation in HCAD cold start ([#575](https://github.com/opensearch-project/anomaly-detection/pull/575))
+* Expose model accuracy metrics in tests ([#600](https://github.com/opensearch-project/anomaly-detection/pull/600))
 
 ### Bug Fixes
 

src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java

@@ -122,6 +122,7 @@ private void verifyTestResults(
         assertTrue(recall >= minRecall);
 
         assertTrue(errors <= maxError);
+        LOG.info("Precision: {}, Window recall: {}", precision, recall);
     }
 
     private int isAnomaly(Instant time, List<Entry<Instant, Instant>> labels) {

src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java

@@ -853,6 +853,7 @@ public int compare(Entry<Long, Long> p1, Entry<Long, Long> p2) {
 
         assertTrue("precision is " + prec, prec >= precisionThreshold);
         assertTrue("recall is " + recall, recall >= recallThreshold);
+        LOG.info("Interval {}, Precision: {}, recall: {}", detectorIntervalMins, prec, recall);
     }
 
     public int searchInsert(long[] timestamps, long target) {