Adding windows and mac CI to 1.3 (#745)

Signed-off-by: Amit Galitzky <[email protected]>

.github/workflows/test_build_multi_platform.yml

@@ -0,0 +1,71 @@
+name: Build and Test Anomaly detection
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad-windows:
+    strategy:
+      matrix:
+        java: [8, 11, 14]
+    name: Build and Test Anomaly Detection Plugin on Windows
+    runs-on: windows-latest
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Checkout Anomaly Detection
+        uses: actions/checkout@v2
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew build
+      - name: Publish to Maven Local
+        run: |
+          ./gradlew publishToMavenLocal
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
+  Build-ad:
+    strategy:
+      matrix:
+        java: [8, 11, 14]
+        os: [ubuntu-latest, macos-latest]
+      fail-fast: false
+
+    name: Build and Test Anomaly detection Plugin on multiple platforms
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Assemble anomaly-detection
+        run: |
+          ./gradlew assemble
+      - name: Build and Run Tests
+        run: |
+          ./gradlew build -Dtest.logs=true
+      - name: Publish to Maven Local
+        run: |
+          ./gradlew publishToMavenLocal
+      - name: Multi Nodes Integration Testing
+        run: |
+          ./gradlew integTest -PnumNodes=3
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin

.github/workflows/test_bwc.yml

@@ -0,0 +1,45 @@
+name: Test Anomaly detection BWC
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [8, 11, 14]
+      fail-fast: false
+
+    name: Test Anomaly detection BWC
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Assemble anomaly-detection
+        run: |
+          plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
+          echo plugin_version $plugin_version
+          ./gradlew assemble
+          echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+      - name: Run AD Backwards Compatibility Tests
+        run: |
+          echo "Running backwards compatibility tests ..."
+          ./gradlew bwcTestSuite -Dtests.security.manager=false

.github/workflows/test_security.yml

@@ -0,0 +1,78 @@
+name: Security test workflow for Anomaly Detection
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [8, 11, 14]
+      fail-fast: false
+
+    name: Security test workflow for Anomaly Detection
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Build Anomaly Detection
+        run: |
+          ./gradlew assemble
+      # example of variables:
+      # plugin = opensearch-anomaly-detection-1.3.7.0-SNAPSHOT.zip
+      # version = 1.3.7, plugin_version = 1.3.7.0, qualifier = SNAPSHOT
+      - name: Pull and Run Docker
+        run: |
+          plugin=`basename $(ls build/distributions/*.zip)`
+          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
+          if $qualifier!=SNAPSHOT
+          then
+            docker_version=$version-$qualifier
+          else
+            docker_version=$version
+          fi
+          echo plugin version plugin_version qualifier docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
+          cd ..
+          if docker pull opensearchstaging/opensearch:$docker_version
+          then
+            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile
+            echo "ADD anomaly-detection/build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
+            docker build -t opensearch-ad:test .
+            echo "imagePresent=true" >> $GITHUB_ENV
+          else
+            echo "imagePresent=false" >> $GITHUB_ENV
+          fi
+      - name: Run Docker Image
+        if: env.imagePresent == 'true'
+        run: |
+          cd ..
+          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-ad:test
+          sleep 90
+      - name: Run AD Test
+        if: env.imagePresent == 'true'
+        run: |
+          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
+          if [ $security -gt 0 ]
+          then
+            echo "Security plugin is available"
+            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
+          else
+            echo "Security plugin is NOT available, skipping integration tests"
+          fi

build.gradle

@@ -121,7 +121,6 @@ configurations.all {
     if (it.state != Configuration.State.UNRESOLVED) return
     resolutionStrategy {
         force "joda-time:joda-time:${versions.joda}"
-        force "com.fasterxml.jackson.core:jackson-core:2.13.4"
         force "commons-logging:commons-logging:${versions.commonslogging}"
         force "org.apache.httpcomponents:httpcore:${versions.httpcore}"
         force "commons-codec:commons-codec:${versions.commonscodec}"
@@ -659,9 +658,8 @@ dependencies {
     implementation 'software.amazon.randomcutforest:randomcutforest-serialization:2.0.1'
     compile files('lib/randomcutforest-parkservices-2.0.1.jar')
     compile files('lib/randomcutforest-core-2.0.1.jar')
-    implementation "com.fasterxml.jackson.core:jackson-core:2.13.4"
-    implementation "com.fasterxml.jackson.core:jackson-databind:2.13.4.2"
-    implementation "com.fasterxml.jackson.core:jackson-annotations:2.13.4"
+    implementation "com.fasterxml.jackson.core:jackson-databind:2.14.1"
+    implementation "com.fasterxml.jackson.core:jackson-annotations:2.14.1"
 
     // used for serializing/deserializing rcf models.
     compile group: 'io.protostuff', name: 'protostuff-core', version: '1.8.0'
@@ -789,12 +787,6 @@ task updateVersion {
         ext.newVersion = System.getProperty('newVersion')
         println "Setting version to ${newVersion}."
         // String tokenization to support -SNAPSHOT
-        ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) {
-            fileset(dir: projectDir) {
-                // Include the required files that needs to be updated with new Version
-                include(name: ".github/workflows/CI.yml")
-            }
-        }
         ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
     }
 }

src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java

@@ -141,12 +141,14 @@ public void testValidationIntervalRecommendation() throws Exception {
 
     public void testValidationWindowDelayRecommendation() throws Exception {
         RestClient client = client();
-        long recDetectorIntervalMillis = 180000;
+        long recDetectorIntervalMillisForDataSet = 180000;
         // this would be equivalent to the window delay in this data test
-        long recDetectorIntervalMinutes = recDetectorIntervalMillis / 60000;
-        List<JsonObject> data = createData(2000, recDetectorIntervalMillis);
+        List<JsonObject> data = createData(2000, recDetectorIntervalMillisForDataSet);
         indexTrainData("validation", data, 2000, client);
         long detectorInterval = 4;
+        long expectedWindowDelayMillis = Instant.now().toEpochMilli() - data.get(0).get("timestamp").getAsLong();
+        // we always round up for window delay recommendation to reduce chance of missed data.
+        long expectedWindowDelayMinutes = (long) Math.ceil(expectedWindowDelayMillis / 60000.0);
         String requestBody = String
             .format(
                 Locale.ROOT,
@@ -171,10 +173,8 @@ public void testValidationWindowDelayRecommendation() throws Exception {
         @SuppressWarnings("unchecked")
         Map<String, Map<String, String>> messageMap = (Map<String, Map<String, String>>) XContentMapValues
             .extractValue("model", responseMap);
-        // adding plus one since window delay always rounds up another minute
         assertEquals(
-            String
-                .format(Locale.ROOT, CommonErrorMessages.WINDOW_DELAY_REC, +recDetectorIntervalMinutes + 1, recDetectorIntervalMinutes + 1),
+            String.format(Locale.ROOT, CommonErrorMessages.WINDOW_DELAY_REC, expectedWindowDelayMinutes, expectedWindowDelayMinutes),
             messageMap.get("window_delay").get("message")
         );
     }