Add Windows IntegTest support on Jenkins Libs (#308)

Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Sep 13, 2023 · 96b32c7 · 96b32c7
1 parent e4e7078
commit 96b32c7
Show file tree

Hide file tree

Showing 11 changed files with 3,218 additions and 7 deletions.
diff --git a/build.gradle b/build.gradle
@@ -120,7 +120,7 @@ jacocoTestReport {
     }
 }
 
-String version = '5.8.0'
+String version = '5.9.0'
 
 task updateVersion {
     doLast {

diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy
@@ -165,6 +165,10 @@ class BuildManifest implements Serializable {
         return this.build.architecture
     }
 
+    public String getArtifactPlatform() {
+        return this.build.platform
+    }
+
     public String getArtifactBuildId() {
         return this.build.id
     }

diff --git a/tests/data/opensearch-2.8.0-build-windows.yml b/tests/data/opensearch-2.8.0-build-windows.yml
diff --git a/tests/data/opensearch-2.8.0-test.yml b/tests/data/opensearch-2.8.0-test.yml
@@ -0,0 +1,132 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1]
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: cross-cluster-replication
+    integ-test:
+      topology:
+        - cluster_name: leader
+          data_nodes: 1
+          cluster_manager_nodes: 0
+        - cluster_name: follower
+          data_nodes: 1
+          cluster_manager_nodes: 0
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: geospatial
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo: [/tmp]
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: neural-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: notifications
+    working-directory: notifications
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: opensearch-observability
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: opensearch-reports
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: security
+    integ-test:
+      test-configs:
+        - with-security
+
+  - name: security-analytics
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+    bwc-test:
+      test-configs:
+        - with-security
diff --git a/tests/jenkins/TestRunIntegTestScript.groovy b/tests/jenkins/TestRunIntegTestScript.groovy
@@ -79,6 +79,23 @@ class TestRunIntegTestScript extends BuildPipelineTest {
         super.testPipeline("tests/jenkins/jobs/RunIntegTestScript_LocalPath_Switch_Non_Root_Jenkinsfile")
     }
 
+    @Test
+    void 'IntegTest Windows Zip'() {
+        this.registerLibTester(new RunIntegTestScriptLibTester(
+                'dummy_job',
+                'OpenSearch',
+                'tests/data/opensearch-2.8.0-build-windows.yml',
+                'tests/data/opensearch-2.8.0-test.yml',
+                '',
+                '',
+            )
+        )
+        super.setUp()
+        runScript("tests/jenkins/jobs/RunIntegTestScript_Windows_Jenkinsfile")
+        assertThat(getShellCommands('sh', 'test.sh'), hasItems('env PATH=$PATH  ./test.sh integ-test tests/data/opensearch-2.8.0-test.yml --component OpenSearch --test-run-id 987 --paths opensearch=https://ci.opensearch.org/ci/dbc/dummy_job/2.8.0/7923/windows/x64/zip --base-path https://dummy_link/dummy_integ_test/2.8.0/7923/windows/x64/zip '))
+
+    }
+
     @Test
     void 'IntegTest LocalPath SwitchNonRoot=false'() {
         this.registerLibTester(new RunIntegTestScriptLibTester(

diff --git a/tests/jenkins/jobs/RunIntegTestScript_Jenkinsfile.txt b/tests/jenkins/jobs/RunIntegTestScript_Jenkinsfile.txt
@@ -8,8 +8,10 @@
                   runIntegTestScript.library({identifier=jenkins@main, retriever=null})
                   runIntegTestScript.readYaml({file=tests/data/opensearch-1.3.0-build.yml})
                   BuildManifest.asBoolean()
+                  BuildManifest.getArtifactArchitecture()
                   BuildManifest.getDistribution()
-                  runIntegTestScript.echo(Start integTest for distribution type: tar)
+                  BuildManifest.getArtifactPlatform()
+                  runIntegTestScript.echo(Start integTest on: tar x64 linux)
                   runIntegTestScript.echo(Possible Java Home: )
                   runIntegTestScript.echo(Build Id: c3ff7a232d25403fa8cc14c97799c323)
                   BuildManifest.getArtifactRootUrl(dummy_job, c3ff7a232d25403fa8cc14c97799c323)

diff --git a/tests/jenkins/jobs/RunIntegTestScript_LocalPath_Jenkinsfile.txt b/tests/jenkins/jobs/RunIntegTestScript_LocalPath_Jenkinsfile.txt
@@ -8,8 +8,10 @@
                   runIntegTestScript.library({identifier=jenkins@main, retriever=null})
                   runIntegTestScript.readYaml({file=tests/data/opensearch-1.3.0-build.yml})
                   BuildManifest.asBoolean()
+                  BuildManifest.getArtifactArchitecture()
                   BuildManifest.getDistribution()
-                  runIntegTestScript.echo(Start integTest for distribution type: tar)
+                  BuildManifest.getArtifactPlatform()
+                  runIntegTestScript.echo(Start integTest on: tar x64 linux)
                   runIntegTestScript.echo(Possible Java Home: )
                   runIntegTestScript.echo(Build Id: c3ff7a232d25403fa8cc14c97799c323)
                   BuildManifest.getArtifactRootUrl(dummy_job, c3ff7a232d25403fa8cc14c97799c323)

diff --git a/tests/jenkins/jobs/RunIntegTestScript_LocalPath_Switch_Non_Root_Jenkinsfile.txt b/tests/jenkins/jobs/RunIntegTestScript_LocalPath_Switch_Non_Root_Jenkinsfile.txt
@@ -8,8 +8,10 @@
                   runIntegTestScript.library({identifier=jenkins@main, retriever=null})
                   runIntegTestScript.readYaml({file=tests/data/opensearch-1.3.0-build.yml})
                   BuildManifest.asBoolean()
+                  BuildManifest.getArtifactArchitecture()
                   BuildManifest.getDistribution()
-                  runIntegTestScript.echo(Start integTest for distribution type: tar)
+                  BuildManifest.getArtifactPlatform()
+                  runIntegTestScript.echo(Start integTest on: tar x64 linux)
                   runIntegTestScript.detectTestDockerAgent({testManifest=tests/data/opensearch-1.3.0-test.yml})
                      detectTestDockerAgent.legacySCM(groovy.lang.Closure)
                      detectTestDockerAgent.library({identifier=jenkins@main, retriever=null})

diff --git a/tests/jenkins/jobs/RunIntegTestScript_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/RunIntegTestScript_OpenSearch_Dashboards_Jenkinsfile.txt
@@ -8,8 +8,10 @@
                   runIntegTestScript.library({identifier=jenkins@main, retriever=null})
                   runIntegTestScript.readYaml({file=tests/data/opensearch-dashboards-1.2.0-build.yml})
                   BuildManifest.asBoolean()
+                  BuildManifest.getArtifactArchitecture()
                   BuildManifest.getDistribution()
-                  runIntegTestScript.echo(Start integTest for distribution type: tar)
+                  BuildManifest.getArtifactPlatform()
+                  runIntegTestScript.echo(Start integTest on: tar x64 linux)
                   runIntegTestScript.echo(Possible Java Home: )
                   runIntegTestScript.echo(Build Id: 215)
                   BuildManifest.getArtifactRootUrl(dummy_job, 215)

diff --git a/tests/jenkins/jobs/RunIntegTestScript_Windows_Jenkinsfile b/tests/jenkins/jobs/RunIntegTestScript_Windows_Jenkinsfile
@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('integ-test') {
+            steps {
+                script {
+                    runIntegTestScript(
+                        jobName: 'dummy_job',
+                        componentName: 'OpenSearch',
+                        buildManifest: 'tests/data/opensearch-2.8.0-build-windows.yml',
+                        testManifest: 'tests/data/opensearch-2.8.0-test.yml',
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/vars/runIntegTestScript.groovy b/vars/runIntegTestScript.groovy
@@ -12,10 +12,19 @@ void call(Map args = [:]) {
     String jobName = args.jobName ?: 'distribution-build-opensearch'
     def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.buildManifest))
 
-    echo "Start integTest for distribution type: " + buildManifest.getDistribution()
+    String architecture = buildManifest.getArtifactArchitecture()
+    String distribution = buildManifest.getDistribution()
+    String platform = buildManifest.getArtifactPlatform()
+    echo "Start integTest on: " + distribution + " " + architecture + " " + platform
 
-    javaVersion = (jobName.equals('distribution-build-opensearch')) ? detectTestDockerAgent(testManifest: args.testManifest).javaVersion : ''
+    def javaVersion = (jobName.equals('distribution-build-opensearch')) ? detectTestDockerAgent(testManifest: args.testManifest).javaVersion : ''
     String javaHomeCommand = (jobName.equals('distribution-build-opensearch') && ! javaVersion.equals('')) ? "JAVA_HOME=/opt/java/${javaVersion}" : ''
+    if (platform.equals('windows')) { // Windows use scoop to switch the Java Version
+        String javaVersionNumber = javaVersion.replaceAll("[^0-9]", "") // Only get number
+        echo("Switching to Java ${javaVersionNumber} on Windows Docker Container")
+        sh("scoop reset `scoop list jdk | grep ${javaVersionNumber} | head -1 | cut -d ' ' -f1`")
+        javaHomeCommand = ''
+    }
     echo "Possible Java Home: ${javaHomeCommand}"
 
     String buildId = buildManifest.build.id