Refactoring manifest to include CI section with image and args

Signed-off-by: Andriy Redko <[email protected]>
opensearch-project · Oct 20, 2021 · bdcedc2 · bdcedc2
1 parent fd024c9
commit bdcedc2
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 33 deletions.
diff --git a/jenkins/opensearch/Jenkinsfile b/jenkins/opensearch/Jenkinsfile
@@ -21,19 +21,27 @@ pipeline {
                 }
             }
         }
-        stage('detect JDK version to build') {
+        stage('detect Docker image + args to use for the build') {
             steps {
                 script { 
                     manifest = readYaml(file: "manifests/$INPUT_MANIFEST") 
 
-                    jdk = "${manifest.build.jdk}" 
-                    // If the 'jdk' key is not present, it is populated with "null" string 
-                    if (jdk == null || jdk == "null") {
-                        // If JDK is not set, use 14
-                        jdk = "14"
+                    dockerImage = "${manifest.ci?.image?.name}" 
+                    // If the 'image' key is not present, it is populated with "null" string 
+                    if (dockerImage == null || dockerImage == "null") {
+                        // Use default image
+                        dockerImage = 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019'
                     }
 
-                    echo "Using JDK version: " + jdk
+                    dockerArgs = "${manifest.ci?.image?.args}" 
+                    // If the 'args' key is not present, it is populated with "null" string 
+                    if (dockerArgs == null || dockerArgs == "null") {
+                        // Use default args
+                        dockerArgs = '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot'
+                    }
+
+                    echo "Using Docker image: " + dockerImage
+                    echo "Using Docker container args: " + dockerArgs
                 }
             }
         }
@@ -46,21 +54,19 @@ pipeline {
                     agent {
                         docker {
                             label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                            image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019'
+                            image dockerImage
                             // Unlike freestyle docker, pipeline docker does not login to the container and run commands
                             // It use executes which does not source the docker container internal ENV VAR
-                            args '-e JAVA_HOME=$JAVA' + jdk + '_HOME'
+                            args dockerArgs
                             alwaysPull true
                         }
                     }
                     steps {
                         script {
-                            withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) {
-                                git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
-                                sh "./build.sh manifests/$INPUT_MANIFEST --snapshot"
-                                withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) {
-                                    sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/maven')
-                                }
+                            git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
+                            sh "./build.sh manifests/$INPUT_MANIFEST --snapshot"
+                            withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) {
+                                sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/maven')
                             }
                         }
                     }
@@ -74,18 +80,16 @@ pipeline {
                     agent {
                         docker {
                             label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                            image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019'
+                            image dockerImage
                             // Unlike freestyle docker, pipeline docker does not login to the container and run commands
                             // It use executes which does not source the docker container internal ENV VAR
-                            args '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot'
+                            args dockerArgs
                             alwaysPull true
                         }
                     }
                     steps {
                         script {
-                            withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) {
-                                build()
-                            }
+                            build()
                         }
                     }
                     post() {
@@ -98,18 +102,16 @@ pipeline {
                     agent {
                         docker {
                             label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
-                            image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019'
+                            image dockerImage
                             // Unlike freestyle docker, pipeline docker does not login to the container and run commands
                             // It use executes which does not source the docker container internal ENV VAR
-                            args '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot'
+                            args dockerArgs
                             alwaysPull true
                         }
                     }
                     steps {
                         script {
-                            withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) {
-                                build()
-                            }
+                            build()
                         }
                     }
                     post() {

diff --git a/manifests/1.0.0/opensearch-1.0.0.yml b/manifests/1.0.0/opensearch-1.0.0.yml
@@ -3,7 +3,6 @@ schema-version: "1.0"
 build:
   name: OpenSearch
   version: "1.0.0"
-  jdk: "14"
 components:
   - name: OpenSearch
     repository: https://github.com/opensearch-project/OpenSearch.git

diff --git a/manifests/1.0.1/opensearch-1.0.1.yml b/manifests/1.0.1/opensearch-1.0.1.yml
@@ -3,7 +3,6 @@ schema-version: "1.0"
 build:
   name: OpenSearch
   version: 1.0.1
-  jdk: "14"
 components:
   - name: OpenSearch
     repository: https://github.com/opensearch-project/OpenSearch.git

diff --git a/manifests/1.1.0/opensearch-1.1.0.yml b/manifests/1.1.0/opensearch-1.1.0.yml
@@ -3,7 +3,6 @@ schema-version: "1.0"
 build:
   name: OpenSearch
   version: 1.1.0
-  jdk: "14"
 components:
   - name: OpenSearch
     repository: https://github.com/opensearch-project/OpenSearch.git

diff --git a/manifests/1.2.0/opensearch-1.2.0.yml b/manifests/1.2.0/opensearch-1.2.0.yml
@@ -1,9 +1,12 @@
 ---
 schema-version: "1.0"
+ci:
+  image:
+     name: "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019"
+     args: "-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot"
 build:
   name: OpenSearch
   version: 1.2.0
-  jdk: "14"
 components:
   - name: OpenSearch
     repository: https://github.com/opensearch-project/OpenSearch.git

diff --git a/manifests/2.0.0/opensearch-2.0.0.yml b/manifests/2.0.0/opensearch-2.0.0.yml
@@ -1,7 +1,12 @@
+---
+schema-version: "1.0"
+ci:
+  image:
+     name: "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019"
+     args: "-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot"
 build:
   name: OpenSearch
   version: 2.0.0
-  jdk: "11"
 components:
 - name: OpenSearch
   ref: main

diff --git a/src/manifests/input_manifest.py b/src/manifests/input_manifest.py
@@ -36,9 +36,22 @@ class InputManifest(Manifest):
             "schema": {
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
-                "jdk": {"required": False, "type": "string"},
             },
         },
+        "ci": {
+            "required": False,
+            "type": "dict",
+            "schema": {
+                "image": {
+                    "required": False,
+                    "type": "dict",
+                    "schema": {
+                        "name": {"required": True, "type": "string"},
+                        "args": {"required": False, "type": "string"}
+                    }
+                }
+            }
+        },
         "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]},
         "components": {
             "type": "list",
@@ -62,23 +75,39 @@ def __init__(self, data):
         super().__init__(data)
 
         self.build = self.Build(data["build"])
+        self.ci = self.Ci(data.get("ci", None))
         self.components = list(map(lambda entry: self.Component(entry), data["components"]))
 
     def __to_dict__(self):
         return {
             "schema-version": "1.0",
             "build": self.build.__to_dict__(),
+            "ci": None if self.ci is None else self.ci.__to_dict__(),
             "components": list(map(lambda component: component.__to_dict__(), self.components)),
         }
 
+    class Ci:
+        def __init__(self, data):
+            self.image = None if data is None else self.Image(data.get("image", None))
+
+        def __to_dict__(self):
+            return None if self.image is None else {"image": self.image.__to_dict__()} 
+
+        class Image:
+            def __init__(self, data):
+                self.name = data["name"]
+                self.args = data.get("args", None)
+
+            def __to_dict__(self):
+                return {"name": self.name, "args": self.args}
+
     class Build:
         def __init__(self, data):
             self.name = data["name"]
             self.version = data["version"]
-            self.jdk = data.get("jdk", None)
 
         def __to_dict__(self):
-            return {"name": self.name, "version": self.version, "jdk": self.jdk}
+            return {"name": self.name, "version": self.version}
 
     class Component:
         def __init__(self, data):