From bdcedc24e127394f67bf0bb1555f401e4045a691 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 20 Oct 2021 15:48:40 -0400 Subject: [PATCH] Refactoring manifest to include CI section with image and args Signed-off-by: Andriy Redko --- jenkins/opensearch/Jenkinsfile | 52 +++++++++++++++------------- manifests/1.0.0/opensearch-1.0.0.yml | 1 - manifests/1.0.1/opensearch-1.0.1.yml | 1 - manifests/1.1.0/opensearch-1.1.0.yml | 1 - manifests/1.2.0/opensearch-1.2.0.yml | 5 ++- manifests/2.0.0/opensearch-2.0.0.yml | 7 +++- src/manifests/input_manifest.py | 35 +++++++++++++++++-- 7 files changed, 69 insertions(+), 33 deletions(-) diff --git a/jenkins/opensearch/Jenkinsfile b/jenkins/opensearch/Jenkinsfile index 799bc40a23..54b58832d3 100644 --- a/jenkins/opensearch/Jenkinsfile +++ b/jenkins/opensearch/Jenkinsfile @@ -21,19 +21,27 @@ pipeline { } } } - stage('detect JDK version to build') { + stage('detect Docker image + args to use for the build') { steps { script { manifest = readYaml(file: "manifests/$INPUT_MANIFEST") - jdk = "${manifest.build.jdk}" - // If the 'jdk' key is not present, it is populated with "null" string - if (jdk == null || jdk == "null") { - // If JDK is not set, use 14 - jdk = "14" + dockerImage = "${manifest.ci?.image?.name}" + // If the 'image' key is not present, it is populated with "null" string + if (dockerImage == null || dockerImage == "null") { + // Use default image + dockerImage = 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019' } - echo "Using JDK version: " + jdk + dockerArgs = "${manifest.ci?.image?.args}" + // If the 'args' key is not present, it is populated with "null" string + if (dockerArgs == null || dockerArgs == "null") { + // Use default args + dockerArgs = '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot' + } + + echo "Using Docker image: " + dockerImage + echo "Using Docker container args: " + dockerArgs } } } @@ -46,21 +54,19 @@ pipeline { agent { docker { label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019' + image dockerImage // Unlike freestyle docker, pipeline docker does not login to the container and run commands // It use executes which does not source the docker container internal ENV VAR - args '-e JAVA_HOME=$JAVA' + jdk + '_HOME' + args dockerArgs alwaysPull true } } steps { script { - withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) { - git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' - sh "./build.sh manifests/$INPUT_MANIFEST --snapshot" - withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { - sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/maven') - } + git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' + sh "./build.sh manifests/$INPUT_MANIFEST --snapshot" + withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + sh('$WORKSPACE/publish/publish-snapshot.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/maven') } } } @@ -74,18 +80,16 @@ pipeline { agent { docker { label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019' + image dockerImage // Unlike freestyle docker, pipeline docker does not login to the container and run commands // It use executes which does not source the docker container internal ENV VAR - args '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot' + args dockerArgs alwaysPull true } } steps { script { - withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) { - build() - } + build() } } post() { @@ -98,18 +102,16 @@ pipeline { agent { docker { label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019' + image dockerImage // Unlike freestyle docker, pipeline docker does not login to the container and run commands // It use executes which does not source the docker container internal ENV VAR - args '-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot' + args dockerArgs alwaysPull true } } steps { script { - withEnv(['JAVA_HOME=$JAVA' + jdk + '_HOME']) { - build() - } + build() } } post() { diff --git a/manifests/1.0.0/opensearch-1.0.0.yml b/manifests/1.0.0/opensearch-1.0.0.yml index a272c27042..845783f767 100644 --- a/manifests/1.0.0/opensearch-1.0.0.yml +++ b/manifests/1.0.0/opensearch-1.0.0.yml @@ -3,7 +3,6 @@ schema-version: "1.0" build: name: OpenSearch version: "1.0.0" - jdk: "14" components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/manifests/1.0.1/opensearch-1.0.1.yml b/manifests/1.0.1/opensearch-1.0.1.yml index bf0185f25f..73233c09c4 100644 --- a/manifests/1.0.1/opensearch-1.0.1.yml +++ b/manifests/1.0.1/opensearch-1.0.1.yml @@ -3,7 +3,6 @@ schema-version: "1.0" build: name: OpenSearch version: 1.0.1 - jdk: "14" components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/manifests/1.1.0/opensearch-1.1.0.yml b/manifests/1.1.0/opensearch-1.1.0.yml index d5ce9a87f6..03b808f3ac 100644 --- a/manifests/1.1.0/opensearch-1.1.0.yml +++ b/manifests/1.1.0/opensearch-1.1.0.yml @@ -3,7 +3,6 @@ schema-version: "1.0" build: name: OpenSearch version: 1.1.0 - jdk: "14" components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/manifests/1.2.0/opensearch-1.2.0.yml b/manifests/1.2.0/opensearch-1.2.0.yml index 683256dd25..9a26989e83 100644 --- a/manifests/1.2.0/opensearch-1.2.0.yml +++ b/manifests/1.2.0/opensearch-1.2.0.yml @@ -1,9 +1,12 @@ --- schema-version: "1.0" +ci: + image: + name: "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019" + args: "-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot" build: name: OpenSearch version: 1.2.0 - jdk: "14" components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git diff --git a/manifests/2.0.0/opensearch-2.0.0.yml b/manifests/2.0.0/opensearch-2.0.0.yml index cd651fde5c..2bb38ee48d 100644 --- a/manifests/2.0.0/opensearch-2.0.0.yml +++ b/manifests/2.0.0/opensearch-2.0.0.yml @@ -1,7 +1,12 @@ +--- +schema-version: "1.0" +ci: + image: + name: "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211019" + args: "-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot" build: name: OpenSearch version: 2.0.0 - jdk: "11" components: - name: OpenSearch ref: main diff --git a/src/manifests/input_manifest.py b/src/manifests/input_manifest.py index 85884208b0..5a7d5780fe 100644 --- a/src/manifests/input_manifest.py +++ b/src/manifests/input_manifest.py @@ -36,9 +36,22 @@ class InputManifest(Manifest): "schema": { "name": {"required": True, "type": "string"}, "version": {"required": True, "type": "string"}, - "jdk": {"required": False, "type": "string"}, }, }, + "ci": { + "required": False, + "type": "dict", + "schema": { + "image": { + "required": False, + "type": "dict", + "schema": { + "name": {"required": True, "type": "string"}, + "args": {"required": False, "type": "string"} + } + } + } + }, "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]}, "components": { "type": "list", @@ -62,23 +75,39 @@ def __init__(self, data): super().__init__(data) self.build = self.Build(data["build"]) + self.ci = self.Ci(data.get("ci", None)) self.components = list(map(lambda entry: self.Component(entry), data["components"])) def __to_dict__(self): return { "schema-version": "1.0", "build": self.build.__to_dict__(), + "ci": None if self.ci is None else self.ci.__to_dict__(), "components": list(map(lambda component: component.__to_dict__(), self.components)), } + class Ci: + def __init__(self, data): + self.image = None if data is None else self.Image(data.get("image", None)) + + def __to_dict__(self): + return None if self.image is None else {"image": self.image.__to_dict__()} + + class Image: + def __init__(self, data): + self.name = data["name"] + self.args = data.get("args", None) + + def __to_dict__(self): + return {"name": self.name, "args": self.args} + class Build: def __init__(self, data): self.name = data["name"] self.version = data["version"] - self.jdk = data.get("jdk", None) def __to_dict__(self): - return {"name": self.name, "version": self.version, "jdk": self.jdk} + return {"name": self.name, "version": self.version} class Component: def __init__(self, data):