[Build] Add OpenSearch Dashboards to Jenkinsfile (#717)

* [Build] create Dashboards Jenkinsfile and move OpenSearch Jenkinsfile

Creates OpenSearch Dashboards Jenkinsfile basically copying what was
in the existing OpenSearch Jenkinsfile. But skipping snapshots since
it was not doing anything with snapshots and publishing to S3
bucket paths of `/builds/dashboards/` and `/bundles/dashboards/` which
has not been verified yet. It was done this way, while keeping OpenSearch
to build to `/builds` and `/bundles`, to reduce the impact of adding
OpenSearch Dashboards.

For consistency, moved the root lvl Jenkinsfile which was only  building
OpenSearch to `jenkins/opensearch`.

Issue resolved:
#620

Signed-off-by: Kawika Avilla <[email protected]>

* [Build] remove Java arg, x86 to x64, s3 bucket rename

Handling the feedback without any question:

* Remove JAVA arg since it is unneeded for OpenSearch Dashboards.
* Change from x86 to x64
* Change S3 bucket from dashboards to opensearch-dashboards

Signed-off-by: Kawika Avilla <[email protected]>

* [Build] rename folder and update build id for dashboards

Update from feedback with potential to require more discussion.
* Rename folder from jenkins/opensearch_dashboards to jenkins/opensearch-dashboards
* Build ID from OPENSEARCH_BUILD_ID to OPENSEARCH_DASHBOARDS_BUILD_ID

Signed-off-by: Kawika Avilla <[email protected]>

jenkins/opensearch-dashboards/Jenkinsfile

@@ -0,0 +1,137 @@
+pipeline {
+    agent none
+    environment {
+        OPENSEARCH_DASHBOARDS_BUILD_ID = "${BUILD_NUMBER}"
+    }
+    triggers {
+      parameterizedCron '''
+        H */2 * * * %INPUT_MANIFEST=1.1.0/opensearch-dashboards-1.1.0.yml
+      '''
+    }
+    stages {
+        stage('parameters') {
+            steps {
+                script {
+                    properties([
+                            parameters([
+                                    string(
+                                            defaultValue: '',
+                                            name: 'INPUT_MANIFEST',
+                                            trim: true
+                                    )
+                            ])
+                    ])
+                }
+            }
+        }
+        stage('build') {
+            parallel {
+                stage('build-x64') {
+                    agent {
+                        docker {
+                            label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
+                            image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdk14-node10.24.1-cypress6.9.1-20211005'
+                            alwaysPull true
+                        }
+                    }
+                    steps {
+                        script {
+                            build()
+                        }
+                    }
+                    post() {
+                        always {
+                            cleanWs disableDeferredWipeout: true, deleteDirs: true
+                        }
+                    }
+                }
+                stage('build-arm64') {
+                    agent {
+                        docker {
+                            label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
+                            image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdk14-node10.24.1-cypress6.9.1-20211005'
+                            alwaysPull true
+                        }
+                    }
+                    steps {
+                        script {
+                            build()
+                        }
+                    }
+                    post() {
+                        always {
+                            cleanWs disableDeferredWipeout: true, deleteDirs: true
+                        }
+                    }
+                }
+            }
+            post() {
+                success {
+                    node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') {
+                        publishNotification(":white_check_mark:", "Successful Build", "\n${getAllJenkinsMessages()}")
+                    }
+                }
+                failure {
+                    node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') {
+                        publishNotification(":warning:", "Failed Build", "")
+                    }
+                }
+            }
+        }
+    }
+}
+
+void build() {
+    git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
+
+    sh "./build.sh manifests/$INPUT_MANIFEST"
+    sh './assemble.sh artifacts/manifest.yml'
+
+    script { manifest = readYaml(file: 'artifacts/manifest.yml') }
+    def artifactPath = "${manifest.build.version}/${OPENSEARCH_DASHBOARDS_BUILD_ID}/${manifest.build.architecture}";
+
+    withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') {
+        s3Upload(file: 'artifacts', bucket: "${ARTIFACT_BUCKET_NAME}", path: "builds/opensearch-dashboards/${artifactPath}")
+        s3Upload(file: "bundle", bucket: "${ARTIFACT_BUCKET_NAME}", path: "bundles/opensearch-dashboards/${artifactPath}")
+    }
+
+    addJenkinsMessage("${PUBLIC_ARTIFACT_URL}/builds/opensearch-dashboards/${artifactPath}/manifest.yml\n" +
+                      "${PUBLIC_ARTIFACT_URL}/bundles/opensearch-dashboards/${artifactPath}/manifest.yml")
+}
+
+/** Publishes a notification to a slack instance*/
+void publishNotification(icon, msg, extra) {
+    withCredentials([string(credentialsId: 'BUILD_NOTICE_WEBHOOK', variable: 'TOKEN')]) {
+        sh("""curl -XPOST --header "Content-Type: application/json" --data '{"result_text": "$icon ${env.JOB_NAME} [${env.BUILD_NUMBER}] $msg ${env.BUILD_URL}\nManifest: ${INPUT_MANIFEST} $extra"}' """ + "$TOKEN")
+    }
+}
+
+/** Add a message to the jenkins queue */
+void addJenkinsMessage(message) {
+    writeFile(file: "notifications/${STAGE_NAME}.msg", text: message)
+    stash(includes: "notifications/*" , name: "notifications-${STAGE_NAME}")
+}
+
+/** Load all message in the jenkins queue and append them with a leading newline into a mutli-line string */
+String getAllJenkinsMessages() {
+    script {
+        // Stages must be explicitly added to prevent overwriting
+        // See https://ryan.himmelwright.net/post/jenkins-parallel-stashing/
+        def stages = ['build-x64', 'build-arm64']
+        for (stage in stages) {
+            unstash "notifications-${stage}"
+        }
+
+        def files = findFiles(excludes: '', glob: 'notifications/*')
+        def data = ""
+        for (file in files) {
+            data = data + "\n" + readFile (file: file.path)
+        }
+
+        // Delete all the notifications from the workspace
+        dir('notifications') {
+            deleteDir()
+        }
+        return data
+    }
+}

src/build_workflow/build_target.py

@@ -27,7 +27,7 @@ def __init__(
         build_id=None,
         output_dir="artifacts",
     ):
-        self.build_id = os.getenv("OPENSEARCH_BUILD_ID") or build_id or uuid.uuid4().hex
+        self.build_id = os.getenv("OPENSEARCH_BUILD_ID") or os.getenv("OPENSEARCH_DASHBOARDS_BUILD_ID") or build_id or uuid.uuid4().hex
         self.name = name
         self.version = version
         self.snapshot = snapshot

tests/tests_build_workflow/test_build_target.py

@@ -24,6 +24,10 @@ def test_build_id_hex(self):
     def test_build_id_from_env(self):
         self.assertEqual(BuildTarget(version="1.1.0", arch="x86").build_id, "id")
 
+    @patch.dict(os.environ, {"OPENSEARCH_DASHBOARDS_BUILD_ID": "id"})
+    def test_build_id_from_dashboards_env(self):
+        self.assertEqual(BuildTarget(version="1.1.0", arch="x86").build_id, "id")
+
     def test_build_id_from_arg(self):
         self.assertEqual(
             BuildTarget(version="1.1.0", arch="x86", build_id="id").build_id, "id"