Merge branch 'opensearch-project:main' into validation

.codecov.yml

@@ -7,4 +7,3 @@ coverage:
 ignore:
   - '**/tests/*.py'
   - '**/test/*.js'
-  - '**/test/*.ts'

.github/ISSUE_TEMPLATE/plugin_onboarding_template.yaml

@@ -0,0 +1,64 @@
+---
+name: On-board plugins
+description: On-board plugins
+title: '[On-boarding]: '
+labels: [on-boarding, untriaged]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to submit on-boarding request!
+  - type: textarea
+    attributes:
+      description: 'Provide the name link of your plugin'
+      label: 'What is the name of your plugin?'
+    id: plugin-name
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Provide the GitHub link for your repository'
+      label: 'What is the link to your GitHub repo?'
+    id: Github-link
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'What is the targeted OpenSearch release version for this plugin?'
+      label: 'Targeted release version'
+    id: expected-release-version
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Is this OpenSearch, OpenSearch-dashboards plugin?'
+      label: 'What type of the plugin are we on-boarding?'
+    id: Type
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Does this plugin have necessary scripts to invoke integration and BWC tests?'
+      label: 'Does this plugin has all the necessary automated tests?'
+    id: tests
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Confirm that all the necessary reviews for this component has been completed.'
+      label: 'Have you completed the required reviews including Security reviews, UX reviews?'
+    id: reviews
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Do we need to enable any automated CVE scanning on this repo?'
+      label: 'Have you on-boarded automated security scanning for the GitHub repo associated with this plugin?'
+    id: repo-scanning
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      description: 'Add any additional context along with contact details for this component here.'
+      label: 'Additional context'
+    id: additional-context

.github/ISSUE_TEMPLATE/standalone_releases_template.yaml

@@ -0,0 +1,72 @@
+---
+name: Release standalone components
+description: Standalone components release template
+title: '[release]: '
+labels: [release, untriaged]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to submit release request!
+  -
+    attributes:
+      description: 'Provide the name of your component'
+      label: 'What is the name of your component?'
+    id: component-name
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Provide the GitHub link for your repository'
+      label: 'What is the link to your GitHub repo?'
+    id: Github-link
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'What is the targeted date for the release?'
+      label: 'Targeted release date'
+    id: expected-release-date
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Does this component need to be published to specific distribution channel(s) like pypi, npm, Docker etc..?'
+      label: 'Where should we publish this component?'
+    id: Type
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Please list the aftifact types along with any other additional information related to generated artifacts?'
+      label: 'What type of artifact(s) will be generated for this component?'
+    id: tests
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Confirm that all the necessary reviews for this component has been completed.'
+      label: 'Have you completed the required reviews including security reviews, UX reviews?'
+    id: reviews
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Do we need to enable any automated CVE scanning on this repo?'
+      label: 'Have you on-boarded automated security scanning for the GitHub repo associated with this component?'
+    id: repo-scanning
+    type: textarea
+    validations:
+      required: true
+  -
+    attributes:
+      description: 'Add any additional context along with contact details for this component here.'
+      label: 'Additional context'
+    id: additional-context
+    type: textarea

DEVELOPER_GUIDE.md

@@ -276,7 +276,29 @@ Each jenkins library should have a test case associated with it. Eg: [TestSignAr
 - Jenkins' library test should extend [BuildPipelineTest.groovy](tests/jenkins/BuildPipelineTest.groovy)
 - Create a dummy job such as [Hello_Jenkinsfile](tests/jenkins/jobs/Hello_Jenkinsfile) to call and test the function
   and output [Hello_Jenkinsfile.txt](tests/jenkins/jobs/Hello_Jenkinsfile.txt)
+- If using remote libs from [opensearch-build-libraries](https://github.com/opensearch-project/opensearch-build-libraries) repository with tag (ex: 1.0.0), make sure
+  both the Jenkins Test file as well as the Jenkins Job file are overriding the libs version with the same tag (ex: 1.0.0), or Jacoco test will fail to generate reports.
+  This would happen if defaultVersion in BuildPipelineTest.groovy (default to 'main') have a different HEAD commit id compares to tag commit id you defined to use.
+```
+super.setUp()
+......
+helper.registerSharedLibrary(
+    library().name('jenkins')
+        .defaultVersion('1.0.0')
+        .allowOverride(true)
+        .implicit(true)
+        .targetPath('vars')
+        .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+        .build()
+)
+```
 
+```
+lib = library(identifier: '[email protected]', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+```
 
 #### Testing in Jenkins
 * [Build_OpenSearch_Dashboards_Jenkinsfile](tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile): is similar to [OpenSearch Dashboards Jenkinsfile](jenkins/opensearch-dashboards/Jenkinsfile) w/o notifications.

build.gradle

@@ -31,7 +31,7 @@ dependencies {
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.9', ext: 'pom'
     implementation group: 'com.cloudbees', name: 'groovy-cps', version: '1.31'
-    testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.32'
+    testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.33'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.4.1'
     testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13'
 }
@@ -40,7 +40,7 @@ configurations.all {
     resolutionStrategy {
         force group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
         force group: 'commons-codec', name: 'commons-codec', version: '1.15'
-        force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.19.3'
+        force group: 'com.google.protobuf', name: 'protobuf-java', version: '3.21.7'
     }
 }
 
@@ -71,7 +71,7 @@ sharedLibrary {
         dependency('org.jenkins-ci.plugins', 'script-security', '1172.v35f6a_0b_8207e')
         dependency('org.jenkins-ci.plugins', 'credentials', '1112.vc87b_7a_3597f6')
         dependency('org.jenkins-ci.plugins', 'git-client', '3.11.1')
-        dependency('org.jenkins-ci.plugins', 'junit', '1.55')
+        dependency('org.jenkins-ci.plugins', 'junit', '1143.v8d9a_e3355270')
         dependency('org.jenkins-ci.plugins', 'mailer', '408.vd726a_1130320') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/mailer/
     }
 }
@@ -117,4 +117,4 @@ jacocoTestReport {
     reports {
         xml.required = true
     }
-}
+}

jenkins/check-for-build.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))
@@ -14,12 +14,12 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
-            H 1 * * * %INPUT_MANIFEST=1.3.6/opensearch-dashboards-1.3.6.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
+            H 1 * * * %INPUT_MANIFEST=1.3.7/opensearch-dashboards-1.3.7.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
+            H 1 * * * %INPUT_MANIFEST=1.3.7/opensearch-1.3.7.yml;TARGET_JOB_NAME=distribution-build-opensearch
             H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-dashboards-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
             H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch
             H 1 * * * %INPUT_MANIFEST=2.4.0/opensearch-dashboards-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
-            H 1 * * * %INPUT_MANIFEST=2.4.0/opensearch-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
-            H 1 * * * %INPUT_MANIFEST=1.3.6/opensearch-1.3.6.yml;TARGET_JOB_NAME=distribution-build-opensearch
+            H 1 * * * %INPUT_MANIFEST=2.4.0/opensearch-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
         '''
@@ -40,6 +40,11 @@ pipeline {
             description: 'Job to trigger if build has changed',
             trim: true
         )
+        string(
+            name: 'BUILD_PLATFORM',
+            description: 'Platform to build',
+            trim: true
+        )
     }
     stages {
         stage('detect docker image + args') {
@@ -81,7 +86,8 @@ pipeline {
                             }
                             build job: "${TARGET_JOB_NAME}", parameters: [
                             string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
-                            string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}")
+                            string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"),
+                            string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}")
                             ], wait: true
 
                             echo "Build succeeded, uploading build SHA for that job"

jenkins/cross-cluster-replication/perf-test.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))
@@ -11,6 +11,8 @@ pipeline {
     parameters {
         string(name: 'VERSION', description: 'The version of Data Prepper', trim: true)
         string(name: 'DATA_PREPPER_BUILD_NUMBER', description: 'The build number of the Data Prepper build from GitHub to release.', trim: true)
+        booleanParam(name: 'RELEASE_MAJOR_TAG', description: 'Whether to create major tag of docker image or not', defaultValue: false)
+        booleanParam(name: 'RELEASE_LATEST_TAG', description: 'Whether to create latest tag of docker image or not', defaultValue: false)
     }
     environment {
         DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site')
@@ -92,6 +94,29 @@ pipeline {
                                     string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
                                     string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
                                 ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize(".")[0].trim()
+                                def dockerCopyHubMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyHubLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:latest")
+                                    ]
+                            }
                         }
                     }
                 }
@@ -106,6 +131,29 @@ pipeline {
                                     string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
                                     string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
                                 ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize(".")[0].trim()
+                                def dockerCopyECRMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyECRLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:latest")
+                                    ]
+                            }
                         }
                     }
                 }

jenkins/docker/docker-copy.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/docker/docker-scan.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/gradle/gradle-check.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/opensearch-dashboards/bwc-test.jenkinsfile

@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].0', retriever: modernSCM([
+lib = library(identifier: '[email protected].4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))