resolve conflicts

.buildkite/hooks/pre-command

@@ -12,6 +12,7 @@ export UPLOAD_SAFE_LOGS=${UPLOAD_SAFE_LOGS:-"0"}
 export SERVERLESS=${SERVERLESS:-"false"}
 export STACK_VERSION=${STACK_VERSION:-""}
 export FORCE_CHECK_ALL=${FORCE_CHECK_ALL:-"false"}
+export PUBLISH_COVERAGE_REPORTS=${PUBLISH_COVERAGE_REPORTS:-"false"}
 
 BASE_DIR=$(pwd)
 export BASE_DIR
@@ -37,13 +38,52 @@ GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 
+# variables required for terraform
+export ENVIRONMENT="ci"
+export REPO="${REPO_NAME}"
+
+branch_name_label() {
+    local branch="$1"
+
+    if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]] ; then
+        # remove fork from branch name
+        branch=$(echo $branch | cut -d : -f 2)
+    fi
+
+    # From Jenkins
+    # BRANCH_NAME_LOWER_CASE = "${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}"
+    # to lower case and replace characters
+    branch=$(echo "$branch" | tr '[:upper:]' '[:lower:]' | tr '_/\:. ' '-')
+
+    # truncate up to 63 characters limit
+    echo $branch | head -c 63
+}
+
+BRANCH_NAME_LOWER_CASE=$(branch_name_label "$BUILDKITE_BRANCH")
+export BRANCH_NAME_LOWER_CASE
+# This variable contains the build number https://buildkite.com/elastic/elastic-package/<number>
+export BUILD_ID="${BUILDKITE_BUILD_NUMBER}"
+# get current timestamp in milliseconds
+# From Jenkins
+# CREATED_DATE = "${new Date().getTime()}"
+CREATED_DATE=$(date +%s%3N)
+export CREATED_DATE
+
 if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     # first upload pipeline does not have the environment variables defined in the YAML
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
+    if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
+        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
+        export BUILDKITE_API_TOKEN
+    fi
+fi
+
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
+        # TODO: To be removed
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
         export BUILDKITE_API_TOKEN
     fi

.buildkite/pipeline.publish.yml

@@ -0,0 +1,49 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.2"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "false"
+  YQ_VERSION: 'v4.35.2'
+  JQ_VERSION: '1.7'
+  # Elastic package settings
+  # Manage docker output/logs
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  # Default license to use by `elastic-package build`
+  ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
+  # Link definitions path (full path to be set in the corresponding step)
+  ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
+  # Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
+  ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
+  NOTIFY_TO: "[email protected]"
+
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: ":package: Build packages"
+    key: "build-packages"
+    command: ".buildkite/scripts/build_packages.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "8G"
+    env:
+      ARTIFACTS_FOLDER: "artifacts-to-sign"
+      DRY_RUN: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+    artifact_paths:
+      - artifacts-to-sign/*.zip
+
+notify:
+  - email: "$NOTIFY_TO"
+    if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"

.buildkite/pipeline.schedule-daily.yml

@@ -22,19 +22,21 @@ steps:
         SERVERLESS: "false"
         SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 7.17-SNAPSHOT
+        # STACK_VERSION: 7.17-SNAPSHOT # Using 7.17.19 till https://github.com/elastic/fleet-server/issues/3435 is solved.
+        STACK_VERSION: 7.17.19
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.12"
+  - label: "Check integrations local stacks - Stack Version v8.14"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.12-SNAPSHOT
+        STACK_VERSION: 8.14.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false
@@ -58,3 +60,10 @@ steps:
     depends_on:
       - step: "check"
         allow_failure: false
+
+  - label: ":package: Publish missing packages"
+    key: "trigger-integrations-publish"
+    trigger: "integrations-publish"
+    depends_on:
+      - step: "check"
+        allow_failure: false

.buildkite/pipeline.serverless.yml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  SETUP_GVM_VERSION: "v0.5.1"
+  SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "false"

.buildkite/pipeline.yml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  SETUP_GVM_VERSION: "v0.5.1"
+  SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "false"
@@ -67,6 +67,15 @@ steps:
       cpu: "8"
       memory: "4G"
 
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command: ".buildkite/scripts/run_sonar_scanner.sh"
+    artifact_paths:
+      - build/test-coverage/coverage_merged.xml
+
   - label: ":junit: Junit annotate"
     plugins:
       - junit-annotate#v2.4.1:

.buildkite/pull-requests.json

@@ -47,6 +47,22 @@
       "skip_target_branches": [],
       "skip_ci_on_only_changed": [],
       "always_require_ci_on_changed": []
+    },
+    {
+      "enabled": false,
+      "pipelineSlug": "integrations-publish",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": [],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "trigger_comment_regex": "",
+      "always_trigger_comment_regex": "",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
+      "always_require_ci_on_changed": []
     }
   ]
 }

.buildkite/scripts/backport_branch.sh

@@ -84,6 +84,7 @@ removeOtherPackages() {
 updateBackportBranchContents() {
   local BUILDKITE_FOLDER_PATH=".buildkite"
   local JENKINS_FOLDER_PATH=".ci"
+  local files_cached_num=""
   if git ls-tree -d --name-only main:.ci >/dev/null 2>&1; then
     git checkout $BACKPORT_BRANCH_NAME
     echo "Copying $BUILDKITE_FOLDER_PATH from $SOURCE_BRANCH..."
@@ -108,15 +109,26 @@ updateBackportBranchContents() {
   git config --global user.name "${GITHUB_USERNAME_SECRET}"
   git config --global user.email "${GITHUB_EMAIL_SECRET}"
 
+  echo "Commiting"
+  git add $BUILDKITE_FOLDER_PATH
+  if [ -d "${JENKINS_FOLDER_PATH}" ]; then
+    git add $JENKINS_FOLDER_PATH
+  fi
+  git add $PACKAGES_FOLDER_PATH/
+  git status
+
+  files_cached_num=$(git diff --name-only --cached | wc -l)
+  if [ "${files_cached_num}" -gt 0 ]; then
+    git commit -m "Add $BUILDKITE_FOLDER_PATH and $JENKINS_FOLDER_PATH to backport branch: $BACKPORT_BRANCH_NAME from the $SOURCE_BRANCH branch"
+  else
+    echo "Nothing to commit, skip."
+  fi
+
   if [ "$DRY_RUN" == "true" ];then
     echo "DRY_RUN mode, nothing will be pushed."
     git diff $SOURCE_BRANCH...$BACKPORT_BRANCH_NAME
   else
-    echo "Commiting and pushing..."
-    git add $BUILDKITE_FOLDER_PATH
-    git add $JENKINS_FOLDER_PATH
-    git add $PACKAGES_FOLDER_PATH/
-    git commit -m "Add $BUILDKITE_FOLDER_PATH and $JENKINS_FOLDER_PATH to backport branch: $BACKPORT_BRANCH_NAME from the $SOURCE_BRANCH branch"
+    echo "Pushing..."
     git push origin $BACKPORT_BRANCH_NAME
   fi
 }
@@ -152,15 +164,15 @@ if branchExist "$BACKPORT_BRANCH_NAME"; then
 fi
 
 # backport branch does not exist, running checks and create branch
-echo "Check the entered version and PACKAGE_VERSION are equal"
-version="$(cat packages/${PACKAGE_NAME}/manifest.yml | yq -r .version)"
+version="$(git show "${BASE_COMMIT}":"packages/${PACKAGE_NAME}/manifest.yml" | yq -r .version)"
+echo "Check if version from ${BASE_COMMIT} (${version}) matches with version from input step ${PACKAGE_VERSION}"
 if [[ "${version}" != "${PACKAGE_VERSION}" ]]; then
   buildkite-agent annotate "Unexpected version found in packages/${PACKAGE_NAME}/manifest.yml" --style "error"
   exit 1
 fi
 
 echo "Check that this changeset is the one creating the version $PACKAGE_NAME"
-if ! git show -p ${BASE_COMMIT} packages/${PACKAGE_NAME}/manifest.yml | grep -E "^\+version: ${PACKAGE_VERSION}" ; then
+if ! git show -p ${BASE_COMMIT} packages/${PACKAGE_NAME}/manifest.yml | grep -E "^\+version: \"{0,1}${PACKAGE_VERSION}" ; then
   buildkite-agent annotate "This changeset does not creates the version ${PACKAGE_VERSION}" --style "error"
   exit 1
 fi
@@ -172,4 +184,7 @@ MSG="The backport branch: **$BACKPORT_BRANCH_NAME** has been created."
 echo "Adding CI files into the branch ${BACKPORT_BRANCH_NAME}"
 updateBackportBranchContents
 
+if [ "${DRY_RUN}" == "true" ]; then
+  MSG="[DRY_RUN] ${MSG}."
+fi
 buildkite-agent annotate "$MSG" --style "success"