[7.17](backport #29710) use main default branch

.ci/apm-beats-update.groovy

@@ -81,7 +81,7 @@ pipeline {
           when {
             beforeAgent true
             anyOf {
-              branch 'master'
+              branch 'main'
               branch "\\d+\\.\\d+"
               branch "v\\d?"
               tag "v\\d+\\.\\d+\\.\\d+*"

.ci/beats-tester-bc.groovy

@@ -0,0 +1,47 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main'
+    BASE_URL = "https://staging.elastic.co/${params.version}/downloads"
+    APM_BASE_URL = "${env.BASE_URL}/apm-server"
+    BEATS_BASE_URL = "${env.BASE_URL}/beats"
+    VERSION = "${params.version?.split('-')[0]}"
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  parameters {
+    string(name: 'version', defaultValue: '', description: 'Id of the Build Candidate (7.10.0-b55684ff).')
+    string(name: 'BRANCH_REFERENCE', defaultValue: 'main', description: 'Branch to grab the Groovy script(for test changes).')
+  }
+  stages {
+    stage('Run Beat Tester') {
+      options { skipDefaultCheckout() }
+      when {
+        expression {
+          return '' != "${VERSION}"
+        }
+      }
+      steps {
+        build(job: env.BEATS_TESTER_JOB, propagate: true, wait: true,
+          parameters: [
+            string(name: 'APM_URL_BASE', value: "${APM_BASE_URL}"),
+            string(name: 'BEATS_URL_BASE', value: "${BEATS_BASE_URL}"),
+            string(name: 'VERSION', value: "${VERSION}")
+        ])
+      }
+    }
+  }
+}

.ci/beats-tester.groovy

@@ -50,9 +50,9 @@ pipeline {
             setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim())
           }
         }
-        stage('Build master') {
+        stage('Build main') {
           options { skipDefaultCheckout() }
-          when { branch 'master' }
+          when { branch 'main' }
           steps {
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
@@ -78,7 +78,7 @@ pipeline {
           when {
             not {
               anyOf {
-                branch comparator: 'REGEXP', pattern: '(master|.*x)'
+                branch comparator: 'REGEXP', pattern: '(main|.*x)'
                 changeRequest()
               }
             }

.ci/build-docker-images.groovy

@@ -0,0 +1,111 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-20' }
+  environment {
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    HOME = "${env.WORKSPACE}"
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(0-5) * * 1-5'
+  }
+  parameters {
+    booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images")
+    string(name: 'BRANCH_REFERENCE', defaultValue: "main", description: "Git branch/tag to use")
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "${params.BRANCH_REFERENCE}",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
+        dir("${BASE_DIR}"){
+          setEnvVar("GO_VERSION", readFile(file: ".go-version")?.trim())
+        }
+      }
+    }
+    stage('Metricbeat Test Docker images'){
+      options {
+        warnError('Metricbeat Test Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        withMageEnv(){
+          dir("${BASE_DIR}/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
+          }
+        }
+      }
+    }
+    stage('Metricbeat x-pack Test Docker images'){
+      options {
+        warnError('Metricbeat x-pack Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
+          }
+        }
+      }
+    }
+    stage('Filebeat x-pack Test Docker images'){
+      options {
+        warnError('Filebeat x-pack Test Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/filebeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/jobs/apm-beats-update.yml

@@ -0,0 +1,60 @@
+- job:
+    name: Beats/apm-beats-update
+    display-name: 'APM Server Beats update'
+    description: 'Check if the Beats lib update works on the APM server main branch'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    prune-dead-branches: true
+    number-to-keep: 10
+    days-to-keep: 30
+    script-path: '.ci/apm-beats-update.groovy'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        discover-tags: true
+        head-filter-regex: '(main|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        disable-pr-notifications: true
+        notification-context: 'apm-beats-update'
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: github-app-beats-ci
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - skip-initial-build: true
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: 30
+        - named-branches:
+            - exact-name:
+                name: 'main'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.1[6789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        - change-request:
+            ignore-target-only-changes: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 10
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true

.ci/jobs/beats-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-daily
+    display-name: Jobs scheduled daily (weekdays)
+    description: Jobs scheduled daily (weekdays)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: main
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-daily.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(2-3) * * 1-5'

.ci/jobs/beats-schedule-weekly.yml

@@ -8,7 +8,7 @@
     parameters:
       - string:
           name: branch_specifier
-          default: master
+          default: main
           description: the Git branch specifier to build
     pipeline-scm:
       script-path: .ci/schedule-weekly.groovy

.ci/jobs/beats-test-infra.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: Beats/beats-test-infra
+    display-name: 'Beats test infra'
+    description: 'Pipeline to run the test infra'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: .ci/validateWorkersBeatsCi.groovy
+        scm:
+        - git:
+            url: [email protected]:elastic/apm-pipeline-library.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/apm-test-pipeline.git
+            branches:
+                - main
+    triggers:
+        - timed: 'H H(3-4) * * 1-5'

.ci/jobs/beats-tester-bc.yml

@@ -0,0 +1,26 @@
+---
+- job:
+    name: Beats/beats-tester-bc
+    display-name: Beats Tester orchestrator for Build Candidates
+    description: Launch the beat-tester suit on Build Candidates easily
+    view: Beats
+    disabled: false
+    project-type: pipeline
+    parameters:
+      - string:
+          name: BRANCH_REFERENCE
+          default: main
+          description: the Git branch specifier
+    pipeline-scm:
+      script-path: .ci/beats-tester-bc.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: true
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $BRANCH_REFERENCE

.ci/jobs/beats-tester.yml

@@ -14,7 +14,11 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
+<<<<<<< HEAD
           head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+=======
+          head-filter-regex: '(main|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+>>>>>>> 237937085a (use main default branch (#29710))
           disable-pr-notifications: true
           notification-context: 'beats-tester'
           repo: 'beats'
@@ -29,7 +33,7 @@
               ignore-tags-newer-than: 30
           - named-branches:
               - exact-name:
-                  name: 'master'
+                  name: 'main'
                   case-sensitive: true
               - regex-name:
                   regex: '7\.([x9]|1\d+)'