ci(jenkins): backport commits from long time ago (#3438)

* ci(jenkins): avoid builds in the master worker (#2599)
* ci(jenkins): build and test stages are now merged. (#2614)
* ci(jenkins): rename APM ITs to avoid any misleading (#2629)
* ci(jenkins): revert none agent (#2645)
* ci(jenkins): reuse top level agent (#2774)
* ci(jenkins): update argument (#2904)
* [apm-ci] When asciidoc speedup skip some stages (#2891)
* ci(jenkins): change log rotation, use cached repo (#2970)
* ci(jenkins): use git reference repo (#2971)
* feat: grab docker container logs after run tests (#2948)
* ci(jenkins): shallow cloning doesn't work with isGitRegionMatch (#2998)
* ci(jenkins): cancel previous running builds (#2983)
* feat: enable DIAGNOSTIC_INTERVAL env var (#3032)
* ci(jenkins): when regexp comparator (#2644)
* fix: trigger on _beats changes (#3189)
* ci(jenkins): retry build docker image up to 3 times with some sleep
* Simplify build scripts (#3371)
* ci(jenkins): apm-its-downstream params has changed (#3406)
* fix: build inside a Golang Docker container (#2375)
* ci: move benchmark stage to run in a tool Docker container (#2391)
* add package install test, add labels to script steps (#2268)
* [apm-ci] suppress the NativeCommandError in windows builds (#2890)
* ci(jenkins): fix issues with python3 in branch 6.8 (#3428)
* ci(jenkins): apm-its are not required in the CI Pipeline

.ci/check-changelogs.groovy

@@ -0,0 +1,65 @@
+#!/usr/bin/env groovy
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    REPO = 'apm-server'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    NOTIFY_TO = credentials('notify-to')
+    JOB_GCS_BUCKET = credentials('gcs-bucket')
+    JOB_GCS_CREDENTIALS = 'apm-ci-gcs-plugin'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '100', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(3-4) * * 1-5'
+  }
+  stages {
+    /**
+     Checkout the code and stash it, to use it on other stages.
+    */
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      environment {
+        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+      }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true,
+                    depth: 3, reference: "/var/lib/jenkins/.git-references/${REPO}.git")
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+      }
+    }
+    /**
+      Validate changelog tests.
+    */
+    stage('Changelog Test') {
+      options { skipDefaultCheckout() }
+      environment {
+        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+        HOME = "${env.WORKSPACE}"
+      }
+      steps {
+        deleteDir()
+        unstash 'source'
+        dir("${BASE_DIR}"){
+          sh(label: 'Run check changelogs', script: './script/jenkins/check-changelogs.sh')
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/docker/golang-mage/Dockerfile

@@ -0,0 +1,26 @@
+ARG GO_VERSION=1.12.4
+FROM golang:${GO_VERSION}
+
+WORKDIR /tools
+
+RUN echo "Building from golang:${GO_VERSION}"
+RUN git clone https://github.com/magefile/mage \
+  && cd mage \
+  && go run bootstrap.go \
+  && mage -version \
+  && cd /tools \
+  && go get golang.org/x/tools/cmd/benchcmp \
+  && go get github.com/t-yuki/gocover-cobertura \
+  && go get github.com/jstemmer/go-junit-report \
+  && go get github.com/kardianos/govendor \
+  && go get github.com/elastic/go-licenser \
+  && go get github.com/elastic/apm-server/vendor/github.com/pierrre/gotestcover \
+  && go get github.com/elastic/apm-server/vendor/github.com/stretchr/testify/assert \
+  && go get github.com/elastic/apm-server/vendor/golang.org/x/tools/cmd/goimports
+
+RUN apt-get update -y -qq \
+  && apt-get install -y -qq  python-pip \
+  && rm -rf /var/lib/apt/lists/* \
+  && pip install virtualenv
+
+WORKDIR /go

.ci/jobs/apm-server-check-changelogs-mbp.yml

@@ -0,0 +1,40 @@
+---
+- job:
+    name: apm-server/apm-server-check-changelogs-mbp
+    display-name: APM Server check changelogs MBP
+    description: APM Server check changelogs MBP
+    project-type: multibranch
+    script-path: .ci/check-changelogs.groovy
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: false
+        notification-context: 'apm-ci'
+        property-strategies:
+          all-branches:
+          - suppress-scm-triggering: true
+        repo: apm-server
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+          reference-repo: /var/lib/jenkins/.git-references/apm-server.git
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'
+    periodic-folder-trigger: 1w

.ci/jobs/apm-server-mbp.yml

@@ -0,0 +1,44 @@
+---
+- job:
+    name: apm-server/apm-server-mbp
+    display-name: APM Server
+    description: APM Server
+    project-type: multibranch
+    concurrent: true
+    script-path: Jenkinsfile
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: true
+        notification-context: 'apm-ci'
+        repo: apm-server
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - regular-branches: true
+        - change-request:
+            ignore-target-only-changes: false
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+          reference-repo: /var/lib/jenkins/.git-references/apm-server.git
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'

.ci/jobs/defaults.yml

@@ -0,0 +1,19 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: apm-ci
+
+##### JOB DEFAULTS
+
+- job:
+    view: APM-CI
+    logrotate:
+      numToKeep: 100
+    node: linux
+    periodic-folder-trigger: 1d
+    prune-dead-branches: true
+    publishers:
+    - email:
+        recipients: [email protected]