feat: run E2E tests for Fleet Server

.ci/e2eFleetServer.groovy

@@ -0,0 +1,190 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    REPO = 'fleet-server'
+    ELASTIC_REPO = "elastic/${env.REPO}"
+    BASE_DIR = "src/github.com/${env.ELASTIC_REPO}"
+    BEATS_REPO = 'beats'
+    BEATS_ELASTIC_REPO = "elastic/${env.BEATS_REPO}"
+    BEATS_BASE_DIR = "src/github.com/${env.BEATS_ELASTIC_REPO}"
+    E2E_REPO = 'e2e-testing'
+    E2E_ELASTIC_REPO = "elastic/${env.E2E_REPO}"
+    E2E_BASE_DIR = "src/github.com/${env.E2E_ELASTIC_REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_NAMESPACE = 'observability-ci'
+    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    GITHUB_APP_SECRET = 'secret/observability-team/ci/github-app'
+    GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
+    JOB_GIT_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken"
+    PIPELINE_LOG_LEVEL = "INFO"
+    AGENT_DROP_PATH = "/tmp/agent-drop-path"
+  }
+  options {
+    timeout(time: 3, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  // http://JENKINS_URL/generic-webhook-trigger/invoke
+  // Pull requests events: https://docs.github.com/en/developers/webhooks-and-events/github-event-types#pullrequestevent
+  triggers {
+    GenericTrigger(
+     genericVariables: [
+      [key: 'GT_REPO', value: '$.repository.full_name'],
+      [key: 'GT_PR', value: '$.issue.number'],
+      [key: 'GT_BODY', value: '$.comment.body'],
+      [key: 'GT_COMMENT_ID', value: '$.comment.id']
+     ],
+    genericHeaderVariables: [
+     [key: 'x-github-event', regexpFilter: 'comment']
+    ],
+     causeString: 'Triggered on #$GT_PR via comment: $GT_BODY',
+     printContributedVariables: false,
+     printPostContent: false,
+     silentResponse: true,
+     regexpFilterText: '$GT_REPO$GT_BODY',
+     regexpFilterExpression: '^elastic/fleet-server/run-fleet-e2e-tests$'
+    )
+  }
+  parameters {
+    string(name: 'fleet_server_pr', defaultValue: "master", description: "PR ID to use to run the E2E tests (e.g 10000)")
+  }
+  stages {
+    stage('Initialize'){
+      agent { label 'ubuntu-20 && immutable' }
+      options { skipDefaultCheckout() }
+      environment {
+        HOME = "${env.WORKSPACE}/${BASE_DIR}"
+      }
+      stages {
+        stage('Check permissions') {
+          steps {
+            checkPermissions()
+            setEnvVar('E2E_BASE_BRANCH', getE2EBaseBranch())
+            sh(label:'Prepare Agent Drop path', script: 'mkdir -p ${AGENT_DROP_PATH}')
+          }
+        }
+        stage('Build Elastic Agent dependencies') {
+          options { skipDefaultCheckout() }
+          parallel {
+            stage('Build Fleet Server') {
+              options { skipDefaultCheckout() }
+              steps {
+                gitCheckout(basedir: BASE_DIR, branch: 'master', githubNotifyFirstTimeContributor: true, repo: "[email protected]:${env.ELASTIC_REPO}.git", credentialsId: env.JOB_GIT_CREDENTIALS)
+                dir("${BASE_DIR}") {
+                  withGoEnv(){
+                    sh(label: 'Build Fleet Server', script: "make release")
+                    sh(label: 'Copy binaries to Agent Drop path', script: 'cp build/distributions/* ${AGENT_DROP_PATH}')
+                  }
+                }
+              }
+            }
+            stage('Build Elastic Agent Dependencies') {
+              options { skipDefaultCheckout() }
+              steps {
+                gitCheckout(basedir: BEATS_BASE_DIR, branch: 'master', githubNotifyFirstTimeContributor: true, repo: "[email protected]:${env.BEATS_ELASTIC_REPO}.git", credentialsId: env.JOB_GIT_CREDENTIALS)
+                dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") {
+                  withGoEnv(){
+                    withEnv(["DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) {
+                      sh(label: 'Build Fleet Server', script: 'mage package')
+                    }
+                  }
+                }
+                dir("${BEATS_BASE_DIR}/x-pack") {
+                  sh(label:'Copy Filebeat binaries to Agent Drop path', script: 'cp filebeat/build/distributions/* ${AGENT_DROP_PATH}')
+                  sh(label:'Copy Heartbeat binaries to Agent Drop path', script: 'cp metricbeat/build/distributions/* ${AGENT_DROP_PATH}')
+                  sh(label:'Copy Metricbeat binaries to Agent Drop path', script: 'cp heartbeat/build/distributions/* ${AGENT_DROP_PATH}')
+                }
+              }
+            }
+          }
+        }
+        stage('Build Elastic Agent including Fleet Server') {
+          options { skipDefaultCheckout() }
+          steps {
+            dir("${BEATS_BASE_DIR}/x-pack/elastic-agent") {
+              withGoEnv(){
+                withEnv(["AGENT_DROP_PATH='${env.AGENT_DROP_PATH}'", "DEV=true", "SNAPSHOT=true", "PLATFORMS='+all linux/amd64'"]) {
+                  sh(label: 'Build Fleet Server', script: 'mage package')
+                }
+              }
+            }
+          }
+        }
+        stage('Run E2E Tests') {
+          options { skipDefaultCheckout() }
+          steps {
+            gitCheckout(basedir: E2E_BASE_DIR, branch: "${env.E2E_BASE_BRANCH}", githubNotifyFirstTimeContributor: true, repo: '[email protected]:elastic/e2e-testing.git', credentialsId: env.JOB_GIT_CREDENTIALS)
+            dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+            dir("${E2E_BASE_DIR}") {
+              withGoEnv(){
+                withEnv(["BEATS_LOCAL_PATH='${env.BEATS_BASE_DIR}"]) {
+                  sh(label: 'Run E2E Tests', script: './.ci/scripts/fleet-tests.sh ')
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+def checkPermissions(){
+  if(env.GT_PR){
+    if(!githubPrCheckApproved(changeId: "${env.GT_PR}", org: 'elastic', repo: 'kibana')){
+      error("Only PRs from Elasticians can be tested with Fleet E2E tests")
+    }
+
+    if(!hasCommentAuthorWritePermissions(repoName: "${env.ELASTIC_REPO}", commentId: env.GT_COMMENT_ID)){
+      error("Only Elasticians can trigger Fleet E2E tests")
+    }
+  }
+}
+
+def getE2EBaseBranch() {
+  // we need a second API request, as the issue_comment API does not retrieve data about the pull request
+  // See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#issue_comment
+  def prID = getID()
+
+  if (!prID.isInteger()) {
+    // in the case we are triggering the job for a branch (i.e master, 7.x) we directly use branch name as Docker tag
+    return getMaintenanceBranch(prID)
+  }
+
+  def token = githubAppToken(secret: "${env.GITHUB_APP_SECRET}")
+
+  def pullRequest = githubApiCall(token: token, url: "https://api.github.com/repos/${env.ELASTIC_REPO}/pulls/${prID}")
+  def baseRef = pullRequest?.base?.ref
+  //def headSha = pullRequest?.head?.sha
+
+  return getMaintenanceBranch(baseRef)
+}
+
+def getID(){
+  if(env.GT_PR){
+    return "${env.GT_PR}"
+  }
+
+  return "${params.fleet_server_pr}"
+}
+
+def getMaintenanceBranch(String branch){
+  if (branch == 'master' || branch == 'main') {
+    return branch
+  }
+
+  if (!branch.endsWith('.x')) {
+    // use maintenance branches mode (i.e. 7.16 translates to 7.16.x)
+    return branch + '.x'
+  }
+
+  return branch
+}

.ci/e2eKibana.groovy

@@ -63,7 +63,7 @@ pipeline {
             setEnvVar('DOCKER_TAG', getDockerTagFromPayload())
           }
         }
-        stage('Process GitHub Event') {
+        stage('Build Linux Images') {
           options { skipDefaultCheckout() }
           parallel {
             stage('AMD build') {
@@ -151,6 +151,12 @@ def getDockerTagFromPayload() {
   // we need a second API request, as the issue_comment API does not retrieve data about the pull request
   // See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#issue_comment
   def prID = getID()
+
+  if (!prID.isInteger()) {
+    // in the case we are triggering the job for a branch (i.e master, 7.x) we directly use branch name as Docker tag
+    return prID
+  }
+
   def token = githubAppToken(secret: "${env.GITHUB_APP_SECRET}")
 
   def pullRequest = githubApiCall(token: token, url: "https://api.github.com/repos/${env.ELASTIC_REPO}/pulls/${prID}")

.ci/jobs/fleet-server-e2e-tests.yml

@@ -0,0 +1,18 @@
+---
+- job:
+    name: e2e-tests/e2e-testing-fleet-server
+    display-name: Fleet Server e2e tests Pipeline
+    description: Jenkins pipeline to run the end2end tests for Fleet Server
+    project-type: pipeline
+    pipeline-scm:
+      script-path: .ci/e2eFleetServer.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/e2e-testing.git
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            branches:
+              - "master"
+    triggers: []

e2e/_suites/fleet/stand-alone.go

@@ -15,7 +15,6 @@ import (
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
-	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
 
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
@@ -92,9 +91,8 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en
 	}
 
 	useCISnapshots := elasticversion.GithubCommitSha1 != ""
-	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
 
-	if useCISnapshots || beatsLocalPath != "" {
+	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
 		// load the docker images that were already:
 		// a. downloaded from the GCP bucket
 		// b. fetched from the local beats binaries

e2e/_suites/kubernetes-autodiscover/autodiscover_test.go

@@ -32,7 +32,6 @@ import (
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/kubernetes"
-	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
 )
 
@@ -136,8 +135,7 @@ func (m *podsManager) configureDockerImage(podName string) error {
 	beatVersion := elasticversion.GetSnapshotVersion(common.BeatVersion) + "-amd64"
 
 	useCISnapshots := elasticversion.GithubCommitSha1 != ""
-	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
-	if useCISnapshots || beatsLocalPath != "" {
+	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
 		log.Debugf("Configuring Docker image for %s", podName)
 
 		_, imagePath, err := elasticversion.FetchElasticArtifact(m.ctx, podName, common.BeatVersion, "linux", "amd64", "tar.gz", true, true)

internal/deploy/docker.go

@@ -278,9 +278,8 @@ func (c *dockerDeploymentManifest) Stop(ctx context.Context, service ServiceRequ
 // the images produced by local Beats build, or not.
 // If an error occurred reading the environment, will return the passed namespace as fallback
 func GetDockerNamespaceEnvVar(fallback string) string {
-	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
 	useCISnapshots := elasticversion.GithubCommitSha1 != ""
-	if useCISnapshots || beatsLocalPath != "" {
+	if useCISnapshots || elasticversion.BeatsLocalPath != "" {
 		return "observability-ci"
 	}
 	return fallback

internal/versions.go

@@ -24,6 +24,10 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// BeatsLocalPath is the path to a local copy of the Beats git repository
+// It can be overriden by BEATS_LOCAL_PATH env var. Using the empty string as a default.
+var BeatsLocalPath = ""
+
 // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key
 // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the
 // already downloaded artifact.
@@ -39,6 +43,11 @@ var GithubCommitSha1 string
 
 func init() {
 	GithubCommitSha1 = shell.GetEnv("GITHUB_CHECK_SHA1", "")
+
+	BeatsLocalPath = shell.GetEnv("BEATS_LOCAL_PATH", BeatsLocalPath)
+	if BeatsLocalPath != "" {
+		log.Infof(`Beats local path will be used for artifacts. Please make sure all binaries are properly built in their "build/distributions" folder: %s`, BeatsLocalPath)
+	}
 }
 
 // elasticVersion represents a version
@@ -321,8 +330,7 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch
 		}
 	}
 
-	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
-	if beatsLocalPath != "" && isDocker {
+	if BeatsLocalPath != "" && isDocker {
 		dockerString = ".docker"
 		return fmt.Sprintf("%s-%s-%s-%s%s.%s", artifact, artifactVersion, OS, arch, dockerString, lowerCaseExtension)
 	}
@@ -345,16 +353,15 @@ func buildArtifactName(artifact string, artifactVersion string, OS string, arch
 // Else, if the environment variable GITHUB_CHECK_SHA1 is set, then the artifact
 // to be downloaded will be defined by the snapshot produced by the Beats CI for that commit.
 func fetchBeatsBinary(ctx context.Context, artifactName string, artifact string, version string, timeoutFactor int, xpack bool) (string, error) {
-	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
-	if beatsLocalPath != "" {
+	if BeatsLocalPath != "" {
 		span, _ := apm.StartSpanOptions(ctx, "Fetching Beats binary", "beats.local.fetch-binary", apm.SpanOptions{
 			Parent: apm.SpanFromContext(ctx).TraceContext(),
 		})
 		defer span.End()
 
-		distributions := path.Join(beatsLocalPath, artifact, "build", "distributions")
+		distributions := path.Join(BeatsLocalPath, artifact, "build", "distributions")
 		if xpack {
-			distributions = path.Join(beatsLocalPath, "x-pack", artifact, "build", "distributions")
+			distributions = path.Join(BeatsLocalPath, "x-pack", artifact, "build", "distributions")
 		}
 
 		log.Debugf("Using local snapshots for the %s: %s", artifact, distributions)