Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/windows-2016
Browse files Browse the repository at this point in the history
* upstream/master:
  [CI] Setup git config globally (elastic#21562)
  docs: update generate_fields_docs.py (elastic#21359)
  Add support for additional fields from V2 ALB logs (elastic#21540)
  Move Prometheus query & remote_write to GA (elastic#21507)
  feat: add a new step to run the e2e tests for certain parts of Beats (elastic#21100)
  [Elastic Agent] Add elastic agent ID and version to events from filebeat and metricbeat. (elastic#21543)
  Release cloudfoundry input and processor as GA (elastic#21525)
  [Packetbeat] New SIP protocol (elastic#21221)
  [Filebeat][New Module] Add support for Microsoft MTP / 365 Defender (elastic#21446)
  [Beats][pytest] Asserting if filebeat logs include errors (elastic#20999)
  junipersrx-module initial release (elastic#20017)
  Add a persistent cache for cloudfoundry metadata based on badger (elastic#20775)
  Add missing changelog entry for cisco umbrella (elastic#21550)
  [Elastic Agent] Add upgrade CLI to initiate upgrade of Agent locally (elastic#21425)
  Enable filestream input (elastic#21533)
  Add filestream input reader (elastic#21481)
  [CI] fix 'no matches found within 10000' (elastic#21466)
  Fix billing.go aws.GetStartTimeEndTime (elastic#21531)
  • Loading branch information
v1v committed Oct 6, 2020
2 parents 07afe0b + 1ce876d commit fcd6db3
Show file tree
Hide file tree
Showing 147 changed files with 20,690 additions and 2,414 deletions.
50 changes: 49 additions & 1 deletion .ci/packaging.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
pipeline {
agent none
environment {
BASE_DIR = 'src/github.com/elastic/beats'
REPO = 'beats'
BASE_DIR = "src/github.com/elastic/${env.REPO}"
JOB_GCS_BUCKET = 'beats-ci-artifacts'
JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
DOCKER_REGISTRY = 'docker.elastic.co'
GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
SNAPSHOT = "true"
PIPELINE_LOG_LEVEL = "INFO"
}
Expand Down Expand Up @@ -119,6 +121,7 @@ pipeline {
release()
pushCIDockerImages()
}
runE2ETestForPackages()
}
}
stage('Package Mac OS'){
Expand Down Expand Up @@ -209,6 +212,25 @@ def tagAndPush(name){
}
}

def runE2ETestForPackages(){
def suite = ''

catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
suite = 'helm,ingest-manager'
} else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
suite = ''
} else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
suite = 'ingest-manager'
} else {
echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
return
}

triggerE2ETests(suite)
}
}

def release(){
withBeatsEnv(){
dir("${env.BEATS_FOLDER}") {
Expand All @@ -218,6 +240,32 @@ def release(){
}
}

def triggerE2ETests(String suite) {
echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")

def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
build(job: "${e2eTestsPipeline}",
parameters: [
booleanParam(name: 'forceSkipGitChecks', value: true),
booleanParam(name: 'forceSkipPresubmit', value: true),
booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
booleanParam(name: 'USE_CI_SNAPSHOTS', value: true),
string(name: 'ELASTIC_AGENT_VERSION', value: "pr-${env.CHANGE_ID}"),
string(name: 'METRICBEAT_VERSION', value: "pr-${env.CHANGE_ID}"),
string(name: 'runTestsSuites', value: suite),
string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
],
propagate: false,
wait: false
)

def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
}

def withMacOSEnv(Closure body){
withEnvMask( vars: [
[var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],
Expand Down
9 changes: 9 additions & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
- Added experimental dataset `sophos/utm`. {pull}20820[20820]
- Add Cloud Foundry tags in related events. {pull}21177[21177]
- Cloud Foundry metadata is cached to disk. {pull}20775[20775]
- Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
- Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]

*Auditbeat*

Expand Down Expand Up @@ -605,6 +607,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add related.hosts ecs field to all modules {pull}21160[21160]
- Keep cursor state between httpjson input restarts {pull}20751[20751]
- Convert aws s3 to v2 input {pull}20005[20005]
- Add support for additional fields from V2 ALB logs. {pull}21540[21540]
- Release Cloud Foundry input as GA. {pull}21525[21525]
- New Cisco Umbrella dataset {pull}21504[21504]
- New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
- Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]

*Heartbeat*

Expand Down Expand Up @@ -732,6 +739,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
- Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
- Move Prometheus query & remote_write to GA. {pull}21507[21507]
- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381]

Expand All @@ -742,6 +750,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
port. {pull}19209[19209]
- Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
- Add 100-continue support {issue}15830[15830] {pull}19349[19349]
- Add initial SIP protocol support {pull}21221[21221]


*Functionbeat*
Expand Down
13 changes: 9 additions & 4 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ pipeline {
}
steps {
withGithubNotify(context: 'Lint') {
withBeatsEnv(archive: true) {
withBeatsEnv(archive: true, id: 'lint') {
dumpVariables()
cmd(label: 'make check', script: 'make check')
}
Expand Down Expand Up @@ -265,8 +265,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
// See https://github.com/elastic/beats/issues/17787.
sh(label: 'check git config', script: '''
if [ -z "$(git config --get user.email)" ]; then
git config user.email "[email protected]"
git config user.name "beatsmachine"
git config --global user.email "[email protected]"
git config --global user.name "beatsmachine"
fi''')
}
try {
Expand Down Expand Up @@ -345,8 +345,13 @@ def archiveTestOutput(Map args = [:]) {
}
cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
dir('build') {
if (isUnix()) {
cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)',
returnStatus: true,
script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;')
} else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') }
junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id)
archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true)
}
catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
Expand Down
Loading

0 comments on commit fcd6db3

Please sign in to comment.