Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into disable_tests
Browse files Browse the repository at this point in the history
  • Loading branch information
kaiyan-sheng committed Jul 30, 2021
2 parents 7a487ab + 55053a9 commit d67ddf2
Show file tree
Hide file tree
Showing 2,748 changed files with 286,861 additions and 71,824 deletions.
2 changes: 1 addition & 1 deletion .backportrc.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"upstream": "elastic/beats",
"branches": [ { "name": "7.x", "checked": true }, "7.12", "7.11" ],
"branches": [ { "name": "7.x", "checked": true }, "7.14", "7.13", "7.12" ],
"labels": ["backport"],
"autoAssign": true,
"prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"
Expand Down
2 changes: 1 addition & 1 deletion .ci/beats-tester.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ pipeline {
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-20' }
when {
beforeAgent true
anyOf {
Expand Down
2 changes: 1 addition & 1 deletion .ci/build-docker-images.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
@Library('apm@current') _

pipeline {
agent { label 'ubuntu-16 && immutable' }
agent { label 'ubuntu-20' }
environment {
REPO = 'beats'
BASE_DIR = "src/github.com/elastic/${env.REPO}"
Expand Down
40 changes: 40 additions & 0 deletions .ci/bump-go-release-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
#
# Given the Golang release version this script will bump the version.
#
# This script is executed by the automation we are putting in place
# and it requires the git add/commit commands.
#
# Parameters:
# $1 -> the Golang release version to be bumped. Mandatory.
#
set -euo pipefail
MSG="parameter missing."
GO_RELEASE_VERSION=${1:?$MSG}

OS=$(uname -s| tr '[:upper:]' '[:lower:]')

if [ "${OS}" == "darwin" ] ; then
SED="sed -i .bck"
else
SED="sed -i"
fi

echo "Update go version ${GO_RELEASE_VERSION}"
echo "${GO_RELEASE_VERSION}" > .go-version
git add .go-version

find . -maxdepth 3 -name Dockerfile -print0 |
while IFS= read -r -d '' line; do
${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
git add "${line}"
done

${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
git add libbeat/docs/version.asciidoc

git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
git --no-pager log -1

echo "You can now push and create a Pull Request"
49 changes: 49 additions & 0 deletions .ci/bump-stack-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
#
# Given the stack version this script will bump the version.
#
# This script is executed by the automation we are putting in place
# and it requires the git add/commit commands.
#
# Parameters:
# $1 -> the version to be bumped. Mandatory.
# $2 -> whether to create a branch where to commit the changes to.
# this is required when reusing an existing Pull Request.
# Optional. Default true.
#
set -euo pipefail
MSG="parameter missing."
VERSION=${1:?$MSG}
CREATE_BRANCH=${2:-true}

OS=$(uname -s| tr '[:upper:]' '[:lower:]')

if [ "${OS}" == "darwin" ] ; then
SED="sed -i .bck"
else
SED="sed -i"
fi

FILES="testing/environments/snapshot-oss.yml
testing/environments/snapshot.yml
"

echo "Update stack with version ${VERSION}"
for FILE in ${FILES} ; do
${SED} -E -e "s#(image: docker\.elastic\.co/.*):[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1:${VERSION}#g" $FILE
done

echo "Commit changes"
if [ "$CREATE_BRANCH" = "true" ]; then
base=$(git rev-parse --abbrev-ref HEAD | sed 's#/#-#g')
git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")-${base}"
else
echo "Branch creation disabled."
fi
for FILE in ${FILES} ; do
git add $FILE
done
git diff --staged --quiet || git commit -m "[Automation] Update elastic stack version to ${VERSION} for testing"
git --no-pager log -1

echo "You can now push and create a Pull Request"
11 changes: 10 additions & 1 deletion .ci/packaging.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ pipeline {
'x-pack/heartbeat',
// 'x-pack/journalbeat',
'x-pack/metricbeat',
'x-pack/osquerybeat',
'x-pack/packetbeat',
'x-pack/winlogbeat'
)
Expand All @@ -137,7 +138,11 @@ pipeline {
'linux/amd64',
'linux/386',
'linux/arm64',
'linux/armv7',
// armv7 packaging isn't working, and we don't currently
// need it for release. Do not re-enable it without
// confirming it is fixed, you will break the packaging
// pipeline!
//'linux/armv7',
// The platforms above are disabled temporarly as crossbuild images are
// not available. See: https://github.com/elastic/golang-crossbuild/issues/71
//'linux/ppc64le',
Expand Down Expand Up @@ -290,6 +295,8 @@ def pushCIDockerImages(Map args = [:]) {
tagAndPush(beatName: 'journalbeat', arch: arch)
} else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
tagAndPush(beatName: 'metricbeat', arch: arch)
} else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {
tagAndPush(beatName: 'osquerybeat', arch: arch)
} else if ("${env.BEATS_FOLDER}" == "packetbeat"){
tagAndPush(beatName: 'packetbeat', arch: arch)
} else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
Expand Down Expand Up @@ -422,11 +429,13 @@ def triggerE2ETests(String suite) {

def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"

def parameters = [
booleanParam(name: 'forceSkipGitChecks', value: true),
booleanParam(name: 'forceSkipPresubmit', value: true),
booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
string(name: 'BEAT_VERSION', value: beatVersion),
booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
string(name: 'runTestsSuites', value: suite),
string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
Expand Down
132 changes: 67 additions & 65 deletions .ci/packer_cache.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -9,78 +9,80 @@ source /usr/local/bin/bash_standard_lib.sh
# shellcheck disable=SC1091
source ./dev-tools/common.bash

# Docker images used on Dockerfiles 2019-07-12
# aerospike:3.9.0
# alpine:edge
# apache/couchdb:1.7
# busybox:latest
# ceph/daemon:master-6373c6a-jewel-centos-7-x86_64
# cockroachdb/cockroach:v19.1.1
# consul:1.4.2
# coredns/coredns:1.5.0
# couchbase:4.5.1
# debian:latest
# debian:stretch
# docker.elastic.co/beats-dev/fpm:1.11.0
# docker.elastic.co/beats/metricbeat:6.5.4
# docker.elastic.co/beats/metricbeat:7.2.0
# docker.elastic.co/elasticsearch/elasticsearch:7.2.0
# docker.elastic.co/kibana/kibana:7.2.0
# docker.elastic.co/logstash/logstash:7.2.0
# docker.elastic.co/observability-ci/database-instantclient:12.2.0.1
# envoyproxy/envoy:v1.7.0
# exekias/localkube-image
# haproxy:1.8
# httpd:2.4.20
# java:8-jdk-alpine
# jplock/zookeeper:3.4.8
# maven:3.3-jdk-8
# memcached:1.4.35-alpine
# microsoft/mssql-server-linux:2017-GA
# mongo:3.4
# mysql:5.7.12
# nats:1.3.0
# nginx:1.9
# oraclelinux:7
# postgres:9.5.3
# prom/prometheus:v2.6.0
# python:3.6-alpine
# quay.io/coreos/etcd:v3.3.10
# rabbitmq:3.7.4-management
# redis:3.2.12-alpine
# redis:3.2.4-alpine
# store/oracle/database-enterprise:12.2.0.1
# traefik:1.6-alpine
# tsouza/nginx-php-fpm:php-7.1
# ubuntu:16.04
# ubuntu:trusty
######################
############ FUNCTIONS
######################
function getBeatsVersion() {
grep 'defaultBeatVersion' libbeat/version/version.go | cut -d= -f2 | sed 's#"##g' | tr -d " "
}

get_go_version

DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1
docker.elastic.co/observability-ci/database-enterprise:12.2.0.1
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x
golang:${GO_VERSION}
"
if [ -x "$(command -v docker)" ]; then
for image in ${DOCKER_IMAGES}
do
(retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image, we continue"
function dockerPullCommonImages() {
DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1
docker.elastic.co/observability-ci/database-enterprise:12.2.0.1
docker.elastic.co/beats-dev/fpm:1.11.0
golang:1.14.12-stretch
centos:7
"
for image in ${DOCKER_IMAGES} ; do
(retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing."
done

docker tag \
docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 \
store/oracle/database-instantclient:12.2.0.1 \
|| echo "Error setting the Oracle Instant Client tag"
docker tag \
docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 \
store/oracle/database-enterprise:12.2.0.1 \
|| echo "Error setting the Oracle Dtabase tag"
|| echo "Error setting the Oracle Database tag"
}

function dockerPullImages() {
SNAPSHOT=$1
get_go_version

DOCKER_IMAGES="
docker.elastic.co/elasticsearch/elasticsearch:${SNAPSHOT}
docker.elastic.co/kibana/kibana:${SNAPSHOT}
docker.elastic.co/logstash/logstash:${SNAPSHOT}
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x
golang:${GO_VERSION}
"
for image in ${DOCKER_IMAGES}
do
(retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing."
done
}

#################
############ MAIN
#################
if [ -x "$(command -v docker)" ]; then
set -x
echo "Docker pull common docker images"
dockerPullCommonImages

## GitHub api returns up to 100 entries.
## Probably we need a different approach to search the latest minor.
latestMinor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^7." | tail -1)

for branch in master 7.x $latestMinor ; do
if [ "$branch" != "master" ] ; then
echo "Checkout the branch $branch"
git checkout "$branch"
fi

VERSION=$(getBeatsVersion)
dockerPullImages "${VERSION}-SNAPSHOT"
done
fi
11 changes: 7 additions & 4 deletions .ci/scripts/generate_build_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@

if __name__ == "__main__":

print("| Beat | Stage | Command | MODULE | Platforms | When |")
print("|-------|--------|----------|---------|------------|------|")
print("| Beat | Stage | Category | Command | MODULE | Platforms | When |")
print("|-------|--------|----------|----------|---------|------------|------|")
for root, dirs, files in os.walk("."):
dirs.sort()
for file in files:
Expand All @@ -18,6 +18,9 @@
withModule = False
platforms = [doc["platform"]]
when = "mandatory"
category = 'default'
if "stage" in doc["stages"][stage]:
category = doc["stages"][stage]["stage"]
if "make" in doc["stages"][stage]:
command = doc["stages"][stage]["make"].replace("\n", " ")
if "mage" in doc["stages"][stage]:
Expand All @@ -33,5 +36,5 @@
if "when" in doc["stages"][stage]:
if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
when = "optional"
print("| {} | {} | `{}` | {} | `{}` | {} |".format(
module, stage, command, withModule, platforms, when))
print("| {} | {} | `{}` | `{}` | {} | `{}` | {} |".format(
module, stage, category, command, withModule, platforms, when))
57 changes: 57 additions & 0 deletions .ci/scripts/install-go.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
set GOPATH=%WORKSPACE%
set MAGEFILE_CACHE=%WORKSPACE%\.magefile

set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%

curl --version >nul 2>&1 && (
echo found curl
) || (
choco install curl -y --no-progress --skipdownloadcache
)

mkdir %WORKSPACE%\bin

IF EXIST "%PROGRAMFILES(X86)%" (
REM Force the gvm installation.
SET GVM_BIN=gvm.exe
curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
IF ERRORLEVEL 1 (
REM gvm installation has failed.
del bin\gvm.exe /s /f /q
exit /b 1
)
) ELSE (
REM Windows 7 workers got a broken gvm installation.
curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
IF ERRORLEVEL 1 (
REM gvm installation has failed.
del bin\gvm.exe /s /f /q
exit /b 1
)
)

SET GVM_BIN=gvm.exe
WHERE /q %GVM_BIN%
%GVM_BIN% version

REM Install the given go version
%GVM_BIN% --debug install %GO_VERSION%

REM Configure the given go version
FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i

go env
IF ERRORLEVEL 1 (
REM go is not configured correctly.
rmdir %WORKSPACE%\.gvm /s /q
exit /b 1
)

where mage
mage -version
IF ERRORLEVEL 1 (
go get github.com/magefile/mage
IF ERRORLEVEL 1 (
exit /b 1
)
)
Loading

0 comments on commit d67ddf2

Please sign in to comment.