Skip to content

Commit

Permalink
Merge branch 'main' into reaper-exit-orph-fix
Browse files Browse the repository at this point in the history
  • Loading branch information
fearful-symmetry authored Jan 24, 2025
2 parents a37b8ce + e0ef169 commit de94730
Show file tree
Hide file tree
Showing 411 changed files with 6,347 additions and 4,959 deletions.
10 changes: 1 addition & 9 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@

set -euo pipefail

# Secrets must be redacted
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"

if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then

Expand All @@ -13,11 +10,6 @@ if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" ==
fi
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
"$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || \
"$BUILDKITE_PIPELINE_SLUG" == "beats-macos-tests" || \
Expand Down Expand Up @@ -55,4 +47,4 @@ elif [[ "${CPU_ARCH}" == "aarch64" || "${CPU_ARCH}" == "arm64" ]]; then
else
echo "Unsupported OS"
exit 1
fi
fi
27 changes: 15 additions & 12 deletions .buildkite/packaging.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,16 @@ steps:
# this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second

- name: Start of concurrency group for DRA Snapshot
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
command: echo "--> Start of concurrency gate dra-snapshot"
concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
concurrency: 1
key: start-gate-snapshot

- name: Start of concurrency group for DRA Staging
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
# exceptionally allow building staging from main when VERSION_QUALIFIER is set, to allow prerelease testing
# TODO remove OR clause below and above comment, and only allow matching /^[0-9]+\.[0-9x]+\$/ for build.branch
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
command: echo "--> Start of concurrency gate dra-staging"
concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
concurrency: 1
Expand All @@ -36,7 +38,7 @@ steps:
key: dashboards
steps:
- label: Snapshot dashboards
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
depends_on: start-gate-snapshot
key: dashboards-snapshot
# TODO: container with go and make
Expand All @@ -58,7 +60,8 @@ steps:
- build/distributions/**/*

- label: Staging dashboards
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
# TODO remove OR clause below (see earlier comment)
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
depends_on: start-gate-staging
key: dashboards-staging
# TODO: container with go and make
Expand All @@ -80,7 +83,7 @@ steps:
- build/distributions/**/*

- group: Packaging snapshot
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
key: packaging-snapshot
depends_on: start-gate-snapshot
steps:
Expand Down Expand Up @@ -170,8 +173,8 @@ steps:
- group: Packaging Staging
key: packaging-staging
depends_on: start-gate-staging
## Only for release
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
# TODO remove OR clause below (see earlier comment)
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
steps:
- label: "STAGING: {{matrix}}"
env:
Expand Down Expand Up @@ -258,7 +261,7 @@ steps:
steps:
- label: DRA Snapshot
## Only for release branches and main
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
key: dra-snapshot
env:
DRA_WORKFLOW: snapshot
Expand All @@ -276,8 +279,8 @@ steps:
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

- label: DRA Staging
## Only for release branches
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
# TODO remove OR clause below (see earlier comment)
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
key: dra-staging
env:
DRA_WORKFLOW: staging
Expand All @@ -297,13 +300,13 @@ steps:
- wait

- command: echo "End of concurrency gate dra-snapshot <--"
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
concurrency: 1
key: end-gate-snapshot

- command: echo "End of concurrency gate dra-staging <--"
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
concurrency: 1
key: end-gate-staging
2 changes: 1 addition & 1 deletion .buildkite/pull-requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"always_trigger_comment_regex": "^/(?:test)",
"skip_ci_labels": ["skip-ci"],
"skip_target_branches": [ ],
"skip_ci_on_only_changed": [ ],
"skip_ci_on_only_changed": [ "^.github/", "^.pre-commit-config.yaml", "^.mergify.yml", "\\.md$", "^docs/", "^updatecli-compose.yaml"],
"always_require_ci_on_changed": [ ]
}
]
Expand Down
7 changes: 5 additions & 2 deletions .buildkite/scripts/dra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ set -euo pipefail
BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"

BEAT_VERSION=$(make get-version)

VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"

function release_manager_login {
Expand Down Expand Up @@ -49,7 +49,9 @@ docker run --rm \
--commit "${BUILDKITE_COMMIT}" \
--workflow "${DRA_WORKFLOW}" \
--version "${BEAT_VERSION}" \
--artifact-set "main"
--artifact-set "main" \
--qualifier "${VERSION_QUALIFIER}"


echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]"

Expand All @@ -68,6 +70,7 @@ docker run --rm \
--workflow "${DRA_WORKFLOW}" \
--version "${BEAT_VERSION}" \
--artifact-set "main" \
--qualifier "${VERSION_QUALIFIER}" \
${DRY_RUN} | tee rm-output.txt


Expand Down
14 changes: 14 additions & 0 deletions .buildkite/scripts/gcp_auth.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash

set -euo pipefail

echo "~~~ Authenticating GCP"
# Secrets must be redacted
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ./gcp.json
GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json)
export GOOGLE_APPLICATION_CREDENTIALS
gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
56 changes: 56 additions & 0 deletions .buildkite/scripts/install_sysmon.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
$downloadUrl = "https://live.sysinternals.com/Sysmon64.exe"
$tempFolder = "$env:TEMP\SysmonDownload"
$sysmonPath = "$tempFolder\Sysmon64.exe"

if (!(Test-Path $tempFolder)) {
New-Item -ItemType Directory -Path $tempFolder
}

$ProgressPreference = 'SilentlyContinue'
function ParseErrorForResponseBody($Error) {
if ($PSVersionTable.PSVersion.Major -lt 6) {
if ($Error.Exception.Response) {
$Reader = New-Object System.IO.StreamReader($Error.Exception.Response.GetResponseStream())
$Reader.BaseStream.Position = 0
$Reader.DiscardBufferedData()
$ResponseBody = $Reader.ReadToEnd()
if ($ResponseBody.StartsWith('{')) {
$ResponseBody = $ResponseBody | ConvertFrom-Json
}
return $ResponseBody
}
}
else {
return $Error.ErrorDetails.Message
}
}

$attempts=1
do
{
$attempts++
try {
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$result = Invoke-WebRequest -Uri $downloadUrl -OutFile $sysmonPath -UseBasicParsing
break
}
catch {
$resp = ParseErrorForResponseBody($_)
Write-Host "$resp"
if ($attempts -gt 5) {
exit 1
}
}
} while ($attempts -le 5)

Write-Host "Sysmon64.exe downloaded successfully."

if ($sysmonPath) {
Start-Process -FilePath $sysmonPath -ArgumentList "-m" -Wait

Write-Host "Sysmon event manifest installation completed."
} else {
Write-Host "Sysmon executable not found in the downloaded archive."
}

Remove-Item -Path $tempFolder -Force -Recurse
8 changes: 8 additions & 0 deletions .buildkite/scripts/packaging/prepare-release-manager.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,21 @@
set -ueo pipefail

readonly TYPE=${1:-snapshot}
readonly VERSION_QUALIFIER="${VERSION_QUALIFIER:-""}"

# rename dependencies.csv to the name expected by release-manager.
VERSION=$(make get-version)
FINAL_VERSION=$VERSION-SNAPSHOT
if [ "$TYPE" != "snapshot" ] ; then
FINAL_VERSION=$VERSION
fi

set +e
if [[ -n "$VERSION_QUALIFIER" ]]; then
FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}"
fi
set -e

echo "Rename dependencies to $FINAL_VERSION"
mv build/distributions/dependencies.csv \
build/distributions/dependencies-"$FINAL_VERSION".csv
Expand Down
2 changes: 2 additions & 0 deletions .buildkite/x-pack/pipeline.xpack.packetbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ steps:
key: "mandatory-win-2022-system-tests"
skip: "skipping due to elastic/beats#38142"
command: |
source .buildkite/scripts/gcp_auth.sh
Set-Location -Path x-pack/packetbeat
mage systemTest
retry:
Expand Down Expand Up @@ -323,6 +324,7 @@ steps:
key: "extended-win-10-system-tests"
skip: "skipping due to elastic/beats#38142"
command: |
source .buildkite/scripts/gcp_auth.sh
Set-Location -Path x-pack/packetbeat
mage systemTest
retry:
Expand Down
6 changes: 6 additions & 0 deletions .buildkite/x-pack/pipeline.xpack.winlogbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ steps:
- label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests"
key: "mandatory-win-2019-unit-tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
retry:
Expand All @@ -91,6 +92,7 @@ steps:

- label: ":windows: x-pack/winlogbeat: Win 2016 Unit Tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "mandatory-win-2016-unit-tests"
Expand Down Expand Up @@ -118,6 +120,7 @@ steps:

- label: ":windows: x-pack/winlogbeat: Win 2022 Unit Tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "mandatory-win-2022-unit-tests"
Expand Down Expand Up @@ -150,6 +153,7 @@ steps:
steps:
- label: ":windows: x-pack/winlogbeat: Win 10 Unit Tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "extended-win-10-unit-tests"
Expand Down Expand Up @@ -177,6 +181,7 @@ steps:

- label: ":windows: x-pack/winlogbeat: Win 11 Unit Tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "extended-win-11-unit-tests"
Expand Down Expand Up @@ -204,6 +209,7 @@ steps:

- label: ":windows: x-pack/winlogbeat: Win 2019 Unit Tests"
command: |
.buildkite/scripts/install_sysmon.ps1
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "extended-win-2019-unit-tests"
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/bump-elastic-stack-snapshot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
- uses: actions/checkout@v4

- name: Install Updatecli in the runner
uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1

- name: Run Updatecli in Apply mode
run: updatecli --experimental apply --config .github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml --values .github/workflows/updatecli.d/values.d/scm.yml
Expand All @@ -41,11 +41,13 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- if: ${{ failure() }}
uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
with:
channel-id: '#ingest-notifications'
method: chat.postMessage
token: ${{ secrets.SLACK_BOT_TOKEN }}
payload: |
{
"channel": "#ingest-notifications",
"text": "${{ env.SLACK_MESSAGE }}",
"blocks": [
{
Expand All @@ -58,5 +60,4 @@ jobs:
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_MESSAGE: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
9 changes: 5 additions & 4 deletions .github/workflows/bump-golang.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,21 @@ jobs:
- uses: actions/checkout@v4

- name: Install Updatecli in the runner
uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1

- name: Run Updatecli in Apply mode
run: updatecli --experimental apply --config .github/workflows/updatecli.d/${{ matrix.file }} --values .github/workflows/updatecli.d/values.d/scm.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- if: ${{ failure() }}
uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
with:
channel-id: '#ingest-notifications'
method: chat.postMessage
token: ${{ secrets.SLACK_BOT_TOKEN }}
payload: |
{
"channel": "#ingest-notifications",
"text": "${{ env.SLACK_MESSAGE }}",
"blocks": [
{
Expand All @@ -48,5 +50,4 @@ jobs:
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_MESSAGE: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"
2 changes: 1 addition & 1 deletion .github/workflows/check-default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
go-version-file: .go-version
#  when using ubuntu-latest, python 3.10 is not the default version.
- name: Fix Code is not compatible with Python 3.12
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Run check-default
Expand Down
Loading

0 comments on commit de94730

Please sign in to comment.