Skip to content

Commit

Permalink
Merge branch 'main' into default-dsl
Browse files Browse the repository at this point in the history
  • Loading branch information
elasticmachine authored Nov 14, 2023
2 parents 3190d48 + 3d6ba88 commit 0c1677b
Show file tree
Hide file tree
Showing 416 changed files with 22,354 additions and 19,183 deletions.
8 changes: 8 additions & 0 deletions .buildkite/pipelines/intake.template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ steps:
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part4
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
Expand Down
10 changes: 9 additions & 1 deletion .buildkite/pipelines/intake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,22 @@ steps:
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part4
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: ["7.17.15", "8.11.1", "8.12.0"]
BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
Expand Down
32 changes: 32 additions & 0 deletions .buildkite/pipelines/periodic-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1073,6 +1073,22 @@ steps:
env:
BWC_VERSION: 7.17.15

- label: "{{matrix.image}} / 7.17.16 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.16
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2004
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 7.17.16

- label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
timeout_in_minutes: 300
Expand Down Expand Up @@ -1713,6 +1729,22 @@ steps:
env:
BWC_VERSION: 8.11.1

- label: "{{matrix.image}} / 8.11.2 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.2
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2004
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.11.2

- label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
timeout_in_minutes: 300
Expand Down
20 changes: 20 additions & 0 deletions .buildkite/pipelines/periodic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,16 @@ steps:
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 7.17.15
- label: 7.17.16 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.16#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 7.17.16
- label: 8.0.0 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
timeout_in_minutes: 300
Expand Down Expand Up @@ -1052,6 +1062,16 @@ steps:
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.11.1
- label: 8.11.2 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.2#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.11.2
- label: 8.12.0 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
timeout_in_minutes: 300
Expand Down
11 changes: 11 additions & 0 deletions .buildkite/pipelines/pull-request/part-4-fips.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
config:
allow-labels: "Team:Security"
steps:
- label: part-4-fips
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
14 changes: 14 additions & 0 deletions .buildkite/pipelines/pull-request/part-4-windows.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
config:
allow-labels: "test-windows"
steps:
- label: part-4-windows
command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-windows-2022
machineType: custom-32-98304
diskType: pd-ssd
diskSizeGb: 350
env:
GRADLE_TASK: checkPart4
11 changes: 11 additions & 0 deletions .buildkite/pipelines/pull-request/part-4.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
config:
skip-target-branches: "7.17"
steps:
- label: part-4
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
2 changes: 2 additions & 0 deletions .ci/bwcVersions
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ BWC_VERSION:
- "7.17.13"
- "7.17.14"
- "7.17.15"
- "7.17.16"
- "8.0.0"
- "8.0.1"
- "8.1.0"
Expand Down Expand Up @@ -104,4 +105,5 @@ BWC_VERSION:
- "8.10.4"
- "8.11.0"
- "8.11.1"
- "8.11.2"
- "8.12.0"
4 changes: 2 additions & 2 deletions .ci/snapshotBwcVersions
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
BWC_VERSION:
- "7.17.15"
- "8.11.1"
- "7.17.16"
- "8.11.2"
- "8.12.0"
7 changes: 0 additions & 7 deletions build-tools-internal/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,6 @@ gradlePlugin {
id = 'elasticsearch.build'
implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin'
}
buildComplete {
id = 'elasticsearch.build-complete'
implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchBuildCompletePlugin'
}
distro {
id = 'elasticsearch.distro'
implementationClass = 'org.elasticsearch.gradle.internal.distribution.ElasticsearchDistributionPlugin'
Expand Down Expand Up @@ -270,8 +266,6 @@ dependencies {
api buildLibs.apache.rat
api buildLibs.jna
api buildLibs.shadow.plugin
api buildLibs.gradle.enterprise

// for our ide tweaking
api buildLibs.idea.ext
// When upgrading forbidden apis, ensure dependency version is bumped in ThirdPartyPrecommitPlugin as well
Expand All @@ -286,7 +280,6 @@ dependencies {
api buildLibs.asm.tree
api buildLibs.httpclient
api buildLibs.httpcore

compileOnly buildLibs.checkstyle
runtimeOnly "org.elasticsearch.gradle:reaper:$version"
testImplementation buildLibs.checkstyle
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import org.elasticsearch.gradle.util.GradleUtils

import java.nio.file.Files

String buildNumber = System.getenv('BUILD_NUMBER') ?: System.getenv('BUILDKITE_BUILD_NUMBER')
String performanceTest = System.getenv('BUILD_PERFORMANCE_TEST')
Boolean isNested = System.getProperty("scan.tag.NESTED") != null

if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(project) == false && isNested == false) {
def uploadFilePath = "build/${buildNumber}.tar.bz2"
File uploadFile = file(uploadFilePath)
project.gradle.buildFinished { result ->
println "build complete, generating: $uploadFile"
if (uploadFile.exists()) {
project.delete(uploadFile)
}

try {
ant.tar(destfile: uploadFile, compression: "bzip2", longfile: "gnu") {
fileset(dir: projectDir) {
Set<File> fileSet = fileTree(projectDir) {
include("**/*.hprof")
include("**/build/test-results/**/*.xml")
include("**/build/testclusters/**")
include("**/build/testrun/*/temp/**")
include("**/build/**/hs_err_pid*.log")
exclude("**/build/testclusters/**/data/**")
exclude("**/build/testclusters/**/distro/**")
exclude("**/build/testclusters/**/repo/**")
exclude("**/build/testclusters/**/extract/**")
exclude("**/build/testclusters/**/tmp/**")
exclude("**/build/testrun/*/temp/**/data/**")
exclude("**/build/testrun/*/temp/**/distro/**")
exclude("**/build/testrun/*/temp/**/repo/**")
exclude("**/build/testrun/*/temp/**/extract/**")
exclude("**/build/testrun/*/temp/**/tmp/**")
}
.files
.findAll { Files.isRegularFile(it.toPath()) }

if (fileSet.empty) {
// In cases where we don't match any workspace files, exclude everything
ant.exclude(name: "**/*")
} else {
fileSet.each {
ant.include(name: projectDir.toPath().relativize(it.toPath()))
}
}
}

fileset(dir: "${gradle.gradleUserHomeDir}/daemon/${gradle.gradleVersion}", followsymlinks: false) {
include(name: "**/daemon-${ProcessHandle.current().pid()}*.log")
}

fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false)

fileset(dir: "${project.projectDir}/.gradle/reaper", followsymlinks: false, erroronmissingdir: false)
}
} catch (Exception e) {
logger.lifecycle("Failed to archive additional logs", e)
}

if (uploadFile.exists() && System.getenv("BUILDKITE") == "true") {
try {
println "Uploading buildkite artifact: ${uploadFilePath}..."
new ProcessBuilder("buildkite-agent", "artifact", "upload", uploadFilePath)
.start()
.waitFor()

println "Generating buildscan link for artifact..."

def process = new ProcessBuilder("buildkite-agent", "artifact", "search", uploadFilePath, "--step", System.getenv('BUILDKITE_JOB_ID'), "--format", "%i").start()
process.waitFor()
def artifactUuid = (process.text ?: "").trim()

println "Artifact UUID: ${artifactUuid}"
if (artifactUuid) {
buildScan.link 'Artifact Upload', "https://buildkite.com/organizations/elastic/pipelines/${System.getenv('BUILDKITE_PIPELINE_SLUG')}/builds/${buildNumber}/jobs/${System.getenv('BUILDKITE_JOB_ID')}/artifacts/${artifactUuid}"
}
} catch (Exception e) {
logger.lifecycle("Failed to upload buildkite artifact", e)
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ buildScan {
}

buildFinished { result ->

buildScanPublished { scan ->
// Attach build scan link as build metadata
// See: https://buildkite.com/docs/pipelines/build-meta-data
Expand All @@ -143,18 +142,17 @@ buildScan {
// Add a build annotation
// See: https://buildkite.com/docs/agent/v3/cli-annotate
def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: ${result.failure ? 'failed' : 'successful'} build: <a href="${scan.buildScanUri}"><code>gradle ${gradle.startParameter.taskNames.join(' ')}</code></a></div>"""
new ProcessBuilder(
def process = [
'buildkite-agent',
'annotate',
'--context',
result.failure ? 'gradle-build-scans-failed' : 'gradle-build-scans',
'--append',
'--style',
result.failure ? 'error' : 'info',
body
)
.start()
.waitFor()
result.failure ? 'error' : 'info'
].execute()
process.withWriter { it.write(body) } // passing the body in as an argument has issues on Windows, so let's use stdin of the process instead
process.waitFor()
}
}
} else {
Expand Down
Loading

0 comments on commit 0c1677b

Please sign in to comment.