Skip to content

Commit

Permalink
Merge branch 'main' into kbn-team-1309-move-kibana-security-detection…
Browse files Browse the repository at this point in the history
…s-response
  • Loading branch information
marshallmain authored Dec 13, 2024
2 parents b0167fb + bf40e56 commit f8bf591
Show file tree
Hide file tree
Showing 10,325 changed files with 129,223 additions and 64,580 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
12 changes: 6 additions & 6 deletions .buildkite/ftr_oblt_stateful_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ disabled:
- x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts

#FTR configs
- x-pack/plugins/observability_solution/uptime/e2e/config.ts
- x-pack/solutions/observability/plugins/uptime/e2e/config.ts

# Elastic Synthetics configs
- x-pack/plugins/observability_solution/uptime/e2e/uptime/synthetics_run.ts
- x-pack/plugins/observability_solution/synthetics/e2e/config.ts
- x-pack/plugins/observability_solution/synthetics/e2e/synthetics/synthetics_run.ts
- x-pack/plugins/observability_solution/exploratory_view/e2e/synthetics_run.ts
- x-pack/plugins/observability_solution/ux/e2e/synthetics_run.ts
- x-pack/solutions/observability/plugins/uptime/e2e/uptime/synthetics_run.ts
- x-pack/solutions/observability/plugins/synthetics/e2e/config.ts
- x-pack/solutions/observability/plugins/synthetics/e2e/synthetics/synthetics_run.ts
- x-pack/solutions/observability/plugins/exploratory_view/e2e/synthetics_run.ts
- x-pack/solutions/observability/plugins/ux/e2e/synthetics_run.ts
- x-pack/plugins/observability_solution/slo/e2e/synthetics_run.ts

defaultQueue: 'n2-4-spot'
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/ftr_platform_stateful_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ enabled:
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group5/config.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
- x-pack/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts
Expand Down Expand Up @@ -264,7 +265,6 @@ enabled:
- x-pack/test/functional/apps/security/config.ts
- x-pack/test/functional/apps/snapshot_restore/config.ts
- x-pack/test/functional/apps/spaces/config.ts
- x-pack/test/functional/apps/spaces/solution_view_flag_enabled/config.ts
- x-pack/test/functional/apps/status_page/config.ts
- x-pack/test/functional/apps/transform/creation/index_pattern/config.ts
- x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts
Expand Down
3 changes: 2 additions & 1 deletion .buildkite/ftr_security_stateful_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,5 @@ enabled:
- x-pack/test/cloud_security_posture_functional/config.ts
- x-pack/test/cloud_security_posture_functional/config.agentless.ts
- x-pack/test/cloud_security_posture_functional/data_views/config.ts
- x-pack/test/automatic_import_api_integration/security/config_basic.ts
- x-pack/test/automatic_import_api_integration/apis/config_basic.ts
- x-pack/test/automatic_import_api_integration/apis/config_graphs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: bk-kibana-es-forward-compatibility-testing-9-dot-0
description: Forward compatibility testing between Kibana 8.18 and ES 9+
links:
- url: 'https://buildkite.com/elastic/kibana-es-forward-compatibility-testing-9-dot-0'
title: Pipeline link
spec:
type: buildkite-pipeline
system: buildkite
owner: 'group:kibana-operations'
implementation:
apiVersion: buildkite.elastic.dev/v1
kind: Pipeline
metadata:
name: kibana / ES Forward Compatibility Testing 9.0
description: Forward compatibility testing between Kibana 8.18 and ES 9+
spec:
env:
SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
REPORT_FAILED_TESTS_TO_GITHUB: 'true'
allow_rebuilds: false
branch_configuration: main
default_branch: main
repository: elastic/kibana
pipeline_file: .buildkite/pipelines/es_forward_9_dot_0.yml # Note: this file exists in 8.x only and should be moved into 8.18 once the branch is cut
provider_settings:
prefix_pull_request_fork_branch_names: false
trigger_mode: none
teams:
kibana-operations:
access_level: MANAGE_BUILD_AND_READ
appex-qa:
access_level: MANAGE_BUILD_AND_READ
kibana-tech-leads:
access_level: MANAGE_BUILD_AND_READ
everyone:
access_level: BUILD_AND_READ
tags:
- kibana
3 changes: 2 additions & 1 deletion .buildkite/pipeline-resource-definitions/locations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@ spec:
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-console-definitions-sync.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-0.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml
Expand All @@ -50,4 +52,3 @@ spec:
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ spec:
message: Trigger ES forward compatibility tests
env:
TRIGGER_PIPELINE_SET: es-forward
Trigger ES forward compatibility tests 9.0:
cronline: 0 5 * * *
message: Trigger ES forward compatibility tests 9.0
env:
TRIGGER_PIPELINE_SET: es-forward-9-dot-0
Trigger artifact staging builds:
cronline: 0 7 * * * America/New_York
message: Trigger artifact staging builds
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pipelines/on_merge_unsupported_ftrs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ steps:
depends_on: build
timeout_in_minutes: 120
artifact_paths:
- 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
- 'x-pack/solutions/observability/plugins/synthetics/e2e/.journeys/**/*'
retry:
automatic:
- exit_status: '-1'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ steps:
- check_oas_snapshot
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/observability_solution/exploratory_view/e2e/.journeys/**/*'
- 'x-pack/solutions/observability/plugins/exploratory_view/e2e/.journeys/**/*'
retry:
automatic:
- exit_status: '-1'
Expand Down
18 changes: 18 additions & 0 deletions .buildkite/pipelines/pull_request/scout_ui_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
steps:
- command: .buildkite/scripts/steps/functional/scout_ui_tests.sh
label: 'Scout UI Tests'
agents:
machineType: n2-standard-4
preemptible: true
depends_on:
- build
- quick_checks
- checks
- linting
- linting_with_types
- check_types
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '-1'
limit: 2
2 changes: 1 addition & 1 deletion .buildkite/pipelines/pull_request/synthetics_plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ steps:
- check_oas_snapshot
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
- 'x-pack/solutions/observability/plugins/synthetics/e2e/.journeys/**/*'
retry:
automatic:
- exit_status: '-1'
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pipelines/pull_request/uptime_plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ steps:
- check_oas_snapshot
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
- 'x-pack/solutions/observability/plugins/synthetics/e2e/.journeys/**/*'
retry:
automatic:
- exit_status: '-1'
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pipelines/pull_request/ux_plugin_e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ steps:
- check_oas_snapshot
timeout_in_minutes: 60
artifact_paths:
- 'x-pack/plugins/observability_solution/ux/e2e/.journeys/**/*'
- 'x-pack/solutions/observability/plugins/ux/e2e/.journeys/**/*'
retry:
automatic:
- exit_status: '-1'
Expand Down
44 changes: 27 additions & 17 deletions .buildkite/scripts/pipelines/pull_request/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ const getPipeline = (filename: string, removeSteps = true) => {

if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/observability_solution\/observability_onboarding/,
/^x-pack\/solutions\/observability\/plugins\/observability_onboarding/,
/^x-pack\/plugins\/fleet/,
])) ||
GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
Expand All @@ -114,16 +114,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
}

if (
(await doAnyChangesMatch([/^x-pack\/plugins\/observability_solution\/exploratory_view/])) ||
(await doAnyChangesMatch([/^x-pack\/solutions\/observability\/plugins\/exploratory_view/])) ||
GITHUB_PR_LABELS.includes('ci:synthetics-runner-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/exploratory_view_plugin.yml'));
}

if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/observability_solution\/synthetics/,
/^x-pack\/plugins\/observability_solution\/exploratory_view/,
/^x-pack\/solutions\/observability\/plugins\/synthetics/,
/^x-pack\/solutions\/observability\/plugins\/exploratory_view/,
])) ||
GITHUB_PR_LABELS.includes('ci:synthetics-runner-suites')
) {
Expand All @@ -133,8 +133,8 @@ const getPipeline = (filename: string, removeSteps = true) => {

if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/observability_solution\/ux/,
/^x-pack\/plugins\/observability_solution\/exploratory_view/,
/^x-pack\/solutions\/observability\/plugins\/ux/,
/^x-pack\/solutions\/observability\/plugins\/exploratory_view/,
])) ||
GITHUB_PR_LABELS.includes('ci:synthetics-runner-suites')
) {
Expand Down Expand Up @@ -248,16 +248,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
/^x-pack\/plugins\/security_solution_ess/,
/^x-pack\/plugins\/security_solution_serverless/,
/^x-pack\/plugins\/task_manager/,
/^x-pack\/plugins\/timelines/,
/^x-pack\/solutions\/security\/plugins\/timelines/,
/^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
/^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
/^x-pack\/plugins\/triggers_actions_ui\/server\/connector_types\/openai/,
/^x-pack\/plugins\/triggers_actions_ui\/server\/connector_types\/bedrock/,
/^x-pack\/plugins\/usage_collection\/public/,
/^x-pack\/plugins\/elastic_assistant/,
/^x-pack\/solutions\/security\/plugins\/elastic_assistant/,
/^x-pack\/packages\/security-solution/,
/^x-pack\/packages\/kbn-elastic-assistant/,
/^x-pack\/packages\/kbn-elastic-assistant-common/,
/^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant/,
/^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant-common/,
/^x-pack\/test\/functional\/es_archives\/security_solution/,
/^x-pack\/test\/security_solution_cypress/,
])) ||
Expand Down Expand Up @@ -290,7 +290,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
/^packages\/kbn-es-query/,
/^packages\/kbn-i18n/,
/^packages\/kbn-i18n-react/,
/^packages\/kbn-expandable-flyout/,
/^x-pack\/solutions\/security\/packages\/expandable-flyout/,
/^packages\/kbn-grouping/,
/^packages\/kbn-resizable-layout/,
/^packages\/kbn-rison/,
Expand Down Expand Up @@ -321,21 +321,21 @@ const getPipeline = (filename: string, removeSteps = true) => {
/^src\/plugins\/ui_actions/,
/^src\/plugins\/unified_histogram/,
/^src\/plugins\/unified_search/,
/^x-pack\/packages\/kbn-elastic-assistant/,
/^x-pack\/packages\/kbn-elastic-assistant-common/,
/^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant/,
/^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant-common/,
/^x-pack\/packages\/security-solution/,
/^x-pack\/plugins\/alerting/,
/^x-pack\/plugins\/cases/,
/^x-pack\/plugins\/data_views\/common/,
/^x-pack\/plugins\/elastic_assistant/,
/^x-pack\/solutions\/security\/plugins\/elastic_assistant/,
/^x-pack\/plugins\/lists/,
/^x-pack\/plugins\/rule_registry\/common/,
/^x-pack\/plugins\/security_solution/,
/^x-pack\/plugins\/security_solution_ess/,
/^x-pack\/plugins\/security_solution_serverless/,
/^x-pack\/plugins\/task_manager/,
/^x-pack\/plugins\/threat_intelligence/,
/^x-pack\/plugins\/timelines/,
/^x-pack\/solutions\/security\/plugins\/threat_intelligence/,
/^x-pack\/solutions\/security\/plugins\/timelines/,
/^x-pack\/plugins\/triggers_actions_ui/,
/^x-pack\/plugins\/usage_collection\/public/,
/^x-pack\/test\/functional\/es_archives\/security_solution/,
Expand All @@ -350,7 +350,7 @@ const getPipeline = (filename: string, removeSteps = true) => {

if (
((await doAnyChangesMatch([
/^x-pack\/plugins\/osquery/,
/^x-pack\/platform\/plugins\/shared\/osquery/,
/^x-pack\/test\/osquery_cypress/,
/^x-pack\/plugins\/security_solution/,
])) ||
Expand Down Expand Up @@ -378,6 +378,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
);
}

if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/discover_enhanced\/ui_tests/,
/^packages\/kbn-scout/,
])) ||
GITHUB_PR_LABELS.includes('ci:scout-ui-tests')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/scout_ui_tests.yml'));
}

pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));

// remove duplicated steps
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true

source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh

cd x-pack/plugins/osquery
cd x-pack/platform/plugins/shared/osquery
set +e

export BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate serverless-cypress-defend-workflows)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { getVersionsFile, BuildkiteTriggerStep } from '#pipeline-utils';

const pipelineSets = {
'es-forward': 'kibana-es-forward-compatibility-testing',
'es-forward-9-dot-0': 'kibana-es-forward-compatibility-testing-9-dot-0',
'artifacts-snapshot': 'kibana-artifacts-snapshot',
'artifacts-staging': 'kibana-artifacts-staging',
'artifacts-trigger': 'kibana-artifacts-trigger',
Expand Down Expand Up @@ -40,6 +41,10 @@ async function main() {
pipelineSteps.push(...getESForwardPipelineTriggers());
break;
}
case 'es-forward-9-dot-0': {
pipelineSteps.push(...getESForward9Dot0PipelineTriggers());
break;
}
case 'artifacts-snapshot': {
pipelineSteps.push(...getArtifactSnapshotPipelineTriggers());
break;
Expand Down Expand Up @@ -90,6 +95,40 @@ export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] {
});
}

/**
* This pipeline is testing the forward compatibility of Kibana with different versions of Elasticsearch for 9.0.
* Should be triggered for combinations of ([email protected] + [email protected] {current open branches on the same major})
*/
export function getESForward9Dot0PipelineTriggers(): BuildkiteTriggerStep[] {
const versions = getVersionsFile();
const KIBANA_8_X = versions.versions.find((v) => v.branch === '8.x');
if (!KIBANA_8_X) {
throw new Error(
'Update ES forward compatibility 9.0 pipeline to remove 8.x and add version 8.18'
);
}
const targetESVersions = versions.versions.filter(
(v) => v.branch.startsWith('9.') || (v.branch.includes('main') && v.version.startsWith('9.0.0'))
);

return targetESVersions.map(({ version }) => {
return {
trigger: pipelineSets['es-forward-9-dot-0'],
async: true,
label: `Triggering Kibana ${KIBANA_8_X.version} + ES ${version} forward compatibility`,
build: {
message: process.env.MESSAGE || `ES forward-compatibility test for ES ${version}`,
branch: KIBANA_8_X.branch,
commit: 'HEAD',
env: {
ES_SNAPSHOT_MANIFEST: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${version}/manifest-latest-verified.json`,
DRY_RUN: process.env.DRY_RUN,
},
},
} as BuildkiteTriggerStep;
});
}

/**
* This pipeline creates Kibana artifact snapshots for all open branches.
* Should be triggered for all open branches in the versions.json
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ source .buildkite/scripts/common/util.sh

echo --- Elastic Assistant OpenAPI Code Generation

(cd x-pack/packages/kbn-elastic-assistant-common && yarn openapi:generate)
(cd x-pack/platform/packages/shared/kbn-elastic-assistant-common && yarn openapi:generate)
check_for_changed_files "yarn openapi:generate" true
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ source .buildkite/scripts/common/util.sh

echo --- Osquery OpenAPI Code Generation

(cd x-pack/plugins/osquery && yarn openapi:generate)
(cd x-pack/platform/plugins/shared/osquery && yarn openapi:generate)
check_for_changed_files "yarn openapi:generate" true
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ echo "--- Exploratory View plugin @elastic/synthetics Tests"

cd "$XPACK_DIR"

node plugins/observability_solution/exploratory_view/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
node solutions/observability/plugins/exploratory_view/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
Loading

0 comments on commit f8bf591

Please sign in to comment.