From 40817b2faebdd3bcb3e6e9da71c6274506e707c8 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:38:44 -0500 Subject: [PATCH 1/3] [Security Solution][Endpoint] Fix event filters create/update API body validator to allow for nested fields (#124272) * change event filters schema * changes to generator + FTR test --- .../exceptions_list_item_generator.ts | 28 +++++++++++++++---- .../validators/event_filter_validator.ts | 25 ++++++----------- .../apis/endpoint_artifacts/event_filters.ts | 4 +-- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts b/x-pack/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts index 027b4b692c474..f15e3f418427a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts @@ -165,13 +165,31 @@ export class ExceptionsListItemGenerator extends BaseDataGenerator = {}): ExceptionListItemSchema { - const eventFilter = this.generate(overrides); - - return { - ...eventFilter, + return this.generate({ name: `Event filter (${this.randomString(5)})`, list_id: ENDPOINT_EVENT_FILTERS_LIST_ID, - }; + entries: [ + { + field: 'process.pe.company', + operator: 'excluded', + type: 'match', + value: 'elastic', + }, + { + entries: [ + { + field: 'status', + operator: 'included', + type: 'match', + value: 'dfdfd', + }, + ], + field: 'process.Ext.code_signature', + type: 'nested', + }, + ], + ...overrides, + }); } generateEventFilterForCreate( diff --git a/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/event_filter_validator.ts b/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/event_filter_validator.ts index 55eb3a9a2a63b..875bc04646321 100644 --- a/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/event_filter_validator.ts +++ b/x-pack/plugins/security_solution/server/lists_integration/endpoint/validators/event_filter_validator.ts @@ -26,24 +26,17 @@ function validateField(field: string) { } } -const EntrySchema = schema.object({ - field: schema.string({ validate: validateField }), - operator: schema.oneOf([schema.literal('included'), schema.literal('excluded')]), - type: schema.oneOf([schema.literal('match'), schema.literal('match_any')]), - value: schema.oneOf([schema.arrayOf(schema.string()), schema.string()]), -}); - -const NestedEntrySchema = schema.object({ - field: schema.string({ validate: validateField }), - type: schema.literal('nested'), - entries: schema.arrayOf(EntrySchema), -}); - -const EntriesSchema = schema.oneOf([EntrySchema, NestedEntrySchema]); - const EventFilterDataSchema = schema.object( { - entries: schema.arrayOf(EntriesSchema, { minSize: 1 }), + entries: schema.arrayOf( + schema.object( + { + field: schema.string({ validate: validateField }), + }, + { unknowns: 'ignore' } + ), + { minSize: 1 } + ), }, { unknowns: 'ignore', diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts b/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts index 8505beebf82ac..019d278481bcf 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts @@ -138,7 +138,7 @@ export default function ({ getService }: FtrProviderContext) { describe('and has authorization to manage endpoint security', () => { for (const eventFilterCall of eventFilterCalls) { - it(`should error on [${eventFilterCall.method} if invalid field`, async () => { + it(`should error on [${eventFilterCall.method}] if invalid field`, async () => { const body = eventFilterCall.getBody({}); body.entries[0].field = 'some.invalid.field'; @@ -148,7 +148,7 @@ export default function ({ getService }: FtrProviderContext) { .send(body) .expect(400) .expect(anEndpointArtifactError) - .expect(anErrorMessageWith(/types that failed validation:/)); + .expect(anErrorMessageWith(/invalid field: some\.invalid\.field/)); }); it(`should error on [${eventFilterCall.method}] if more than one OS is set`, async () => { From 68466226b3b212532e2402cf627c128b8bab6afc Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 1 Feb 2022 14:29:34 -0500 Subject: [PATCH 2/3] [Fleet] Support bundling packages as zip archives with Kibana source (#122297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support installing bundled packages during Fleet setup * Fix failing API integration test * Sort packages in test * Move get bundled packages + add tests * Fix mock installPackage type * Don't attempt to install already-installed bundled packageS * Use unique installation status for bundled packages * Fix failing tests * Fix type error * Fix logic for latest value * Remove bundled packages from test since they no longer come back from list endpoint * Change assertion for empty preconfiguration update request * Add all bundled packages + refactor install to use Promise.allSettled * Fix name comparison again * Rework approach to use preferred source at install-time instead * Add helper script + update bundled packages + allow updates for upload path * Fix failing test * Address code review + add additional tests for update cases * Fix type error * Remove configOverride concept for now * Fix another type error 🙃 * Add build step for bundling fleet packages * Fix unused import in build step * Keep bundled_packages directory in place to prevent setup failures * Remove gitkeep + make bundled package directory lookup fault tolerant * Update gitignore * Add doc comment + use build.resolvePath * Add --use-snapshot-registry CLI flag * Update snapshots * Rename arg to use-snapshot-epr * Don't fail the build on failure to bundled packages * Update all checksums + don't attempt download multiple times * Skip checksum process for bundled packages for now * mkdirp destination directory to fix tests * Revert build_distributables file Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .gitignore | 2 + fleet_packages.json | 36 + src/dev/build/args.test.ts | 7 + src/dev/build/args.ts | 2 + src/dev/build/build_distributables.ts | 2 + src/dev/build/tasks/bundle_fleet_packages.ts | 104 +++ src/dev/build/tasks/index.ts | 1 + src/dev/precommit_hook/casing_check_config.js | 3 + x-pack/plugins/fleet/common/constants/epm.ts | 1 + .../plugins/fleet/common/types/models/epm.ts | 19 +- .../server/services/epm/archive/validation.ts | 4 +- .../epm/packages/bulk_install_packages.ts | 143 ++- .../epm/packages/get_bundled_packages.ts | 43 + .../services/epm/packages/install.test.ts | 7 +- .../server/services/epm/packages/install.ts | 18 +- .../server/services/preconfiguration.test.ts | 883 +++++++++++------- .../fleet/server/services/preconfiguration.ts | 20 +- .../fleet_api_integration/apis/epm/list.ts | 2 +- .../fleet_api_integration/apis/epm/setup.ts | 4 +- .../apis/preconfiguration/preconfiguration.ts | 6 +- 20 files changed, 891 insertions(+), 416 deletions(-) create mode 100644 fleet_packages.json create mode 100644 src/dev/build/tasks/bundle_fleet_packages.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts diff --git a/.gitignore b/.gitignore index 7cf2e31ac0a01..18302cebd1641 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,5 @@ elastic-agent-* fleet-server-* elastic-agent.yml fleet-server.yml + +/x-pack/plugins/fleet/server/bundled_packages diff --git a/fleet_packages.json b/fleet_packages.json new file mode 100644 index 0000000000000..9dd281dacae8a --- /dev/null +++ b/fleet_packages.json @@ -0,0 +1,36 @@ +/* + + Packages in this file are considered "bundled" and are installed as part of Fleet's setup process. Each entry points to a valid version name + avaiable in the Elastic Package Registry service, and must include a sha-512 checksum of the `.zip` archive for the given package. + + You may opt in to using the "snapshot" environment of the EPR service by passing the `--use-epr-snapshot-registry` flag to `yarn build`. This will + cause the package archive download to pull from the "spapshot" environment instead of the "production" environment. Be aware that not all packages + exist in the snapshot environment, so you may have errors when fetching package versions. It's recommended to alter this file to contain _only_ the + packages you're testing when using the snapshot environment. + + These files don't include any kind of checksum, but they should eventually include a package signature as introduced in https://github.com/elastic/elastic-package/issues/583 + in order to verify package integrity. +*/ + +[ + { + "name": "apm", + "version": "8.0.0" + }, + { + "name": "elastic_agent", + "version": "1.3.0" + }, + { + "name": "endpoint", + "version": "1.4.1" + }, + { + "name": "fleet_server", + "version": "1.1.0" + }, + { + "name": "synthetics", + "version": "0.9.0" + } +] diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index 5601e063414b3..f372a9e88f2f9 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -43,6 +43,7 @@ it('build default and oss dist for current platform, without packages, by defaul "initialize": true, "isRelease": false, "targetAllPlatforms": false, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -73,6 +74,7 @@ it('builds packages if --all-platforms is passed', () => { "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -103,6 +105,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -133,6 +136,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -164,6 +168,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -202,6 +207,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , @@ -233,6 +239,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () => "initialize": true, "isRelease": false, "targetAllPlatforms": true, + "useSnapshotEpr": false, "versionQualifier": "", }, "log": , diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index d890dbef4e74f..9c38fe1866856 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -40,6 +40,7 @@ export function readCliArgs(argv: string[]) { 'silent', 'debug', 'help', + 'use-snapshot-epr', ], alias: { v: 'verbose', @@ -115,6 +116,7 @@ export function readCliArgs(argv: string[]) { createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']), createDockerContexts: !Boolean(flags['skip-docker-contexts']), targetAllPlatforms: Boolean(flags['all-platforms']), + useSnapshotEpr: Boolean(flags['use-snapshot-epr']), }; return { diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 992486796dfad..ba271eb6ba4da 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -30,6 +30,7 @@ export interface BuildOptions { versionQualifier: string | undefined; targetAllPlatforms: boolean; createExamplePlugins: boolean; + useSnapshotEpr: boolean; } export async function buildDistributables(log: ToolingLog, options: BuildOptions): Promise { @@ -84,6 +85,7 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions await run(Tasks.CleanTypescript); await run(Tasks.CleanExtraFilesFromModules); await run(Tasks.CleanEmptyFolders); + await run(Tasks.BundleFleetPackages); } /** diff --git a/src/dev/build/tasks/bundle_fleet_packages.ts b/src/dev/build/tasks/bundle_fleet_packages.ts new file mode 100644 index 0000000000000..0fd584354494f --- /dev/null +++ b/src/dev/build/tasks/bundle_fleet_packages.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import axios from 'axios'; +import JSON5 from 'json5'; + +// @ts-expect-error untyped internal module used to prevent axios from using xhr adapter in tests +import AxiosHttpAdapter from 'axios/lib/adapters/http'; + +import { ToolingLog } from '@kbn/dev-utils'; +import { closeSync, openSync, writeSync } from 'fs'; +import { dirname } from 'path'; +import { readCliArgs } from '../args'; + +import { Task, read, mkdirp } from '../lib'; + +const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/server/bundled_packages'; + +interface FleetPackage { + name: string; + version: string; +} + +export const BundleFleetPackages: Task = { + description: 'Bundling fleet packages', + + async run(config, log, build) { + log.info('Fetching fleet packages from package registry'); + log.indent(4); + + // Support the `--use-snapshot-epr` command line argument to fetch from the snapshot registry + // in development or test environments + const { buildOptions } = readCliArgs(process.argv); + const eprUrl = buildOptions?.useSnapshotEpr + ? 'https://epr-snapshot.elastic.co' + : 'https://epr.elastic.co'; + + const configFilePath = config.resolveFromRepo('fleet_packages.json'); + const fleetPackages = (await read(configFilePath)) || '[]'; + + await Promise.all( + JSON5.parse(fleetPackages).map(async (fleetPackage: FleetPackage) => { + const archivePath = `${fleetPackage.name}-${fleetPackage.version}.zip`; + const archiveUrl = `${eprUrl}/epr/${fleetPackage.name}/${fleetPackage.name}-${fleetPackage.version}.zip`; + + const destination = build.resolvePath(BUNDLED_PACKAGES_DIR, archivePath); + + try { + await downloadPackageArchive({ log, url: archiveUrl, destination }); + } catch (error) { + log.warning(`Failed to download bundled package archive ${archivePath}`); + log.warning(error); + } + }) + ); + }, +}; + +/** + * We need to skip the checksum process on Fleet's bundled packages for now because we can't reliably generate + * a consistent checksum for the `.zip` file returned from the EPR service. This download process should be updated + * to verify packages using the proposed package signature field provided in https://github.com/elastic/elastic-package/issues/583 + */ +async function downloadPackageArchive({ + log, + url, + destination, +}: { + log: ToolingLog; + url: string; + destination: string; +}) { + log.info(`Downloading bundled package from ${url}`); + + await mkdirp(dirname(destination)); + const file = openSync(destination, 'w'); + + try { + const response = await axios.request({ + url, + responseType: 'stream', + adapter: AxiosHttpAdapter, + }); + + await new Promise((resolve, reject) => { + response.data.on('data', (chunk: Buffer) => { + writeSync(file, chunk); + }); + + response.data.on('error', reject); + response.data.on('end', resolve); + }); + } catch (error) { + log.warning(`Error downloading bundled package from ${url}`); + log.warning(error); + } finally { + closeSync(file); + } +} diff --git a/src/dev/build/tasks/index.ts b/src/dev/build/tasks/index.ts index 5043be288928e..f158634829100 100644 --- a/src/dev/build/tasks/index.ts +++ b/src/dev/build/tasks/index.ts @@ -10,6 +10,7 @@ export * from './bin'; export * from './build_kibana_platform_plugins'; export * from './build_kibana_example_plugins'; export * from './build_packages_task'; +export * from './bundle_fleet_packages'; export * from './clean_tasks'; export * from './copy_source_task'; export * from './create_archives_sources_task'; diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index e60eb936bf578..e5581631f6baf 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -61,6 +61,9 @@ export const IGNORE_FILE_GLOBS = [ 'x-pack/plugins/maps/server/fonts/**/*', + // Bundled package names typically use a format like ${pkgName}-${pkgVersion}, so don't lint them + 'x-pack/plugins/fleet/server/bundled_packages/**/*', + // Bazel default files '**/WORKSPACE.bazel', '**/BUILD.bazel', diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 12885be5a8aea..fba5893400d8c 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -60,4 +60,5 @@ export const installationStatuses = { Installing: 'installing', InstallFailed: 'install_failed', NotInstalled: 'not_installed', + InstalledBundled: 'installed_bundled', } as const; diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index de92ddf1fcfae..983ee7fff3db1 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -45,7 +45,11 @@ export interface DefaultPackagesInstallationError { export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown'; export type InstallSource = 'registry' | 'upload'; -export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed'; +export type EpmPackageInstallStatus = + | 'installed' + | 'installing' + | 'install_failed' + | 'installed_bundled'; export type DetailViewPanelName = 'overview' | 'policies' | 'assets' | 'settings' | 'custom'; export type ServiceName = 'kibana' | 'elasticsearch'; @@ -410,13 +414,22 @@ export interface PackageUsageStats { agent_policy_count: number; } -export type Installable = Installed | Installing | NotInstalled | InstallFailed; +export type Installable = + | InstalledRegistry + | Installing + | NotInstalled + | InstallFailed + | InstalledBundled; -export type Installed = T & { +export type InstalledRegistry = T & { status: InstallationStatus['Installed']; savedObject: SavedObject; }; +export type InstalledBundled = T & { + status: InstallationStatus['InstalledBundled']; +}; + export type Installing = T & { status: InstallationStatus['Installing']; savedObject: SavedObject; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts index 8d20d88c7a9aa..5fdbdc5b17332 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/validation.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/validation.ts @@ -209,9 +209,9 @@ export function parseAndVerifyDataStreams( streams: manifestStreams, ...restOfProps } = manifest; - if (!(dataStreamTitle && release && type)) { + if (!(dataStreamTitle && type)) { throw new PackageInvalidArchiveError( - `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'release', 'type'` + `Invalid manifest for data stream '${dataStreamPath}': one or more fields missing of 'title', 'type'` ); } const streams = parseAndVerifyStreams(manifestStreams, dataStreamPath); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 3178881b7ce09..a32809672e1b4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -14,6 +14,7 @@ import type { InstallResult } from '../../../types'; import { installPackage, isPackageVersionOrLaterInstalled } from './install'; import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; +import { getBundledPackages } from './get_bundled_packages'; interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; @@ -21,6 +22,7 @@ interface BulkInstallPackagesParams { esClient: ElasticsearchClient; force?: boolean; spaceId: string; + preferredSource?: 'registry' | 'bundled'; } export async function bulkInstallPackages({ @@ -29,9 +31,12 @@ export async function bulkInstallPackages({ esClient, spaceId, force, + preferredSource = 'registry', }: BulkInstallPackagesParams): Promise { const logger = appContextService.getLogger(); - const installSource = 'registry'; + + const bundledPackages = await getBundledPackages(); + const packagesResults = await Promise.allSettled( packagesToInstall.map((pkg) => { if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg); @@ -39,57 +44,113 @@ export async function bulkInstallPackages({ }) ); - logger.debug(`kicking off bulk install of ${packagesToInstall.join(', ')} from registry`); + logger.debug( + `kicking off bulk install of ${packagesToInstall.join( + ', ' + )} with preferred source of "${preferredSource}"` + ); + const bulkInstallResults = await Promise.allSettled( packagesResults.map(async (result, index) => { const packageName = getNameFromPackagesToInstall(packagesToInstall, index); - if (result.status === 'fulfilled') { - const pkgKeyProps = result.value; - const installedPackageResult = await isPackageVersionOrLaterInstalled({ - savedObjectsClient, - pkgName: pkgKeyProps.name, - pkgVersion: pkgKeyProps.version, - }); - if (installedPackageResult) { - const { - name, - version, - installed_es: installedEs, - installed_kibana: installedKibana, - } = installedPackageResult.package; - return { - name, - version, - result: { - assets: [...installedEs, ...installedKibana], - status: 'already_installed', - installType: installedPackageResult.installType, - } as InstallResult, - }; + + if (result.status === 'rejected') { + return { name: packageName, error: result.reason }; + } + + const pkgKeyProps = result.value; + const installedPackageResult = await isPackageVersionOrLaterInstalled({ + savedObjectsClient, + pkgName: pkgKeyProps.name, + pkgVersion: pkgKeyProps.version, + }); + + if (installedPackageResult) { + const { + name, + version, + installed_es: installedEs, + installed_kibana: installedKibana, + } = installedPackageResult.package; + return { + name, + version, + result: { + assets: [...installedEs, ...installedKibana], + status: 'already_installed', + installType: installedPackageResult.installType, + } as InstallResult, + }; + } + + let installResult: InstallResult; + const pkgkey = Registry.pkgToPkgKey(pkgKeyProps); + + const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey); + + // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry + if (preferredSource === 'bundled') { + if (bundledPackage) { + logger.debug( + `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` + ); + installResult = await installPackage({ + savedObjectsClient, + esClient, + installSource: 'upload', + archiveBuffer: bundledPackage.buffer, + contentType: 'application/zip', + spaceId, + }); + } else { + installResult = await installPackage({ + savedObjectsClient, + esClient, + pkgkey, + installSource: 'registry', + spaceId, + force, + }); } - const installResult = await installPackage({ + } else { + // If preferred source is registry, attempt to install from registry first, then fall back to bundled packages on disk + installResult = await installPackage({ savedObjectsClient, esClient, - pkgkey: Registry.pkgToPkgKey(pkgKeyProps), - installSource, + pkgkey, + installSource: 'registry', spaceId, force, }); - if (installResult.error) { - return { - name: packageName, - error: installResult.error, - installType: installResult.installType, - }; - } else { - return { - name: packageName, - version: pkgKeyProps.version, - result: installResult, - }; + + // If we initially errored, try to install from bundled package on disk + if (installResult.error && bundledPackage) { + logger.debug( + `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk` + ); + installResult = await installPackage({ + savedObjectsClient, + esClient, + installSource: 'upload', + archiveBuffer: bundledPackage.buffer, + contentType: 'application/zip', + spaceId, + }); } } - return { name: packageName, error: result.reason }; + + if (installResult.error) { + return { + name: packageName, + error: installResult.error, + installType: installResult.installType, + }; + } + return { + name: packageName, + version: pkgKeyProps.version, + result: installResult, + }; }) ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts new file mode 100644 index 0000000000000..a9f9b754640cb --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_bundled_packages.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; +import fs from 'fs/promises'; + +import { appContextService } from '../../app_context'; + +const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages'); + +interface BundledPackage { + name: string; + buffer: Buffer; +} + +export async function getBundledPackages(): Promise { + try { + const dirContents = await fs.readdir(BUNDLED_PACKAGE_DIRECTORY); + const zipFiles = dirContents.filter((file) => file.endsWith('.zip')); + + const result = await Promise.all( + zipFiles.map(async (zipFile) => { + const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile)); + + return { + name: zipFile.replace(/\.zip$/, ''), + buffer: file, + }; + }) + ); + + return result; + } catch (err) { + const logger = appContextService.getLogger(); + logger.debug(`Unable to read bundled packages from ${BUNDLED_PACKAGE_DIRECTORY}`); + + return []; + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index cb04b5f583c5a..b74466bc6271a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -190,10 +190,11 @@ describe('install', () => { }); describe('upload', () => { - it('should send telemetry on install failure', async () => { + it('should send telemetry on update', async () => { jest .spyOn(obj, 'getInstallationObject') .mockImplementationOnce(() => Promise.resolve({ attributes: { version: '1.2.0' } } as any)); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); await installPackage({ spaceId: DEFAULT_SPACE_ID, installSource: 'upload', @@ -206,13 +207,11 @@ describe('install', () => { expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { currentVersion: '1.2.0', dryRun: false, - errorMessage: - 'Package upload only supports fresh installations. Package apache is already installed, please uninstall first.', eventType: 'package-install', installType: 'update', newVersion: '1.3.0', packageName: 'apache', - status: 'failure', + status: 'success', }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index a42e71022e052..21f0ae25d6faf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -19,11 +19,7 @@ import type { InstallSource, } from '../../../../common'; import { AUTO_UPGRADE_POLICIES_PACKAGES } from '../../../../common'; -import { - IngestManagerError, - PackageOperationNotSupportedError, - PackageOutdatedError, -} from '../../../errors'; +import { IngestManagerError, PackageOutdatedError } from '../../../errors'; import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants'; import type { KibanaAssetType } from '../../../types'; import { licenseService } from '../../'; @@ -402,12 +398,6 @@ async function installPackageByUpload({ telemetryEvent.installType = installType; telemetryEvent.currentVersion = installedPkg?.attributes.version || 'not_installed'; - if (installType !== 'install') { - throw new PackageOperationNotSupportedError( - `Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.` - ); - } - const installSource = 'upload'; const paths = await unpackBufferToCache({ name: packageInfo.name, @@ -463,7 +453,9 @@ async function installPackageByUpload({ } } -export type InstallPackageParams = { spaceId: string } & ( +export type InstallPackageParams = { + spaceId: string; +} & ( | ({ installSource: Extract } & InstallRegistryPackageParams) | ({ installSource: Extract } & InstallUploadedArchiveParams) ); @@ -472,6 +464,7 @@ export async function installPackage(args: InstallPackageParams) { if (!('installSource' in args)) { throw new Error('installSource is required'); } + const logger = appContextService.getLogger(); const { savedObjectsClient, esClient } = args; @@ -488,7 +481,6 @@ export async function installPackage(args: InstallPackageParams) { return response; } else if (args.installSource === 'upload') { const { archiveBuffer, contentType, spaceId } = args; - logger.debug(`kicking off install of uploaded package`); const response = installPackageByUpload({ savedObjectsClient, esClient, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 3bba721d0dffb..89b8098f477fd 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -6,6 +6,7 @@ */ import uuid from 'uuid'; + import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsErrorHelpers } from '../../../../../src/core/server'; @@ -16,6 +17,7 @@ import type { PackagePolicy, PreconfiguredAgentPolicy, PreconfiguredOutput, + RegistrySearchResult, } from '../../common/types'; import type { AgentPolicy, NewPackagePolicy, Output } from '../types'; @@ -31,12 +33,19 @@ import { } from './preconfiguration'; import { outputService } from './output'; import { packagePolicyService } from './package_policy'; +import { getBundledPackages } from './epm/packages/get_bundled_packages'; +import type { InstallPackageParams } from './epm/packages/install'; jest.mock('./agent_policy_update'); jest.mock('./output'); +jest.mock('./epm/packages/get_bundled_packages'); +jest.mock('./epm/archive'); const mockedOutputService = outputService as jest.Mocked; const mockedPackagePolicyService = packagePolicyService as jest.Mocked; +const mockedGetBundledPackages = getBundledPackages as jest.MockedFunction< + typeof getBundledPackages +>; const mockInstalledPackages = new Map(); const mockInstallPackageErrors = new Map(); @@ -110,42 +119,104 @@ function getPutPreconfiguredPackagesMock() { return soClient; } +jest.mock('./epm/registry', () => ({ + ...jest.requireActual('./epm/registry'), + async fetchFindLatestPackage(packageName: string): Promise { + return { + name: packageName, + version: '1.0.0', + description: '', + release: 'experimental', + title: '', + path: '', + download: '', + }; + }, +})); + jest.mock('./epm/packages/install', () => ({ - async installPackage({ - pkgkey, - force, - }: { - pkgkey: string; - force?: boolean; - }): Promise { - const [pkgName, pkgVersion] = pkgkey.split('-'); - const installError = mockInstallPackageErrors.get(pkgName); - if (installError) { + async installPackage(args: InstallPackageParams): Promise { + if (args.installSource === 'registry') { + const [pkgName, pkgVersion] = args.pkgkey.split('-'); + const installError = mockInstallPackageErrors.get(pkgName); + if (installError) { + return { + error: new Error(installError), + installType: 'install', + }; + } + + const installedPackage = mockInstalledPackages.get(pkgName); + if (installedPackage) { + if (installedPackage.version === pkgVersion) return installedPackage; + } + + const packageInstallation = { name: pkgName, version: pkgVersion, title: pkgName }; + mockInstalledPackages.set(pkgName, packageInstallation); + return { - error: new Error(installError), + status: 'installed', installType: 'install', }; - } + } else if (args.installSource === 'upload') { + const { archiveBuffer } = args; - const installedPackage = mockInstalledPackages.get(pkgName); - if (installedPackage) { - if (installedPackage.version === pkgVersion) return installedPackage; - } + // Treat the buffer value passed in tests as the package's name for simplicity + const pkgName = archiveBuffer.toString('utf8'); - const packageInstallation = { name: pkgName, version: pkgVersion, title: pkgName }; - mockInstalledPackages.set(pkgName, packageInstallation); + const installedPackage = mockInstalledPackages.get(pkgName); - return { - status: 'installed', - installType: 'install', - }; + if (installedPackage) { + return installedPackage; + } + + // Just install every bundled package at version '1.0.0' + const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName }; + mockInstalledPackages.set(pkgName, packageInstallation); + + return { status: 'installed', installType: 'install' }; + } }, ensurePackagesCompletedInstall() { return []; }, - isPackageVersionOrLaterInstalled() { + isPackageVersionOrLaterInstalled({ + soClient, + pkgName, + pkgVersion, + }: { + soClient: any; + pkgName: string; + pkgVersion: string; + }) { + const installedPackage = mockInstalledPackages.get(pkgName); + + if (installedPackage) { + if (installedPackage.version === pkgVersion) { + return { package: installedPackage, installType: 'reinstall' }; + } + + // Importing semver methods throws an error in jest, so just use a rough check instead + if (installedPackage.version < pkgVersion) { + return false; + } + if (installedPackage.version > pkgVersion) { + return { package: installedPackage, installType: 'rollback' }; + } + } + return false; }, + getInstallType: jest.fn(), + async updateInstallStatus(soClient: any, pkgName: string, status: string) { + const installedPackage = mockInstalledPackages.get(pkgName); + + if (!installedPackage) { + return; + } + + installedPackage.install_status = status; + }, })); jest.mock('./epm/packages/get', () => ({ @@ -160,6 +231,9 @@ jest.mock('./epm/packages/get', () => ({ getInstallation({ pkgName }: { pkgName: string }) { return mockInstalledPackages.get(pkgName) ?? false; }, + getInstallationObject({ pkgName }: { pkgName: string }) { + return mockInstalledPackages.get(pkgName) ?? false; + }, })); jest.mock('./epm/kibana/index_pattern/install'); @@ -217,271 +291,50 @@ describe('policy preconfiguration', () => { spyAgentPolicyServicBumpAllAgentPoliciesForOutput.mockClear(); }); - it('should perform a no-op when passed no policies or packages', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [], - [], - mockDefaultOutput, - DEFAULT_SPACE_ID - ); - - expect(policies.length).toBe(0); - expect(packages.length).toBe(0); - expect(nonFatalErrors.length).toBe(0); - }); - - it('should install packages successfully', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [], - [{ name: 'test_package', version: '3.0.0' }], - mockDefaultOutput, - DEFAULT_SPACE_ID - ); - - expect(policies.length).toBe(0); - expect(packages).toEqual(expect.arrayContaining(['test_package-3.0.0'])); - expect(nonFatalErrors.length).toBe(0); - }); - - it('should install packages and configure agent policies successfully', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [ - { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - package_policies: [ - { - package: { name: 'test_package' }, - name: 'Test package', - }, - ], - }, - ] as PreconfiguredAgentPolicy[], - [{ name: 'test_package', version: '3.0.0' }], - mockDefaultOutput, - DEFAULT_SPACE_ID - ); - - expect(policies.length).toEqual(1); - expect(policies[0].id).toBe('test-id'); - expect(packages).toEqual(expect.arrayContaining(['test_package-3.0.0'])); - expect(nonFatalErrors.length).toBe(0); - }); - - it('should not add new package policy to existing non managed policies', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - mockedPackagePolicyService.getByIDs.mockResolvedValue([ - { name: 'test_package1' } as PackagePolicy, - ]); - - mockConfiguredPolicies.set('test-id', { - name: 'Test policy', - description: 'Test policy description', - unenroll_timeout: 120, - namespace: 'default', - id: 'test-id', - package_policies: [ - { - name: 'test_package1', - }, - ], - } as PreconfiguredAgentPolicy); - - await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [ - { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - is_managed: false, - package_policies: [ - { - package: { name: 'test_package' }, - name: 'test_package1', - }, - { - package: { name: 'test_package' }, - name: 'test_package2', - }, - ], - }, - ] as PreconfiguredAgentPolicy[], - [{ name: 'test_package', version: '3.0.0' }], - mockDefaultOutput, - DEFAULT_SPACE_ID - ); - - expect(mockedPackagePolicyService.create).not.toBeCalled(); - }); - - it('should add new package policy to existing managed policies', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - mockedPackagePolicyService.getByIDs.mockResolvedValue([ - { name: 'test_package1' } as PackagePolicy, - ]); - - mockConfiguredPolicies.set('test-id', { - name: 'Test policy', - description: 'Test policy description', - unenroll_timeout: 120, - namespace: 'default', - id: 'test-id', - package_policies: [ - { - name: 'test_package1', - }, - ], - is_managed: true, - } as PreconfiguredAgentPolicy); - - await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [ - { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - is_managed: true, - package_policies: [ - { - package: { name: 'test_package' }, - name: 'test_package1', - }, - { - package: { name: 'test_package' }, - name: 'test_package2', - }, - ], - }, - ] as PreconfiguredAgentPolicy[], - [{ name: 'test_package', version: '3.0.0' }], - mockDefaultOutput, - DEFAULT_SPACE_ID - ); - - expect(mockedPackagePolicyService.create).toBeCalledTimes(1); - expect(mockedPackagePolicyService.create).toBeCalledWith( - expect.anything(), // so client - expect.anything(), // es client - expect.objectContaining({ - name: 'test_package2', - }), - expect.anything() // options - ); - }); + describe('with no bundled packages', () => { + mockedGetBundledPackages.mockResolvedValue([]); - it('should throw an error when trying to install duplicate packages', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + it('should perform a no-op when passed no policies or packages', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - await expect( - ensurePreconfiguredPackagesAndPolicies( + const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, [], - [ - { name: 'test_package', version: '3.0.0' }, - { name: 'test_package', version: '2.0.0' }, - ], + [], mockDefaultOutput, DEFAULT_SPACE_ID - ) - ).rejects.toThrow( - 'Duplicate packages specified in configuration: test_package-3.0.0, test_package-2.0.0' - ); - }); + ); - it('should not create a policy and throw an error if install fails for required package', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const policies: PreconfiguredAgentPolicy[] = [ - { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - package_policies: [ - { - package: { name: 'test_package' }, - name: 'Test package', - }, - ], - }, - ]; - mockInstallPackageErrors.set('test_package', 'REGISTRY ERROR'); + expect(policies.length).toBe(0); + expect(packages.length).toBe(0); + expect(nonFatalErrors.length).toBe(0); + }); + + it('should install packages successfully', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - await expect( - ensurePreconfiguredPackagesAndPolicies( + const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, - policies, + [], [{ name: 'test_package', version: '3.0.0' }], mockDefaultOutput, DEFAULT_SPACE_ID - ) - ).rejects.toThrow( - '[Test policy] could not be added. [test_package] could not be installed due to error: [Error: REGISTRY ERROR]' - ); - }); - - it('should not create a policy and throw an error if package is not installed for an unknown reason', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const policies: PreconfiguredAgentPolicy[] = [ - { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - package_policies: [ - { - id: 'test-package', - package: { name: 'test_package' }, - name: 'Test package', - }, - ], - }, - ]; + ); - await expect( - ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - policies, - [{ name: 'CANNOT_MATCH', version: 'x.y.z' }], - mockDefaultOutput, - DEFAULT_SPACE_ID - ) - ).rejects.toThrow( - '[Test policy] could not be added. [test_package] is not installed, add [test_package] to [xpack.fleet.packages] or remove it from [Test package].' - ); - }); + expect(policies.length).toBe(0); + expect(packages).toEqual(expect.arrayContaining(['test_package-3.0.0'])); + expect(nonFatalErrors.length).toBe(0); + }); - it('should not attempt to recreate or modify an agent policy if its ID is unchanged', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + it('should install packages and configure agent policies successfully', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const { policies: policiesA, nonFatalErrors: nonFatalErrorsA } = - await ensurePreconfiguredPackagesAndPolicies( + const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, [ @@ -489,120 +342,476 @@ describe('policy preconfiguration', () => { name: 'Test policy', namespace: 'default', id: 'test-id', - package_policies: [], + package_policies: [ + { + package: { name: 'test_package' }, + name: 'Test package', + }, + ], }, ] as PreconfiguredAgentPolicy[], - [], + [{ name: 'test_package', version: '3.0.0' }], mockDefaultOutput, DEFAULT_SPACE_ID ); - expect(policiesA.length).toEqual(1); - expect(policiesA[0].id).toBe('test-id'); - expect(nonFatalErrorsA.length).toBe(0); + expect(policies.length).toEqual(1); + expect(policies[0].id).toBe('test-id'); + expect(packages).toEqual(expect.arrayContaining(['test_package-3.0.0'])); + expect(nonFatalErrors.length).toBe(0); + }); + + it('should not add new package policy to existing non managed policies', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mockedPackagePolicyService.getByIDs.mockResolvedValue([ + { name: 'test_package1' } as PackagePolicy, + ]); + + mockConfiguredPolicies.set('test-id', { + name: 'Test policy', + description: 'Test policy description', + unenroll_timeout: 120, + namespace: 'default', + id: 'test-id', + package_policies: [ + { + name: 'test_package1', + }, + ], + } as PreconfiguredAgentPolicy); - const { policies: policiesB, nonFatalErrors: nonFatalErrorsB } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, [ { - name: 'Test policy redo', + name: 'Test policy', namespace: 'default', id: 'test-id', + is_managed: false, package_policies: [ { - package: { name: 'some-uninstalled-package' }, - name: 'This package is not installed', + package: { name: 'test_package' }, + name: 'test_package1', + }, + { + package: { name: 'test_package' }, + name: 'test_package2', }, ], }, ] as PreconfiguredAgentPolicy[], - [], + [{ name: 'test_package', version: '3.0.0' }], mockDefaultOutput, DEFAULT_SPACE_ID ); - expect(policiesB.length).toEqual(1); - expect(policiesB[0].id).toBe('test-id'); - expect(policiesB[0].updated_at).toEqual(policiesA[0].updated_at); - expect(nonFatalErrorsB.length).toBe(0); - }); + expect(mockedPackagePolicyService.create).not.toBeCalled(); + }); - it('should update a managed policy if top level fields are changed', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + it('should add new package policy to existing managed policies', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mockedPackagePolicyService.getByIDs.mockResolvedValue([ + { name: 'test_package1' } as PackagePolicy, + ]); + + mockConfiguredPolicies.set('test-id', { + name: 'Test policy', + description: 'Test policy description', + unenroll_timeout: 120, + namespace: 'default', + id: 'test-id', + package_policies: [ + { + name: 'test_package1', + }, + ], + is_managed: true, + } as PreconfiguredAgentPolicy); - mockConfiguredPolicies.set('test-id', { - name: 'Test policy', - description: 'Test policy description', - unenroll_timeout: 120, - namespace: 'default', - id: 'test-id', - package_policies: [], - is_managed: true, - } as PreconfiguredAgentPolicy); - - const { policies, nonFatalErrors: nonFatalErrorsB } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, [ { - name: 'Renamed Test policy', - description: 'Renamed Test policy description', - unenroll_timeout: 999, + name: 'Test policy', namespace: 'default', id: 'test-id', is_managed: true, - package_policies: [], + package_policies: [ + { + package: { name: 'test_package' }, + name: 'test_package1', + }, + { + package: { name: 'test_package' }, + name: 'test_package2', + }, + ], }, ] as PreconfiguredAgentPolicy[], - [], + [{ name: 'test_package', version: '3.0.0' }], mockDefaultOutput, DEFAULT_SPACE_ID ); - expect(spyAgentPolicyServiceUpdate).toBeCalled(); - expect(spyAgentPolicyServiceUpdate).toBeCalledWith( - expect.anything(), // soClient - expect.anything(), // esClient - 'test-id', - expect.objectContaining({ - name: 'Renamed Test policy', - description: 'Renamed Test policy description', - unenroll_timeout: 999, - }) - ); - expect(policies.length).toEqual(1); - expect(policies[0].id).toBe('test-id'); - expect(nonFatalErrorsB.length).toBe(0); + + expect(mockedPackagePolicyService.create).toBeCalledTimes(1); + expect(mockedPackagePolicyService.create).toBeCalledWith( + expect.anything(), // so client + expect.anything(), // es client + expect.objectContaining({ + name: 'test_package2', + }), + expect.anything() // options + ); + }); + + it('should throw an error when trying to install duplicate packages', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + await expect( + ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [], + [ + { name: 'test_package', version: '3.0.0' }, + { name: 'test_package', version: '2.0.0' }, + ], + mockDefaultOutput, + DEFAULT_SPACE_ID + ) + ).rejects.toThrow( + 'Duplicate packages specified in configuration: test_package-3.0.0, test_package-2.0.0' + ); + }); + + it('should not create a policy and throw an error if install fails for required package', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const policies: PreconfiguredAgentPolicy[] = [ + { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + package_policies: [ + { + package: { name: 'test_package' }, + name: 'Test package', + }, + ], + }, + ]; + mockInstallPackageErrors.set('test_package', 'REGISTRY ERROR'); + + await expect( + ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + policies, + [{ name: 'test_package', version: '3.0.0' }], + mockDefaultOutput, + DEFAULT_SPACE_ID + ) + ).rejects.toThrow( + '[Test policy] could not be added. [test_package] could not be installed due to error: [Error: REGISTRY ERROR]' + ); + }); + + it('should not create a policy and throw an error if package is not installed for an unknown reason', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const policies: PreconfiguredAgentPolicy[] = [ + { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + package_policies: [ + { + id: 'test-package', + package: { name: 'test_package' }, + name: 'Test package', + }, + ], + }, + ]; + + await expect( + ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + policies, + [{ name: 'CANNOT_MATCH', version: 'x.y.z' }], + mockDefaultOutput, + DEFAULT_SPACE_ID + ) + ).rejects.toThrow( + '[Test policy] could not be added. [test_package] is not installed, add [test_package] to [xpack.fleet.packages] or remove it from [Test package].' + ); + }); + + it('should not attempt to recreate or modify an agent policy if its ID is unchanged', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const { policies: policiesA, nonFatalErrors: nonFatalErrorsA } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [ + { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + package_policies: [], + }, + ] as PreconfiguredAgentPolicy[], + [], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + + expect(policiesA.length).toEqual(1); + expect(policiesA[0].id).toBe('test-id'); + expect(nonFatalErrorsA.length).toBe(0); + + const { policies: policiesB, nonFatalErrors: nonFatalErrorsB } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [ + { + name: 'Test policy redo', + namespace: 'default', + id: 'test-id', + package_policies: [ + { + package: { name: 'some-uninstalled-package' }, + name: 'This package is not installed', + }, + ], + }, + ] as PreconfiguredAgentPolicy[], + [], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + + expect(policiesB.length).toEqual(1); + expect(policiesB[0].id).toBe('test-id'); + expect(policiesB[0].updated_at).toEqual(policiesA[0].updated_at); + expect(nonFatalErrorsB.length).toBe(0); + }); + + it('should update a managed policy if top level fields are changed', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + mockConfiguredPolicies.set('test-id', { + name: 'Test policy', + description: 'Test policy description', + unenroll_timeout: 120, + namespace: 'default', + id: 'test-id', + package_policies: [], + is_managed: true, + } as PreconfiguredAgentPolicy); + + const { policies, nonFatalErrors: nonFatalErrorsB } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [ + { + name: 'Renamed Test policy', + description: 'Renamed Test policy description', + unenroll_timeout: 999, + namespace: 'default', + id: 'test-id', + is_managed: true, + package_policies: [], + }, + ] as PreconfiguredAgentPolicy[], + [], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + expect(spyAgentPolicyServiceUpdate).toBeCalled(); + expect(spyAgentPolicyServiceUpdate).toBeCalledWith( + expect.anything(), // soClient + expect.anything(), // esClient + 'test-id', + expect.objectContaining({ + name: 'Renamed Test policy', + description: 'Renamed Test policy description', + unenroll_timeout: 999, + }) + ); + expect(policies.length).toEqual(1); + expect(policies[0].id).toBe('test-id'); + expect(nonFatalErrorsB.length).toBe(0); + }); + + it('should not update a managed policy if a top level field has not changed', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const policy: PreconfiguredAgentPolicy = { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + package_policies: [], + is_managed: true, + }; + mockConfiguredPolicies.set('test-id', policy); + + const { policies, nonFatalErrors: nonFatalErrorsB } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [policy], + [], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + expect(spyAgentPolicyServiceUpdate).not.toBeCalled(); + expect(policies.length).toEqual(1); + expect(policies[0].id).toBe('test-id'); + expect(nonFatalErrorsB.length).toBe(0); + }); }); - it('should not update a managed policy if a top level field has not changed', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const policy: PreconfiguredAgentPolicy = { - name: 'Test policy', - namespace: 'default', - id: 'test-id', - package_policies: [], - is_managed: true, - }; - mockConfiguredPolicies.set('test-id', policy); + describe('with bundled packages', () => { + beforeEach(() => { + mockInstalledPackages.clear(); + mockedGetBundledPackages.mockReset(); + }); - const { policies, nonFatalErrors: nonFatalErrorsB } = - await ensurePreconfiguredPackagesAndPolicies( + it('should install each bundled package', async () => { + mockedGetBundledPackages.mockResolvedValue([ + { + name: 'test_package', + buffer: Buffer.from('test_package'), + }, + + { + name: 'test_package_2', + buffer: Buffer.from('test_package_2'), + }, + ]); + + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const { policies, packages, nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, - [policy], [], + [ + { + name: 'test_package', + version: 'latest', + }, + { + name: 'test_package_2', + version: 'latest', + }, + ], mockDefaultOutput, DEFAULT_SPACE_ID ); - expect(spyAgentPolicyServiceUpdate).not.toBeCalled(); - expect(policies.length).toEqual(1); - expect(policies[0].id).toBe('test-id'); - expect(nonFatalErrorsB.length).toBe(0); + + expect(policies).toEqual([]); + expect(packages).toEqual(['test_package-1.0.0', 'test_package_2-1.0.0']); + expect(nonFatalErrors).toEqual([]); + }); + + describe('package updates', () => { + describe('when bundled package is a newer version', () => { + it('installs new version of package from disk', async () => { + mockedGetBundledPackages.mockResolvedValue([ + { + name: 'test_package', + buffer: Buffer.from('test_package'), + }, + ]); + + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + // Install an older version of a test package + mockInstalledPackages.set('test_package', { version: '0.9.0' }); + + const { policies, packages, nonFatalErrors } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [], + [ + { + name: 'test_package', + version: 'latest', + }, + ], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + + // Package version should be updated + expect(mockInstalledPackages.get('test_package').version).toEqual('1.0.0'); + + expect(policies).toEqual([]); + expect(packages).toEqual(['test_package-1.0.0']); + expect(nonFatalErrors).toEqual([]); + }); + }); + + describe('when bundled package is not a newer version', () => { + it('does not install package from disk', async () => { + mockedGetBundledPackages.mockResolvedValue([ + { + name: 'test_package', + buffer: Buffer.from('test_package'), + }, + ]); + + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + // Install an newer version of a test package + mockInstalledPackages.set('test_package', { + version: '2.0.0', + name: 'test_package', + installed_es: [], + installed_kibana: [], + }); + + const { policies, packages, nonFatalErrors } = + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [], + [ + { + name: 'test_package', + version: 'latest', + }, + ], + mockDefaultOutput, + DEFAULT_SPACE_ID + ); + + // Package version should be unchanged + expect(mockInstalledPackages.get('test_package').version).toEqual('2.0.0'); + + expect(packages).toEqual(['test_package-2.0.0']); + expect(policies).toEqual([]); + expect(nonFatalErrors).toEqual([]); + }); + }); + }); }); }); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 8593f753de738..d8c2032596e34 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -21,11 +21,9 @@ import type { PreconfiguredOutput, PackagePolicy, } from '../../common'; +import { PRECONFIGURATION_LATEST_KEYWORD } from '../../common'; import { SO_SEARCH_LIMIT, normalizeHostsForAgents } from '../../common'; -import { - PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, - PRECONFIGURATION_LATEST_KEYWORD, -} from '../constants'; +import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE } from '../constants'; import { escapeSearchQueryPhrase } from './saved_object'; import { pkgToPkgKey } from './epm/registry'; @@ -172,19 +170,25 @@ export async function ensurePreconfiguredPackagesAndPolicies( ); } + const packagesToInstall = packages.map((pkg) => + pkg.version === PRECONFIGURATION_LATEST_KEYWORD ? pkg.name : pkg + ); + // Preinstall packages specified in Kibana config const preconfiguredPackages = await bulkInstallPackages({ savedObjectsClient: soClient, esClient, - packagesToInstall: packages.map((pkg) => - pkg.version === PRECONFIGURATION_LATEST_KEYWORD ? pkg.name : pkg - ), + packagesToInstall, force: true, // Always force outdated packages to be installed if a later version isn't installed spaceId, + // During setup, we'll try to install preconfigured packages from the versions bundled with Kibana + // whenever possible + preferredSource: 'bundled', }); const fulfilledPackages = []; const rejectedPackages: PreconfigurationError[] = []; + for (let i = 0; i < preconfiguredPackages.length; i++) { const packageResult = preconfiguredPackages[i]; if ('error' in packageResult) { @@ -381,7 +385,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( }), } ), - packages: fulfilledPackages.map((pkg) => pkgToPkgKey(pkg)), + packages: fulfilledPackages.map((pkg) => ('version' in pkg ? pkgToPkgKey(pkg) : pkg.name)), nonFatalErrors: [...rejectedPackages, ...rejectedPolicies, ...packagePolicyUpgradeResults], }; } diff --git a/x-pack/test/fleet_api_integration/apis/epm/list.ts b/x-pack/test/fleet_api_integration/apis/epm/list.ts index 64adf094927b6..836582d0609c4 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/list.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/list.ts @@ -54,7 +54,7 @@ export default function (providerContext: FtrProviderContext) { }; const listResponse = await fetchLimitedPackageList(); - expect(listResponse.items).to.eql(['endpoint']); + expect(listResponse.items.sort()).to.eql(['endpoint'].sort()); }); it('allows user with only read permission to access', async () => { diff --git a/x-pack/test/fleet_api_integration/apis/epm/setup.ts b/x-pack/test/fleet_api_integration/apis/epm/setup.ts index 1e828cf63cfd5..edca697cc4a3d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/setup.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/setup.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; -import { GetInfoResponse, Installed } from '../../../../plugins/fleet/common'; +import { GetInfoResponse, InstalledRegistry } from '../../../../plugins/fleet/common'; import { setupFleetAndAgents } from '../agents/services'; export default function (providerContext: FtrProviderContext) { @@ -46,7 +46,7 @@ export default function (providerContext: FtrProviderContext) { .get(`/api/fleet/epm/packages/endpoint/${latestEndpointVersion}`) .expect(200)); expect(body.item).to.have.property('savedObject'); - expect((body.item as Installed).savedObject.attributes.install_version).to.eql( + expect((body.item as InstalledRegistry).savedObject.attributes.install_version).to.eql( latestEndpointVersion ); }); diff --git a/x-pack/test/fleet_api_integration/apis/preconfiguration/preconfiguration.ts b/x-pack/test/fleet_api_integration/apis/preconfiguration/preconfiguration.ts index 7c5c7d7f3f804..012ca79c4a809 100644 --- a/x-pack/test/fleet_api_integration/apis/preconfiguration/preconfiguration.ts +++ b/x-pack/test/fleet_api_integration/apis/preconfiguration/preconfiguration.ts @@ -42,11 +42,7 @@ export default function (providerContext: FtrProviderContext) { .send({}) .expect(200); - expect(body).to.eql({ - packages: [], - policies: [], - nonFatalErrors: [], - }); + expect(body.nonFatalErrors).to.eql([]); }); }); } From ac7745ed0adb3254a70e0b0f4b11df934725d93b Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Tue, 1 Feb 2022 12:58:27 -0700 Subject: [PATCH 3/3] [Security Solutions] Adds a security solutions experimental telemetry preview endpoint and e2e tests (#123989) ## Summary For e2e testing to be possible we have to enable an experimental preview endpoint: ```ts GET /internal/security_solution/telemetry ``` You can only enable this endpoint if you edit your `kibana.dev.yml` and add this line: ```yml xpack.securitySolution.enableExperimental: ['previewTelemetryUrlEnabled'] ``` This PR adds the following: * Pulls up an interface from both `TelemetryEventsSender` and `TelemetryReceiver` * Issue in the e2e FTR where we weren't removing lists from agnostic and regular space * Adds preview capabilities for `DetectionRules`, `SecurityLists`, `Endpoint`, and `Diagnostics` * 19 FTR e2e tests for `DetectionRules` and `SecurityLists`. But does _NOT_ add any for `Endpoint` or `Diagnostics` * Fixes a dangling promise that makes the preview deterministic * Fixes a small issue seen with the trusted applications and endpoint application createLists To run the test server and runner for FTR: Start the FTR server and wait for it to be initialized first: ```sh cd kibana/x-pack node scripts/functional_tests_server.js --config test/detection_engine_api_integration/security_and_spaces/config.ts ``` In a seperate terminal run: ```sh cd kibana/x-pack node scripts/functional_test_runner.js --config test/detection_engine_api_integration/security_and_spaces/config.ts --include test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../security_solution/common/constants.ts | 8 + .../common/experimental_features.ts | 10 + .../signals/open_close_signals_route.ts | 4 +- ...telemetry_detection_rules_preview_route.ts | 68 ++ .../utils/get_detecton_rules_preview.ts | 41 + .../utils/get_diagnostics_preview.ts | 41 + .../telemetry/utils/get_endpoint_preview.ts | 41 + .../utils/get_security_lists_preview.ts | 41 + .../routes/telemetry/utils/parse_ndjson.ts | 20 + .../lib/detection_engine/rule_types/types.ts | 4 +- .../signals/executors/query.ts | 4 +- .../signals/executors/threat_match.ts | 4 +- .../signals/send_telemetry_events.ts | 4 +- .../signals/threat_mapping/types.ts | 6 +- .../lib/detection_engine/signals/types.ts | 4 +- .../server/lib/telemetry/preview_sender.ts | 139 +++ .../server/lib/telemetry/receiver.ts | 95 +- .../server/lib/telemetry/sender.ts | 56 +- .../server/lib/telemetry/task.ts | 16 +- .../lib/telemetry/tasks/detection_rule.ts | 15 +- .../server/lib/telemetry/tasks/diagnostic.ts | 8 +- .../server/lib/telemetry/tasks/endpoint.ts | 17 +- .../lib/telemetry/tasks/security_lists.ts | 29 +- .../security_solution/server/plugin.ts | 11 +- .../security_solution/server/routes/index.ts | 15 +- .../common/config.ts | 5 +- .../tests/telemetry/README.md | 10 +- .../tests/telemetry/index.ts | 7 +- .../tests/telemetry/task_based/all_types.ts | 56 ++ .../telemetry/task_based/detection_rules.ts | 833 ++++++++++++++++++ .../telemetry/task_based/security_lists.ts | 451 ++++++++++ .../telemetry/usage_collector/all_types.ts | 50 ++ .../{ => usage_collector}/detection_rules.ts | 17 +- .../detection_engine_api_integration/utils.ts | 27 +- x-pack/test/lists_api_integration/utils.ts | 29 +- 35 files changed, 2083 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_detecton_rules_preview.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_diagnostics_preview.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_endpoint_preview.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_security_lists_preview.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/parse_ndjson.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/all_types.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/detection_rules.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/security_lists.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts rename x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/{ => usage_collector}/detection_rules.ts (98%) diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index ac26ef80cf21c..fcec53eb0cf30 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -267,6 +267,14 @@ export const DETECTION_ENGINE_RULE_EXECUTION_EVENTS_URL = export const detectionEngineRuleExecutionEventsUrl = (ruleId: string) => `${INTERNAL_DETECTION_ENGINE_URL}/rules/${ruleId}/execution/events` as const; +/** + * Telemetry detection endpoint for any previews requested of what data we are + * providing through UI/UX and for e2e tests. + * curl http//localhost:5601/internal/security_solution/telemetry + * to see the contents + */ +export const SECURITY_TELEMETRY_URL = `/internal/security_solution/telemetry` as const; + export const TIMELINE_RESOLVE_URL = '/api/timeline/resolve' as const; export const TIMELINE_URL = '/api/timeline' as const; export const TIMELINES_URL = '/api/timelines' as const; diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index a3a3c3a23acf0..97642faf6a572 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -24,6 +24,16 @@ export const allowedExperimentalValues = Object.freeze({ pendingActionResponsesWithAck: true, rulesBulkEditEnabled: true, policyListEnabled: false, + + /** + * This is used for enabling the end to end tests for the security_solution telemetry. + * We disable the telemetry since we don't have specific roles or permissions around it and + * we don't want people to be able to violate security by getting access to whole documents + * around telemetry they should not. + * @see telemetry_detection_rules_preview_route.ts + * @see test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md + */ + previewTelemetryUrlEnabled: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/open_close_signals_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/open_close_signals_route.ts index 81dcbd07f4dd3..3e7a82a3fdbda 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/open_close_signals_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/open_close_signals_route.ts @@ -20,7 +20,7 @@ import { DETECTION_ENGINE_SIGNALS_STATUS_URL, } from '../../../../../common/constants'; import { buildSiemResponse } from '../utils'; -import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../../telemetry/sender'; import { INSIGHTS_CHANNEL } from '../../../telemetry/constants'; import { SetupPlugins } from '../../../../plugin'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -33,7 +33,7 @@ export const setSignalsStatusRoute = ( router: SecuritySolutionPluginRouter, logger: Logger, security: SetupPlugins['security'], - sender: TelemetryEventsSender + sender: ITelemetryEventsSender ) => { router.post( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts new file mode 100644 index 0000000000000..2aad64324cfae --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; + +import { SECURITY_TELEMETRY_URL } from '../../../../../common/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; +import { ITelemetryReceiver } from '../../../telemetry/receiver'; +import { ITelemetryEventsSender } from '../../../telemetry/sender'; +import { getDetectionRulesPreview } from './utils/get_detecton_rules_preview'; +import { getSecurityListsPreview } from './utils/get_security_lists_preview'; +import { getEndpointPreview } from './utils/get_endpoint_preview'; +import { getDiagnosticsPreview } from './utils/get_diagnostics_preview'; + +export const telemetryDetectionRulesPreviewRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger, + telemetryReceiver: ITelemetryReceiver, + telemetrySender: ITelemetryEventsSender +) => { + router.get( + { + path: SECURITY_TELEMETRY_URL, + validate: false, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const detectionRules = await getDetectionRulesPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + + const securityLists = await getSecurityListsPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + + const endpoints = await getEndpointPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + + const diagnostics = await getDiagnosticsPreview({ + logger, + telemetryReceiver, + telemetrySender, + }); + + return response.ok({ + body: { + detection_rules: detectionRules, + security_lists: securityLists, + endpoints, + diagnostics, + }, + }); + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_detecton_rules_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_detecton_rules_preview.ts new file mode 100644 index 0000000000000..b7c9215e6c190 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_detecton_rules_preview.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetryDetectionRuleListsTaskConfig } from '../../../../telemetry/tasks/detection_rule'; +import { parseNdjson } from './parse_ndjson'; + +export const getDetectionRulesPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const task = createTelemetryDetectionRuleListsTaskConfig(1000); + await task.runTask( + 'detection-rules-preview', + logger, + telemetryReceiver, + taskSender, + taskExecutionPeriod + ); + const messages = taskSender.getSentMessages(); + return parseNdjson(messages); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_diagnostics_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_diagnostics_preview.ts new file mode 100644 index 0000000000000..e630ef7f250d3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_diagnostics_preview.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetryDiagnosticsTaskConfig } from '../../../../telemetry/tasks/diagnostic'; +import { parseNdjson } from './parse_ndjson'; + +export const getDiagnosticsPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const task = createTelemetryDiagnosticsTaskConfig(); + await task.runTask( + 'diagnostics-preview', + logger, + telemetryReceiver, + taskSender, + taskExecutionPeriod + ); + const messages = taskSender.getSentMessages(); + return parseNdjson(messages); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_endpoint_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_endpoint_preview.ts new file mode 100644 index 0000000000000..0ac9c95dadd40 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_endpoint_preview.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetryEndpointTaskConfig } from '../../../../telemetry/tasks/endpoint'; +import { parseNdjson } from './parse_ndjson'; + +export const getEndpointPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const task = createTelemetryEndpointTaskConfig(1000); + await task.runTask( + 'endpoint-preview', + logger, + telemetryReceiver, + taskSender, + taskExecutionPeriod + ); + const messages = taskSender.getSentMessages(); + return parseNdjson(messages); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_security_lists_preview.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_security_lists_preview.ts new file mode 100644 index 0000000000000..ac79fea7acf9e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_security_lists_preview.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'src/core/server'; + +import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender'; +import { ITelemetryReceiver } from '../../../../telemetry/receiver'; +import { ITelemetryEventsSender } from '../../../../telemetry/sender'; +import { createTelemetrySecurityListTaskConfig } from '../../../../telemetry/tasks/security_lists'; +import { parseNdjson } from './parse_ndjson'; + +export const getSecurityListsPreview = async ({ + logger, + telemetryReceiver, + telemetrySender, +}: { + logger: Logger; + telemetryReceiver: ITelemetryReceiver; + telemetrySender: ITelemetryEventsSender; +}): Promise => { + const taskExecutionPeriod = { + last: new Date(0).toISOString(), + current: new Date().toISOString(), + }; + + const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender); + const task = createTelemetrySecurityListTaskConfig(1000); + await task.runTask( + 'security-lists-preview', + logger, + telemetryReceiver, + taskSender, + taskExecutionPeriod + ); + const messages = taskSender.getSentMessages(); + return parseNdjson(messages); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/parse_ndjson.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/parse_ndjson.ts new file mode 100644 index 0000000000000..25ffa5af1e793 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/parse_ndjson.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const parseNdjson = (messages: string[]): object[][] => { + return messages.map((message) => { + const splitByNewLine = message.split('\n'); + const linesParsed = splitByNewLine.flatMap((line) => { + try { + return JSON.parse(line); + } catch (error) { + return []; + } + }); + return linesParsed; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 9aae815a4188a..728e0e578853b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -37,7 +37,7 @@ import { import { ExperimentalFeatures } from '../../../../common/experimental_features'; import { IEventLogService } from '../../../../../event_log/server'; import { AlertsFieldMap, RulesFieldMap } from '../../../../common/field_maps'; -import { TelemetryEventsSender } from '../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../telemetry/sender'; import { RuleExecutionLoggerFactory } from '../rule_execution_log'; import { commonParamsCamelToSnake } from '../schemas/rule_converters'; @@ -128,6 +128,6 @@ export interface CreateRuleOptions { experimentalFeatures: ExperimentalFeatures; logger: Logger; ml?: SetupPlugins['ml']; - eventsTelemetry?: TelemetryEventsSender | undefined; + eventsTelemetry?: ITelemetryEventsSender | undefined; version: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 47492f1db7fa9..120bf2c2ebfce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -17,7 +17,7 @@ import { getFilter } from '../get_filter'; import { getInputIndex } from '../get_input_output_index'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; import { RuleRangeTuple, BulkCreate, WrapHits } from '../types'; -import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { CompleteRule, SavedQueryRuleParams, QueryRuleParams } from '../../schemas/rule_schemas'; import { ExperimentalFeatures } from '../../../../../common/experimental_features'; @@ -48,7 +48,7 @@ export const queryExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - eventsTelemetry: TelemetryEventsSender | undefined; + eventsTelemetry: ITelemetryEventsSender | undefined; buildRuleMessage: BuildRuleMessage; bulkCreate: BulkCreate; wrapHits: WrapHits; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index b8ffa1a86c718..00edf2fffc8e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -15,7 +15,7 @@ import { import { ListClient } from '../../../../../../lists/server'; import { getInputIndex } from '../get_input_output_index'; import { RuleRangeTuple, BulkCreate, WrapHits } from '../types'; -import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { createThreatSignals } from '../threat_mapping/create_threat_signals'; import { CompleteRule, ThreatRuleParams } from '../../schemas/rule_schemas'; @@ -46,7 +46,7 @@ export const threatMatchExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - eventsTelemetry: TelemetryEventsSender | undefined; + eventsTelemetry: ITelemetryEventsSender | undefined; experimentalFeatures: ExperimentalFeatures; buildRuleMessage: BuildRuleMessage; bulkCreate: BulkCreate; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts index f9a5e4acb3160..5904f943183c3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { TelemetryEventsSender } from '../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../telemetry/sender'; import { TelemetryEvent } from '../../telemetry/types'; import { BuildRuleMessage } from './rule_messages'; import { SignalSearchResponse, SignalSource } from './types'; @@ -29,7 +29,7 @@ export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEve export function sendAlertTelemetryEvents( logger: Logger, - eventsTelemetry: TelemetryEventsSender | undefined, + eventsTelemetry: ITelemetryEventsSender | undefined, filteredEvents: SignalSearchResponse, buildRuleMessage: BuildRuleMessage ) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index b87c236afc140..45fa47288a958 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -25,7 +25,7 @@ import { AlertServices, } from '../../../../../../alerting/server'; import { ElasticsearchClient, Logger } from '../../../../../../../../src/core/server'; -import { TelemetryEventsSender } from '../../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { BulkCreate, @@ -44,7 +44,7 @@ export interface CreateThreatSignalsOptions { bulkCreate: BulkCreate; completeRule: CompleteRule; concurrentSearches: ConcurrentSearches; - eventsTelemetry: TelemetryEventsSender | undefined; + eventsTelemetry: ITelemetryEventsSender | undefined; exceptionItems: ExceptionListItemSchema[]; filters: unknown[]; inputIndex: string[]; @@ -75,7 +75,7 @@ export interface CreateThreatSignalOptions { completeRule: CompleteRule; currentResult: SearchAfterAndBulkCreateReturnType; currentThreatList: ThreatListItem[]; - eventsTelemetry: TelemetryEventsSender | undefined; + eventsTelemetry: ITelemetryEventsSender | undefined; exceptionItems: ExceptionListItemSchema[]; filters: unknown[]; inputIndex: string[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 1af6e62d30ff7..19e0c36bae052 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -30,7 +30,7 @@ import { import { ListClient } from '../../../../../lists/server'; import { Logger } from '../../../../../../../src/core/server'; import { BuildRuleMessage } from './rule_messages'; -import { TelemetryEventsSender } from '../../telemetry/sender'; +import { ITelemetryEventsSender } from '../../telemetry/sender'; import { CompleteRule, RuleParams } from '../schemas/rule_schemas'; import { GenericBulkCreateResponse } from './bulk_create_factory'; import { EcsFieldMap } from '../../../../../rule_registry/common/assets/field_maps/ecs_field_map'; @@ -314,7 +314,7 @@ export interface SearchAfterAndBulkCreateParams { listClient: ListClient; exceptionsList: ExceptionListItemSchema[]; logger: Logger; - eventsTelemetry: TelemetryEventsSender | undefined; + eventsTelemetry: ITelemetryEventsSender | undefined; id: string; inputIndexPattern: string[]; signalsIndex: string; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts new file mode 100644 index 0000000000000..3c5f3bdd52bde --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/preview_sender.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import axios, { AxiosInstance, AxiosResponse } from 'axios'; + +import { Logger } from 'src/core/server'; +import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; +import { UsageCounter } from 'src/plugins/usage_collection/server'; + +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../task_manager/server'; +import { ITelemetryEventsSender } from './sender'; +import { TelemetryEvent } from './types'; +import { ITelemetryReceiver } from './receiver'; + +/** + * Preview telemetry events sender for the telemetry route. + * @see telemetry_detection_rules_preview_route + */ +export class PreviewTelemetryEventsSender implements ITelemetryEventsSender { + /** Inner composite telemetry events sender */ + private composite: ITelemetryEventsSender; + + /** Axios local instance */ + private axiosInstance = axios.create(); + + /** Last sent message */ + private sentMessages: string[] = []; + + /** Logger for this class */ + private logger: Logger; + + constructor(logger: Logger, composite: ITelemetryEventsSender) { + this.logger = logger; + this.composite = composite; + + /** + * Intercept the last message and save it for the preview within the lastSentMessage + * Reject the request intentionally to stop from sending to the server + */ + this.axiosInstance.interceptors.request.use((config) => { + this.logger.debug( + `Intercepting telemetry', ${JSON.stringify( + config.data + )} and not sending data to the telemetry server` + ); + const data = config.data != null ? [config.data] : []; + this.sentMessages = [...this.sentMessages, ...data]; + return Promise.reject(new Error('Not sending to telemetry server')); + }); + + /** + * Create a fake response for the preview on return within the error section. + * @param error The error we don't do anything with + * @returns The response resolved to stop the chain from continuing. + */ + this.axiosInstance.interceptors.response.use( + (response) => response, + (error) => { + // create a fake response for the preview as if the server had sent it back to us + const okResponse: AxiosResponse = { + data: {}, + status: 200, + statusText: 'ok', + headers: {}, + config: {}, + }; + return Promise.resolve(okResponse); + } + ); + } + + public getSentMessages() { + return this.sentMessages; + } + + public setup( + telemetryReceiver: ITelemetryReceiver, + telemetrySetup?: TelemetryPluginSetup, + taskManager?: TaskManagerSetupContract, + telemetryUsageCounter?: UsageCounter + ) { + return this.composite.setup( + telemetryReceiver, + telemetrySetup, + taskManager, + telemetryUsageCounter + ); + } + + public getClusterID(): string | undefined { + return this.composite.getClusterID(); + } + + public start( + telemetryStart?: TelemetryPluginStart, + taskManager?: TaskManagerStartContract, + receiver?: ITelemetryReceiver + ): void { + return this.composite.start(telemetryStart, taskManager, receiver); + } + + public stop(): void { + return this.composite.stop(); + } + + public async queueTelemetryEvents(events: TelemetryEvent[]) { + const result = this.composite.queueTelemetryEvents(events); + await this.composite.sendIfDue(this.axiosInstance); + return result; + } + + public isTelemetryOptedIn(): Promise { + return this.composite.isTelemetryOptedIn(); + } + + public sendIfDue(axiosInstance?: AxiosInstance): Promise { + return this.composite.sendIfDue(axiosInstance); + } + + public processEvents(events: TelemetryEvent[]): TelemetryEvent[] { + return this.composite.processEvents(events); + } + + public async sendOnDemand(channel: string, toSend: unknown[]) { + const result = await this.composite.sendOnDemand(channel, toSend, this.axiosInstance); + return result; + } + + public getV3UrlFromV2(v2url: string, channel: string): string { + return this.composite.getV3UrlFromV2(v2url, channel); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 75bf3650158dc..f96643663b48b 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -11,7 +11,11 @@ import { ElasticsearchClient, SavedObjectsClientContract, } from 'src/core/server'; -import { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + AggregationsAggregate, + SearchRequest, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '@kbn/securitysolution-list-constants'; import { EQL_RULE_TYPE_ID, @@ -22,6 +26,8 @@ import { SIGNALS_ID, THRESHOLD_RULE_TYPE_ID, } from '@kbn/securitysolution-rules'; +import { TransportResult } from '@elastic/elasticsearch'; +import { Agent, AgentPolicy } from '../../../../fleet/common'; import { AgentClient, AgentPolicyServiceInterface } from '../../../../fleet/server'; import { ExceptionListClient } from '../../../../lists/server'; import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; @@ -37,9 +43,89 @@ import type { ESClusterInfo, GetEndpointListResponse, RuleSearchResult, + ExceptionListItem, } from './types'; -export class TelemetryReceiver { +export interface ITelemetryReceiver { + start( + core?: CoreStart, + kibanaIndex?: string, + endpointContextService?: EndpointAppContextService, + exceptionListClient?: ExceptionListClient + ): Promise; + + getClusterInfo(): ESClusterInfo | undefined; + + fetchFleetAgents(): Promise< + | { + agents: Agent[]; + total: number; + page: number; + perPage: number; + } + | undefined + >; + + fetchEndpointPolicyResponses( + executeFrom: string, + executeTo: string + ): Promise< + TransportResult>, unknown> + >; + + fetchEndpointMetrics( + executeFrom: string, + executeTo: string + ): Promise< + TransportResult>, unknown> + >; + + fetchDiagnosticAlerts( + executeFrom: string, + executeTo: string + ): Promise>>; + + fetchPolicyConfigs(id: string): Promise; + + fetchTrustedApplications(): Promise<{ + data: ExceptionListItem[] | undefined; + total: number; + page: number; + per_page: number; + }>; + + fetchEndpointList(listId: string): Promise; + + fetchDetectionRules(): Promise< + TransportResult< + SearchResponse>, + unknown + > + >; + + fetchDetectionExceptionList( + listId: string, + ruleVersion: number + ): Promise<{ + data: ExceptionListItem[]; + total: number; + page: number; + per_page: number; + }>; + + fetchClusterInfo(): Promise; + + fetchLicenseInfo(): Promise; + + copyLicenseFields(lic: ESLicense): { + issuer?: string | undefined; + issued_to?: string | undefined; + uid: string; + status: string; + type: string; + }; +} +export class TelemetryReceiver implements ITelemetryReceiver { private readonly logger: Logger; private agentClient?: AgentClient; private agentPolicyService?: AgentPolicyServiceInterface; @@ -230,6 +316,9 @@ export class TelemetryReceiver { throw Error('exception list client is unavailable: cannot retrieve trusted applications'); } + // Ensure list is created if it does not exist + await this.exceptionListClient.createTrustedAppsList(); + const results = await this.exceptionListClient.findExceptionListItem({ listId: ENDPOINT_TRUSTED_APPS_LIST_ID, page: 1, @@ -254,7 +343,7 @@ export class TelemetryReceiver { } // Ensure list is created if it does not exist - await this.exceptionListClient.createTrustedAppsList(); + await this.exceptionListClient.createEndpointList(); const results = await this.exceptionListClient.findExceptionListItem({ listId, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index fbc42acca036b..07c5ea408cbdf 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -6,18 +6,18 @@ */ import { cloneDeep } from 'lodash'; -import axios from 'axios'; import { URL } from 'url'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; import { Logger } from 'src/core/server'; import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; import { UsageCounter } from 'src/plugins/usage_collection/server'; +import axios, { AxiosInstance } from 'axios'; import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { TelemetryReceiver } from './receiver'; +import { ITelemetryReceiver } from './receiver'; import { allowlistEventFields, copyAllowlistedFields } from './filters'; import { createTelemetryTaskConfigs } from './tasks'; import { createUsageCounterLabel } from './helpers'; @@ -27,7 +27,32 @@ import { SecurityTelemetryTask, SecurityTelemetryTaskConfig } from './task'; const usageLabelPrefix: string[] = ['security_telemetry', 'sender']; -export class TelemetryEventsSender { +export interface ITelemetryEventsSender { + setup( + telemetryReceiver: ITelemetryReceiver, + telemetrySetup?: TelemetryPluginSetup, + taskManager?: TaskManagerSetupContract, + telemetryUsageCounter?: UsageCounter + ): void; + + getClusterID(): string | undefined; + + start( + telemetryStart?: TelemetryPluginStart, + taskManager?: TaskManagerStartContract, + receiver?: ITelemetryReceiver + ): void; + + stop(): void; + queueTelemetryEvents(events: TelemetryEvent[]): void; + isTelemetryOptedIn(): Promise; + sendIfDue(axiosInstance?: AxiosInstance): Promise; + processEvents(events: TelemetryEvent[]): TelemetryEvent[]; + sendOnDemand(channel: string, toSend: unknown[], axiosInstance?: AxiosInstance): Promise; + getV3UrlFromV2(v2url: string, channel: string): string; +} + +export class TelemetryEventsSender implements ITelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; private readonly checkIntervalMs = 60 * 1000; private readonly logger: Logger; @@ -36,7 +61,7 @@ export class TelemetryEventsSender { private telemetrySetup?: TelemetryPluginSetup; private intervalId?: NodeJS.Timeout; private isSending = false; - private receiver: TelemetryReceiver | undefined; + private receiver: ITelemetryReceiver | undefined; private queue: TelemetryEvent[] = []; private isOptedIn?: boolean = true; // Assume true until the first check @@ -48,7 +73,7 @@ export class TelemetryEventsSender { } public setup( - telemetryReceiver: TelemetryReceiver, + telemetryReceiver: ITelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, taskManager?: TaskManagerSetupContract, telemetryUsageCounter?: UsageCounter @@ -74,7 +99,7 @@ export class TelemetryEventsSender { public start( telemetryStart?: TelemetryPluginStart, taskManager?: TaskManagerStartContract, - receiver?: TelemetryReceiver + receiver?: ITelemetryReceiver ) { this.telemetryStart = telemetryStart; this.receiver = receiver; @@ -133,7 +158,7 @@ export class TelemetryEventsSender { return this.isOptedIn === true; } - private async sendIfDue() { + public async sendIfDue(axiosInstance: AxiosInstance = axios) { if (this.isSending) { return; } @@ -180,7 +205,8 @@ export class TelemetryEventsSender { clusterInfo?.cluster_uuid, clusterInfo?.cluster_name, clusterInfo?.version?.number, - licenseInfo?.uid + licenseInfo?.uid, + axiosInstance ); } catch (err) { this.logger.warn(`Error sending telemetry events data: ${err}`); @@ -203,7 +229,11 @@ export class TelemetryEventsSender { * @param channel the elastic telemetry channel * @param toSend telemetry events */ - public async sendOnDemand(channel: string, toSend: unknown[]) { + public async sendOnDemand( + channel: string, + toSend: unknown[], + axiosInstance: AxiosInstance = axios + ) { const clusterInfo = this.receiver?.getClusterInfo(); try { const [telemetryUrl, licenseInfo] = await Promise.all([ @@ -223,7 +253,8 @@ export class TelemetryEventsSender { clusterInfo?.cluster_uuid, clusterInfo?.cluster_name, clusterInfo?.version?.number, - licenseInfo?.uid + licenseInfo?.uid, + axiosInstance ); } catch (err) { this.logger.warn(`Error sending telemetry events data: ${err}`); @@ -258,13 +289,14 @@ export class TelemetryEventsSender { clusterUuid: string | undefined, clusterName: string | undefined, clusterVersionNumber: string | undefined, - licenseId: string | undefined + licenseId: string | undefined, + axiosInstance: AxiosInstance = axios ) { const ndjson = transformDataToNdjson(events); try { this.logger.debug(`Sending ${events.length} telemetry events to ${channel}`); - const resp = await axios.post(telemetryUrl, ndjson, { + const resp = await axiosInstance.post(telemetryUrl, ndjson, { headers: { 'Content-Type': 'application/x-ndjson', 'X-Elastic-Cluster-ID': clusterUuid, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 15ae5cdb39e50..c3d674f721491 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -12,8 +12,8 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { TelemetryReceiver } from './receiver'; -import { TelemetryEventsSender } from './sender'; +import { ITelemetryReceiver } from './receiver'; +import { ITelemetryEventsSender } from './sender'; export interface SecurityTelemetryTaskConfig { type: string; @@ -28,8 +28,8 @@ export interface SecurityTelemetryTaskConfig { export type SecurityTelemetryTaskRunner = ( taskId: string, logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => Promise; @@ -46,14 +46,14 @@ export type LastExecutionTimestampCalculator = ( export class SecurityTelemetryTask { private readonly config: SecurityTelemetryTaskConfig; private readonly logger: Logger; - private readonly sender: TelemetryEventsSender; - private readonly receiver: TelemetryReceiver; + private readonly sender: ITelemetryEventsSender; + private readonly receiver: ITelemetryReceiver; constructor( config: SecurityTelemetryTaskConfig, logger: Logger, - sender: TelemetryEventsSender, - receiver: TelemetryReceiver + sender: ITelemetryEventsSender, + receiver: ITelemetryReceiver ) { this.config = config; this.logger = logger; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts index baca71d100cf0..73931f856a336 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/detection_rule.ts @@ -8,8 +8,8 @@ import { Logger } from 'src/core/server'; import { LIST_DETECTION_RULE_EXCEPTION, TELEMETRY_CHANNEL_LISTS } from '../constants'; import { batchTelemetryRecords, templateExceptionList } from '../helpers'; -import { TelemetryEventsSender } from '../sender'; -import { TelemetryReceiver } from '../receiver'; +import { ITelemetryEventsSender } from '../sender'; +import { ITelemetryReceiver } from '../receiver'; import type { ExceptionListItem, ESClusterInfo, ESLicense, RuleSearchResult } from '../types'; import { TaskExecutionPeriod } from '../task'; @@ -23,8 +23,8 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n runTask: async ( taskId: string, logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([ @@ -88,9 +88,10 @@ export function createTelemetryDetectionRuleListsTaskConfig(maxTelemetryBatch: n LIST_DETECTION_RULE_EXCEPTION ); - batchTelemetryRecords(detectionRuleExceptionsJson, maxTelemetryBatch).forEach((batch) => { - sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); - }); + const batches = batchTelemetryRecords(detectionRuleExceptionsJson, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } return detectionRuleExceptions.length; }, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts index fae6172c268f4..a798077e62aff 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts @@ -7,9 +7,9 @@ import { Logger } from 'src/core/server'; import { getPreviousDiagTaskTimestamp } from '../helpers'; -import { TelemetryEventsSender } from '../sender'; +import { ITelemetryEventsSender } from '../sender'; import type { TelemetryEvent } from '../types'; -import { TelemetryReceiver } from '../receiver'; +import { ITelemetryReceiver } from '../receiver'; import { TaskExecutionPeriod } from '../task'; export function createTelemetryDiagnosticsTaskConfig() { @@ -23,8 +23,8 @@ export function createTelemetryDiagnosticsTaskConfig() { runTask: async ( taskId: string, logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { if (!taskExecutionPeriod.last) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 000182a3d7a9a..6be174cdf33e7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -6,7 +6,7 @@ */ import { Logger } from 'src/core/server'; -import { TelemetryEventsSender } from '../sender'; +import { ITelemetryEventsSender } from '../sender'; import type { EndpointMetricsAggregation, EndpointPolicyResponseAggregation, @@ -14,7 +14,7 @@ import type { ESClusterInfo, ESLicense, } from '../types'; -import { TelemetryReceiver } from '../receiver'; +import { ITelemetryReceiver } from '../receiver'; import { TaskExecutionPeriod } from '../task'; import { batchTelemetryRecords, @@ -47,8 +47,8 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { runTask: async ( taskId: string, logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { if (!taskExecutionPeriod.last) { @@ -255,9 +255,10 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { * * Send the documents in a batches of maxTelemetryBatch */ - batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch).forEach((telemetryBatch) => - sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, telemetryBatch) - ); + const batches = batchTelemetryRecords(telemetryPayloads, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_ENDPOINT_META, batch); + } return telemetryPayloads.length; } catch (err) { logger.warn('could not complete endpoint alert telemetry task'); @@ -268,7 +269,7 @@ export function createTelemetryEndpointTaskConfig(maxTelemetryBatch: number) { } async function fetchEndpointData( - receiver: TelemetryReceiver, + receiver: ITelemetryReceiver, executeFrom: string, executeTo: string ) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts index d27ab801197d6..f655674b6ca96 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts @@ -18,8 +18,8 @@ import { } from '../constants'; import type { ESClusterInfo, ESLicense } from '../types'; import { batchTelemetryRecords, templateExceptionList } from '../helpers'; -import { TelemetryEventsSender } from '../sender'; -import { TelemetryReceiver } from '../receiver'; +import { ITelemetryEventsSender } from '../sender'; +import { ITelemetryReceiver } from '../receiver'; import { TaskExecutionPeriod } from '../task'; export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) { @@ -32,8 +32,8 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) runTask: async ( taskId: string, logger: Logger, - receiver: TelemetryReceiver, - sender: TelemetryEventsSender, + receiver: ITelemetryReceiver, + sender: ITelemetryEventsSender, taskExecutionPeriod: TaskExecutionPeriod ) => { let count = 0; @@ -65,9 +65,10 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) logger.debug(`Trusted Apps: ${trustedAppsJson}`); count += trustedAppsJson.length; - batchTelemetryRecords(trustedAppsJson, maxTelemetryBatch).forEach((batch) => - sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) - ); + const batches = batchTelemetryRecords(trustedAppsJson, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } } // Lists Telemetry: Endpoint Exceptions @@ -83,9 +84,10 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) logger.debug(`EP Exceptions: ${epExceptionsJson}`); count += epExceptionsJson.length; - batchTelemetryRecords(epExceptionsJson, maxTelemetryBatch).forEach((batch) => - sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) - ); + const batches = batchTelemetryRecords(epExceptionsJson, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } } // Lists Telemetry: Endpoint Event Filters @@ -101,9 +103,10 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number) logger.debug(`EP Event Filters: ${epFiltersJson}`); count += epFiltersJson.length; - batchTelemetryRecords(epFiltersJson, maxTelemetryBatch).forEach((batch) => - sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch) - ); + const batches = batchTelemetryRecords(epFiltersJson, maxTelemetryBatch); + for (const batch of batches) { + await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, batch); + } } return count; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 3d7fb8a8743c1..8292f7cec1abe 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -60,8 +60,8 @@ import { EndpointAppContext } from './endpoint/types'; import { initUsageCollectors } from './usage'; import type { SecuritySolutionRequestHandlerContext } from './types'; import { securitySolutionSearchStrategyProvider } from './search_strategy/security_solution'; -import { TelemetryEventsSender } from './lib/telemetry/sender'; -import { TelemetryReceiver } from './lib/telemetry/receiver'; +import { ITelemetryEventsSender, TelemetryEventsSender } from './lib/telemetry/sender'; +import { ITelemetryReceiver, TelemetryReceiver } from './lib/telemetry/receiver'; import { licenseService } from './lib/license'; import { PolicyWatcher } from './endpoint/lib/policy/license_watch'; import { migrateArtifactsToFleet } from './endpoint/lib/artifacts/migrate_artifacts_to_fleet'; @@ -104,8 +104,8 @@ export class Plugin implements ISecuritySolutionPlugin { private readonly appClientFactory: AppClientFactory; private readonly endpointAppContextService = new EndpointAppContextService(); - private readonly telemetryReceiver: TelemetryReceiver; - private readonly telemetryEventsSender: TelemetryEventsSender; + private readonly telemetryReceiver: ITelemetryReceiver; + private readonly telemetryEventsSender: ITelemetryEventsSender; private lists: ListPluginSetup | undefined; // TODO: can we create ListPluginStart? private licensing$!: Observable; @@ -260,7 +260,8 @@ export class Plugin implements ISecuritySolutionPlugin { ruleOptions, core.getStartServices, securityRuleTypeOptions, - previewRuleDataClient + previewRuleDataClient, + this.telemetryReceiver ); registerEndpointRoutes(router, endpointContext); registerLimitedConcurrencyRoutes(core); diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 13788cbae9fde..935948d6d5938 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -58,7 +58,7 @@ import { persistPinnedEventRoute } from '../lib/timeline/routes/pinned_events'; import { SetupPlugins, StartPlugins } from '../plugin'; import { ConfigType } from '../config'; -import { TelemetryEventsSender } from '../lib/telemetry/sender'; +import { ITelemetryEventsSender } from '../lib/telemetry/sender'; import { installPrepackedTimelinesRoute } from '../lib/timeline/routes/prepackaged_timelines/install_prepackaged_timelines'; import { previewRulesRoute } from '../lib/detection_engine/routes/rules/preview_rules_route'; import { @@ -68,13 +68,15 @@ import { // eslint-disable-next-line no-restricted-imports import { legacyCreateLegacyNotificationRoute } from '../lib/detection_engine/routes/rules/legacy_create_legacy_notification'; import { createSourcererDataViewRoute, getSourcererDataViewRoute } from '../lib/sourcerer/routes'; +import { ITelemetryReceiver } from '../lib/telemetry/receiver'; +import { telemetryDetectionRulesPreviewRoute } from '../lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route'; export const initRoutes = ( router: SecuritySolutionPluginRouter, config: ConfigType, hasEncryptionKey: boolean, security: SetupPlugins['security'], - telemetrySender: TelemetryEventsSender, + telemetrySender: ITelemetryEventsSender, ml: SetupPlugins['ml'], ruleDataService: RuleDataPluginService, logger: Logger, @@ -82,7 +84,8 @@ export const initRoutes = ( ruleOptions: CreateRuleOptions, getStartServices: StartServicesAccessor, securityRuleTypeOptions: CreateSecurityRuleTypeWrapperProps, - previewRuleDataClient: IRuleDataClient + previewRuleDataClient: IRuleDataClient, + previewTelemetryReceiver: ITelemetryReceiver ) => { const isRuleRegistryEnabled = ruleDataClient != null; // Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules @@ -161,4 +164,10 @@ export const initRoutes = ( // Sourcerer API to generate default pattern createSourcererDataViewRoute(router, getStartServices); getSourcererDataViewRoute(router, getStartServices); + + const { previewTelemetryUrlEnabled } = config.experimentalFeatures; + if (previewTelemetryUrlEnabled) { + // telemetry preview endpoint for e2e integration tests only at the moment. + telemetryDetectionRulesPreviewRoute(router, logger, previewTelemetryReceiver, telemetrySender); + } }; diff --git a/x-pack/test/detection_engine_api_integration/common/config.ts b/x-pack/test/detection_engine_api_integration/common/config.ts index fe4d4f63f3e75..950ba3e8e2ce7 100644 --- a/x-pack/test/detection_engine_api_integration/common/config.ts +++ b/x-pack/test/detection_engine_api_integration/common/config.ts @@ -74,7 +74,10 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions) '--xpack.ruleRegistry.write.cache.enabled=false', '--xpack.ruleRegistry.unsafe.indexUpgrade.enabled=true', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', - `--xpack.securitySolution.enableExperimental=${JSON.stringify(['ruleRegistryEnabled'])}`, + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'ruleRegistryEnabled', + 'previewTelemetryUrlEnabled', + ])}`, ...(ssl ? [ `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md index 2760001a20626..55048f550876a 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md @@ -1,6 +1,8 @@ -These are tests for the telemetry rules within "security_solution/server/usage" -* detection_rules +These are tests for the telemetry rules within "security_solution/server/usage" and "security_solution/server/lib/telemetry" +* task_based (security_solution/server/lib/telemetry) +* usage_collector (security_solution/server/usage) -Detection rules are tests around each of the rule types to affirm they work such as query, eql, etc... This includes -legacy notifications. Once legacy notifications are moved, those tests can be removed too. +Under usage_collector, these are tests around each of the rule types to affirm they work such as query, eql, etc... This includes +legacy notifications. Once legacy notifications are moved, tests specific to it can be removed. +Under task_based, these are tests around task based types such as "detection_rules" and "security_lists" diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts index cf9db6373033a..ce1966c3175a9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts @@ -12,7 +12,12 @@ export default ({ loadTestFile }: FtrProviderContext): void => { describe('Detection rule type telemetry', function () { describe('', function () { this.tags('ciGroup11'); - loadTestFile(require.resolve('./detection_rules')); + loadTestFile(require.resolve('./usage_collector/all_types')); + loadTestFile(require.resolve('./usage_collector/detection_rules')); + + loadTestFile(require.resolve('./task_based/all_types')); + loadTestFile(require.resolve('./task_based/detection_rules')); + loadTestFile(require.resolve('./task_based/security_lists')); }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/all_types.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/all_types.ts new file mode 100644 index 0000000000000..323fe9041e1b6 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/all_types.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getSecurityTelemetryStats, +} from '../../../../utils'; +import { deleteAllExceptions } from '../../../../../lists_api_integration/utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const log = getService('log'); + const retry = getService('retry'); + + describe('All task telemetry types generically', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + await deleteAllExceptions(supertest, log); + }); + + it('should have initialized empty/zero values when no rules are running', async () => { + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats).to.eql({ + detection_rules: [], + security_lists: [], + endpoints: [], + diagnostics: [], + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/detection_rules.ts new file mode 100644 index 0000000000000..b80cb9d478ca3 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/detection_rules.ts @@ -0,0 +1,833 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRule, + getRuleForSignalTesting, + installPrePackagedRules, + getSecurityTelemetryStats, + createExceptionList, + createExceptionListItem, +} from '../../../../utils'; +import { deleteAllExceptions } from '../../../../../lists_api_integration/utils'; +import { DETECTION_ENGINE_RULES_URL } from '../../../../../../plugins/security_solution/common/constants'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const log = getService('log'); + const retry = getService('retry'); + + // Rule id of "9a1a2dae-0b5f-4c3d-8305-a268d404c306" is from the file: + // x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint_security.json + // This rule has an existing exceptions_list that we are going to use. + const IMMUTABLE_RULE_ID = '9a1a2dae-0b5f-4c3d-8305-a268d404c306'; + + describe('Detection rule task telemetry', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + await deleteAllExceptions(supertest, log); + }); + + describe('custom rules should never show any detection_rules telemetry data for each list type', () => { + it('should NOT give telemetry/stats for an exception list of type "detection"', async () => { + const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); + + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'detection', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // Create the rule with the exception added to it + await createRule(supertest, log, { + ...rule, + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }); + + // Get the stats and ensure they're empty + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).to.eql([]); + }); + }); + + it('should NOT give telemetry/stats for an exception list of type "endpoint"', async () => { + const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); + + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // Create the rule with the exception added to it + await createRule(supertest, log, { + ...rule, + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }); + + // Get the stats and ensure they're empty + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).to.eql([]); + }); + }); + + it('should NOT give telemetry/stats for an exception list of type "endpoint_trusted_apps"', async () => { + const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); + + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_trusted_apps', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // Create the rule with the exception added to it + await createRule(supertest, log, { + ...rule, + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }); + + // Get the stats and ensure they're empty + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).to.eql([]); + }); + }); + + it('should NOT give telemetry/stats for an exception list of type "endpoint_events"', async () => { + const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); + + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_events', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // Create the rule with the exception added to it + await createRule(supertest, log, { + ...rule, + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }); + + // Get the stats and ensure they're empty + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).to.eql([]); + }); + }); + + it('should NOT give telemetry/stats for an exception list of type "endpoint_host_isolation_exceptions"', async () => { + const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); + + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_host_isolation_exceptions', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // Create the rule with the exception added to it + await createRule(supertest, log, { + ...rule, + exceptions_list: [ + { + id, + list_id, + namespace_type, + type, + }, + ], + }); + + // Get the stats and ensure they're empty + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).to.eql([]); + }); + }); + }); + + describe('pre-built/immutable/elastic rules should show detection_rules telemetry data for each list type', () => { + beforeEach(async () => { + // install prepackaged rules to get immutable rules for testing + await installPrePackagedRules(supertest, log); + }); + + it('should return mutating types such as "id", "@timestamp", etc... for list of type "detection"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'detection', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule using "PATCH" endpoint + const { exceptions_list } = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + expect(stats.detection_rules).length(1); + const detectionRule = stats.detection_rules[0][0]; + expect(detectionRule['@timestamp']).to.be.a('string'); + expect(detectionRule.cluster_uuid).to.be.a('string'); + expect(detectionRule.cluster_name).to.be.a('string'); + expect(detectionRule.license_id).to.be.a('string'); + expect(detectionRule.detection_rule.created_at).to.be.a('string'); + expect(detectionRule.detection_rule.id).to.be.a('string'); + }); + }); + + it('should give telemetry/stats for an exception list of type "detection"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'detection', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + ]); + }); + }); + + it('should give telemetry/stats for an exception list of type "endpoint"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + ]); + }); + }); + + it('should give telemetry/stats for an exception list of type "endpoint_trusted_apps"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_trusted_apps', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + ]); + }); + }); + + it('should give telemetry/stats for an exception list of type "endpoint_events"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_events', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + ]); + }); + }); + + it('should give telemetry/stats for an exception list of type "endpoint_host_isolation_exceptions"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'endpoint_host_isolation_exceptions', + }); + + // add 1 item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + ]); + }); + }); + }); + + describe('pre-built/immutable/elastic rules should show detection_rules telemetry data for multiple list items and types', () => { + beforeEach(async () => { + // install prepackaged rules to get immutable rules for testing + await installPrePackagedRules(supertest, log); + }); + + it('should give telemetry/stats for 2 exception lists to the type of "detection"', async () => { + // create an exception list container of type "detection" + const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, { + description: 'description', + list_id: '123', + name: 'test list', + type: 'detection', + }); + + // add 1st item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description 1', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add 2nd item to the exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description 2', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + list_id: '123', + name: 'endpoint_list', + os_types: [], + type: 'simple', + }); + + // add the exception list to the pre-built/immutable/elastic rule + const immutableRule = await getRule(supertest, log, IMMUTABLE_RULE_ID); + await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ + rule_id: IMMUTABLE_RULE_ID, + exceptions_list: [ + ...immutableRule.exceptions_list, + { + id, + list_id, + namespace_type, + type, + }, + ], + }) + .expect(200); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const detectionRules = stats.detection_rules + .flat() + .map((obj: { detection_rule: any }) => obj.detection_rule) + .sort((obj1: { entries: { name: number } }, obj2: { entries: { name: number } }) => { + return obj1.entries.name - obj2.entries.name; + }); + + expect(detectionRules).to.eql([ + { + created_at: detectionRules[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + id: detectionRules[0].id, + name: 'endpoint description 2', + os_types: [], + rule_version: detectionRules[0].rule_version, + }, + { + created_at: detectionRules[1].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + id: detectionRules[1].id, + name: 'endpoint description 1', + os_types: [], + rule_version: detectionRules[1].rule_version, + }, + ]); + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/security_lists.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/security_lists.ts new file mode 100644 index 0000000000000..c56936f016b58 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/task_based/security_lists.ts @@ -0,0 +1,451 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + ENDPOINT_EVENT_FILTERS_LIST_ID, + ENDPOINT_LIST_ID, + ENDPOINT_TRUSTED_APPS_LIST_ID, +} from '@kbn/securitysolution-list-constants'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getSecurityTelemetryStats, + createExceptionListItem, + createExceptionList, +} from '../../../../utils'; +import { deleteAllExceptions } from '../../../../../lists_api_integration/utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const log = getService('log'); + const retry = getService('retry'); + + describe('Security lists task telemetry', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest, log); + // Calling stats endpoint once like this guarantees that the trusted applications and exceptions lists are created for us. + await getSecurityTelemetryStats(supertest, log); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + await deleteAllExceptions(supertest, log); + }); + + describe('Trusted Applications lists', () => { + it('should give telemetry/stats for 1 exception list', async () => { + // add 1 item to the existing trusted application exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: 'ae27a4b4821b13cad2a17a75d219853e', + }, + ], + list_id: ENDPOINT_TRUSTED_APPS_LIST_ID, + name: ENDPOINT_TRUSTED_APPS_LIST_ID, + os_types: ['linux'], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + + const trustedApplication = stats.security_lists + .flat() + .map((obj: { trusted_application: any }) => obj.trusted_application); + expect(trustedApplication).to.eql([ + { + created_at: trustedApplication[0].created_at, + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: 'ae27a4b4821b13cad2a17a75d219853e', + }, + ], + id: trustedApplication[0].id, + name: ENDPOINT_TRUSTED_APPS_LIST_ID, + os_types: ['linux'], + scope: { + type: 'policy', + policies: [], + }, + }, + ]); + }); + }); + + it('should give telemetry/stats for 2 exception list', async () => { + // add 1 item to the existing trusted applications exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: 'ae27a4b4821b13cad2a17a75d219853e', + }, + ], + list_id: ENDPOINT_TRUSTED_APPS_LIST_ID, + name: 'something 1', + os_types: ['linux'], + type: 'simple', + namespace_type: 'agnostic', + }); + + // add 2nd item to the existing trusted application exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: '437b930db84b8079c2dd804a71936b5f', + }, + ], + list_id: ENDPOINT_TRUSTED_APPS_LIST_ID, + name: 'something 2', + os_types: ['macos'], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + + const trustedApplication = stats.security_lists + .flat() + .map((obj: { trusted_application: any }) => obj.trusted_application) + .sort((obj1: { entries: { name: number } }, obj2: { entries: { name: number } }) => { + return obj1.entries.name - obj2.entries.name; + }); + + expect(trustedApplication).to.eql([ + { + created_at: trustedApplication[0].created_at, + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: 'ae27a4b4821b13cad2a17a75d219853e', + }, + ], + id: trustedApplication[0].id, + name: 'something 1', + os_types: ['linux'], + scope: { + type: 'policy', + policies: [], + }, + }, + { + created_at: trustedApplication[1].created_at, + entries: [ + { + field: 'process.hash.md5', + operator: 'included', + type: 'match', + value: '437b930db84b8079c2dd804a71936b5f', + }, + ], + id: trustedApplication[1].id, + name: 'something 2', + os_types: ['macos'], + scope: { + type: 'policy', + policies: [], + }, + }, + ]); + }); + }); + }); + + describe('Endpoint Exception lists', () => { + it('should give telemetry/stats for 1 exception list', async () => { + // add 1 item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: ENDPOINT_LIST_ID, + name: ENDPOINT_LIST_ID, + os_types: [], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const securityLists = stats.security_lists + .flat() + .map((obj: { endpoint_exception: any }) => obj.endpoint_exception); + expect(securityLists).to.eql([ + { + created_at: securityLists[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: securityLists[0].id, + name: ENDPOINT_LIST_ID, + os_types: [], + }, + ]); + }); + }); + + it('should give telemetry/stats for 2 exception lists', async () => { + // add 1st item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + list_id: ENDPOINT_LIST_ID, + name: ENDPOINT_LIST_ID, + os_types: [], + type: 'simple', + namespace_type: 'agnostic', + }); + + // add 2nd item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + list_id: 'endpoint_list', + name: ENDPOINT_LIST_ID, + os_types: [], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const securityLists = stats.security_lists + .flat() + .map((obj: { endpoint_exception: any }) => obj.endpoint_exception) + .sort((obj1: { entries: { name: number } }, obj2: { entries: { name: number } }) => { + return obj1.entries.name - obj2.entries.name; + }); + + expect(securityLists).to.eql([ + { + created_at: securityLists[0].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + id: securityLists[0].id, + name: ENDPOINT_LIST_ID, + os_types: [], + }, + { + created_at: securityLists[1].created_at, + entries: [ + { + field: 'keyword', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + id: securityLists[1].id, + name: ENDPOINT_LIST_ID, + os_types: [], + }, + ]); + }); + }); + }); + + describe('Endpoint Event Filters Exception lists', () => { + beforeEach(async () => { + // We have to manually create this event filter exception list. It does not look + // like there is an auto-create for it within Kibana. It must exist somewhere else. + await createExceptionList(supertest, log, { + description: 'endpoint description', + list_id: ENDPOINT_EVENT_FILTERS_LIST_ID, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + type: 'detection', + namespace_type: 'agnostic', + }); + }); + + it('should give telemetry/stats for 1 exception list', async () => { + // add 1 item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + list_id: ENDPOINT_EVENT_FILTERS_LIST_ID, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['linux'], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const endPointEventFilter = stats.security_lists + .flat() + .map((obj: { endpoint_event_filter: any }) => obj.endpoint_event_filter); + expect(endPointEventFilter).to.eql([ + { + created_at: endPointEventFilter[0].created_at, + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something', + }, + ], + id: endPointEventFilter[0].id, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['linux'], + }, + ]); + }); + }); + + it('should give telemetry/stats for 2 exception lists', async () => { + // add 1st item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + list_id: ENDPOINT_EVENT_FILTERS_LIST_ID, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['linux'], + type: 'simple', + namespace_type: 'agnostic', + }); + + // add 2nd item to the existing endpoint exception list + await createExceptionListItem(supertest, log, { + description: 'endpoint description', + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + list_id: ENDPOINT_EVENT_FILTERS_LIST_ID, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['macos'], + type: 'simple', + namespace_type: 'agnostic', + }); + + await retry.try(async () => { + const stats = await getSecurityTelemetryStats(supertest, log); + const endPointEventFilter = stats.security_lists + .flat() + .map((obj: { endpoint_event_filter: any }) => obj.endpoint_event_filter) + .sort((obj1: { entries: { name: number } }, obj2: { entries: { name: number } }) => { + return obj1.entries.name - obj2.entries.name; + }); + + expect(endPointEventFilter).to.eql([ + { + created_at: endPointEventFilter[0].created_at, + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something 1', + }, + ], + id: endPointEventFilter[0].id, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['linux'], + }, + { + created_at: endPointEventFilter[1].created_at, + entries: [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'something 2', + }, + ], + id: endPointEventFilter[1].id, + name: ENDPOINT_EVENT_FILTERS_LIST_ID, + os_types: ['macos'], + }, + ]); + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts new file mode 100644 index 0000000000000..a8d473597a461 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getStats, +} from '../../../../utils'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const log = getService('log'); + const retry = getService('retry'); + + describe('Detection rule telemetry', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/telemetry'); + }); + + beforeEach(async () => { + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + it('should have initialized empty/zero values when no rules are running', async () => { + await retry.try(async () => { + const stats = await getStats(supertest, log); + expect(stats).to.eql(getInitialDetectionMetrics()); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts similarity index 98% rename from x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/detection_rules.ts rename to x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts index f4228ed31f279..8a956d456edec 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/detection_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts @@ -6,12 +6,12 @@ */ import expect from '@kbn/expect'; -import { DetectionMetrics } from '../../../../../plugins/security_solution/server/usage/detections/types'; +import { DetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/types'; import { ThreatMatchCreateSchema, ThresholdCreateSchema, -} from '../../../../../plugins/security_solution/common/detection_engine/schemas/request'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; +} from '../../../../../../plugins/security_solution/common/detection_engine/schemas/request'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createLegacyRuleAction, createNewAction, @@ -32,8 +32,8 @@ import { waitForRuleSuccessOrStatus, waitForSignalsToBePresent, updateRule, -} from '../../../utils'; -import { getInitialDetectionMetrics } from '../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; +} from '../../../../utils'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { @@ -60,13 +60,6 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllAlerts(supertest, log); }); - it('should have initialized empty/zero values when no rules are running', async () => { - await retry.try(async () => { - const stats = await getStats(supertest, log); - expect(stats).to.eql(getInitialDetectionMetrics()); - }); - }); - describe('"kql" rule type', () => { it('should show "notifications_enabled", "notifications_disabled" "legacy_notifications_enabled", "legacy_notifications_disabled", all to be "0" for "disabled"/"in-active" rule that does not have any actions', async () => { const rule = getRuleForSignalTesting(['telemetry'], 'rule-1', false); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 04bf0eec6f5b8..6825657836184 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -52,6 +52,7 @@ import { DETECTION_ENGINE_SIGNALS_MIGRATION_URL, INTERNAL_IMMUTABLE_KEY, INTERNAL_RULE_ID_KEY, + SECURITY_TELEMETRY_URL, UPDATE_OR_CREATE_LEGACY_ACTIONS, } from '../../plugins/security_solution/common/constants'; import { RACAlert } from '../../plugins/security_solution/server/lib/detection_engine/rule_types/types'; @@ -1848,7 +1849,7 @@ export const getDetectionMetricsFromBody = ( }; /** - * Gets the stats from the stats endpoint + * Gets the stats from the stats endpoint. * @param supertest The supertest agent. * @returns The detection metrics */ @@ -1870,6 +1871,30 @@ export const getStats = async ( return getDetectionMetricsFromBody(response.body); }; +/** + * Gets the stats from the stats endpoint within specifically the security_solutions application. + * This is considered the "batch" telemetry. + * @param supertest The supertest agent. + * @returns The detection metrics + */ +export const getSecurityTelemetryStats = async ( + supertest: SuperTest.SuperTest, + log: ToolingLog +): Promise => { + const response = await supertest + .get(SECURITY_TELEMETRY_URL) + .set('kbn-xsrf', 'true') + .send({ unencrypted: true, refreshCache: true }); + if (response.status !== 200) { + log.error( + `Did not get an expected 200 "ok" when getting the batch stats for security_solutions. CI issues could happen. Suspect this line if you are seeing CI issues. body: ${JSON.stringify( + response.body + )}, status: ${JSON.stringify(response.status)}` + ); + } + return response.body; +}; + /** * This is a typical simple indicator match/threat match for testing that is easy for most basic testing * @param ruleId diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 6e0c13b21596d..c0dc1d930a57e 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -14,6 +14,7 @@ import type { ExceptionListSchema, ExceptionListItemSchema, ExceptionList, + NamespaceType, } from '@kbn/securitysolution-io-ts-list-types'; import { EXCEPTION_LIST_URL, @@ -183,32 +184,48 @@ export const binaryToString = (res: any, callback: any): void => { }; /** - * Remove all exceptions + * Remove all exceptions from both the "single" and "agnostic" spaces. * This will retry 50 times before giving up and hopefully still not interfere with other tests * @param supertest The supertest handle */ export const deleteAllExceptions = async ( supertest: SuperTest.SuperTest, log: ToolingLog +): Promise => { + await deleteAllExceptionsByType(supertest, log, 'single'); + await deleteAllExceptionsByType(supertest, log, 'agnostic'); +}; + +/** + * Remove all exceptions by a given type such as "agnostic" or "single". + * This will retry 50 times before giving up and hopefully still not interfere with other tests + * @param supertest The supertest handle + */ +export const deleteAllExceptionsByType = async ( + supertest: SuperTest.SuperTest, + log: ToolingLog, + type: NamespaceType ): Promise => { await countDownTest( async () => { const { body } = await supertest - .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999`) + .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999&namespace_type=${type}`) .set('kbn-xsrf', 'true') .send(); - const ids: string[] = body.data.map((exception: ExceptionList) => exception.id); for await (const id of ids) { - await supertest.delete(`${EXCEPTION_LIST_URL}?id=${id}`).set('kbn-xsrf', 'true').send(); + await supertest + .delete(`${EXCEPTION_LIST_URL}?id=${id}&namespace_type=${type}`) + .set('kbn-xsrf', 'true') + .send(); } const { body: finalCheck } = await supertest - .get(`${EXCEPTION_LIST_URL}/_find`) + .get(`${EXCEPTION_LIST_URL}/_find?namespace_type=${type}`) .set('kbn-xsrf', 'true') .send(); return finalCheck.data.length === 0; }, - 'deleteAllExceptions', + `deleteAllExceptions by type: "${type}"`, log, 50, 1000