diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index b015b1c96c73a..3db1d194e59aa 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -183,6 +183,7 @@ enabled: - x-pack/test/fleet_api_integration/config.agent.ts - x-pack/test/fleet_api_integration/config.agent_policy.ts - x-pack/test/fleet_api_integration/config.epm.ts + - x-pack/test/fleet_api_integration/config.event_ingested.ts - x-pack/test/fleet_api_integration/config.fleet.ts - x-pack/test/fleet_api_integration/config.package_policy.ts - x-pack/test/fleet_api_integration/config.space_awareness.ts diff --git a/config/serverless.oblt.yml b/config/serverless.oblt.yml index 059094ac87cdd..55e7fec7a3d39 100644 --- a/config/serverless.oblt.yml +++ b/config/serverless.oblt.yml @@ -129,6 +129,9 @@ xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability' ## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration xpack.fleet.agentIdVerificationEnabled: false +## Enable event.ingested separately because agentIdVerification is disabled +xpack.fleet.eventIngestedEnabled: true + ## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules. ## The value need to be a featureId observability Or stackAlerts Or siem xpack.alerting.rules.overwriteProducer: 'observability' diff --git a/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts b/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts index 7b5fbc3088984..30b1b8cd4fe92 100644 --- a/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts +++ b/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts @@ -5,92 +5,142 @@ * 2.0. */ -import type { ILicense } from '@kbn/licensing-plugin/public'; +import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock'; -import { hasEnterpriseLicense } from './licensing'; +import { License } from '@kbn/licensing-plugin/common/license'; + +import { + hasEnterpriseLicense, + hasGoldLicense, + hasPlatinumLicense, + isTrialLicense, +} from './licensing'; describe('licensing utils', () => { - const baseLicense: ILicense = { - isActive: true, - type: 'trial', - isAvailable: true, - signature: 'fake', - toJSON: jest.fn(), - getUnavailableReason: jest.fn().mockReturnValue(undefined), - hasAtLeast: jest.fn().mockReturnValue(false), - check: jest.fn().mockReturnValue({ state: 'valid' }), - getFeature: jest.fn().mockReturnValue({ isAvailable: false, isEnabled: false }), - }; - describe('hasEnterpriseLicense', () => { - let license: ILicense; - beforeEach(() => { - jest.resetAllMocks(); - license = { - ...baseLicense, - }; - }); - it('returns true for active enterprise license', () => { - license.type = 'enterprise'; + const basicLicense = licenseMock.createLicense(); + const basicExpiredLicense = licenseMock.createLicense({ license: { status: 'expired' } }); + const goldLicense = licenseMock.createLicense({ license: { type: 'gold' } }); + const goldLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'gold' }, + }); + const platinumLicense = licenseMock.createLicense({ license: { type: 'platinum' } }); + const platinumLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'platinum' }, + }); + const enterpriseLicense = licenseMock.createLicense({ license: { type: 'enterprise' } }); + const enterpriseLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'enterprise' }, + }); + const trialLicense = licenseMock.createLicense({ license: { type: 'trial' } }); + const trialLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'trial' }, + }); - expect(hasEnterpriseLicense(license)).toEqual(true); + const errorMessage = 'unavailable'; + const errorLicense = new License({ error: errorMessage, signature: '' }); + const unavailableLicense = new License({ signature: '' }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('hasEnterpriseLicense', () => { + it('returns true for active valid licenses', () => { + expect(hasEnterpriseLicense(enterpriseLicense)).toEqual(true); + expect(hasEnterpriseLicense(trialLicense)).toEqual(true); }); - it('returns true for active trial license', () => { - expect(hasEnterpriseLicense(license)).toEqual(true); + it('returns false for active invalid licenses', () => { + expect(hasEnterpriseLicense(basicLicense)).toEqual(false); + expect(hasEnterpriseLicense(goldLicense)).toEqual(false); + expect(hasEnterpriseLicense(platinumLicense)).toEqual(false); }); - it('returns false for active basic license', () => { - license.type = 'basic'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for inactive licenses', () => { + expect(hasEnterpriseLicense(trialLicenseExpired)).toEqual(false); + expect(hasEnterpriseLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasEnterpriseLicense(basicExpiredLicense)).toEqual(false); }); - it('returns false for active gold license', () => { - license.type = 'gold'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for unavailable license', () => { + expect(hasEnterpriseLicense(errorLicense)).toEqual(false); + expect(hasEnterpriseLicense(unavailableLicense)).toEqual(false); }); - it('returns false for active platinum license', () => { - license.type = 'platinum'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for null license', () => { + expect(hasEnterpriseLicense(null)).toEqual(false); }); - it('returns false for inactive enterprise license', () => { - license.type = 'enterprise'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for undefined license', () => { + expect(hasEnterpriseLicense(undefined)).toEqual(false); }); - it('returns false for inactive trial license', () => { - license.isActive = false; + }); - expect(hasEnterpriseLicense(license)).toEqual(false); + describe('hasPlatinumLicense', () => { + it('returns true for valid active licenses', () => { + expect(hasPlatinumLicense(platinumLicense)).toEqual(true); + expect(hasPlatinumLicense(enterpriseLicense)).toEqual(true); + expect(hasPlatinumLicense(trialLicense)).toEqual(true); }); - it('returns false for inactive basic license', () => { - license.type = 'basic'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for invalid active licenses', () => { + expect(hasPlatinumLicense(goldLicense)).toEqual(false); + expect(hasPlatinumLicense(basicLicense)).toEqual(false); }); - it('returns false for inactive gold license', () => { - license.type = 'gold'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for inactive licenses', () => { + expect(hasPlatinumLicense(platinumLicenseExpired)).toEqual(false); + expect(hasPlatinumLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasPlatinumLicense(trialLicenseExpired)).toEqual(false); }); - it('returns false for inactive platinum license', () => { - license.type = 'platinum'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for bad licenses', () => { + expect(hasPlatinumLicense(errorLicense)).toEqual(false); + expect(hasPlatinumLicense(unavailableLicense)).toEqual(false); }); - it('returns false for active license is missing type', () => { - delete license.type; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for null license', () => { + expect(hasPlatinumLicense(null)).toEqual(false); + }); + it('returns false for undefined license', () => { + expect(hasPlatinumLicense(undefined)).toEqual(false); + }); + }); + describe('hasGoldLicense', () => { + it('returns true for valid active licenses', () => { + expect(hasGoldLicense(goldLicense)).toEqual(true); + expect(hasGoldLicense(platinumLicense)).toEqual(true); + expect(hasGoldLicense(enterpriseLicense)).toEqual(true); + expect(hasGoldLicense(trialLicense)).toEqual(true); + }); + it('returns false for invalid active licenses', () => { + expect(hasGoldLicense(basicLicense)).toEqual(false); + }); + it('returns false for inactive licenses', () => { + expect(hasGoldLicense(goldLicenseExpired)).toEqual(false); + expect(hasGoldLicense(platinumLicenseExpired)).toEqual(false); + expect(hasGoldLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasGoldLicense(trialLicenseExpired)).toEqual(false); + }); + it('returns false for bad licenses', () => { + expect(hasGoldLicense(errorLicense)).toEqual(false); + expect(hasGoldLicense(unavailableLicense)).toEqual(false); }); it('returns false for null license', () => { - expect(hasEnterpriseLicense(null)).toEqual(false); + expect(hasGoldLicense(null)).toEqual(false); }); it('returns false for undefined license', () => { - expect(hasEnterpriseLicense(undefined)).toEqual(false); + expect(hasGoldLicense(undefined)).toEqual(false); + }); + }); + describe('isTrialLicense', () => { + it('returns true for active trial license', () => { + expect(hasGoldLicense(trialLicense)).toEqual(true); + }); + it('returns false for non-trial license', () => { + expect(isTrialLicense(platinumLicense)).toEqual(false); + }); + it('returns false for invalid license', () => { + expect(isTrialLicense(trialLicenseExpired)).toEqual(false); + expect(isTrialLicense(errorLicense)).toEqual(false); + expect(isTrialLicense(unavailableLicense)).toEqual(false); + }); + it('returns false for null license', () => { + expect(isTrialLicense(null)).toEqual(false); + }); + it('returns false for undefined license', () => { + expect(isTrialLicense(undefined)).toEqual(false); }); }); }); diff --git a/x-pack/plugins/enterprise_search/common/utils/licensing.ts b/x-pack/plugins/enterprise_search/common/utils/licensing.ts index a78e603b3650d..6b6063516cca9 100644 --- a/x-pack/plugins/enterprise_search/common/utils/licensing.ts +++ b/x-pack/plugins/enterprise_search/common/utils/licensing.ts @@ -7,10 +7,38 @@ import type { ILicense } from '@kbn/licensing-plugin/public'; -/* hasEnterpriseLicense return if the given license is an active `enterprise` or `trial` license +/* hasEnterpriseLicense return if the given license is an active `enterprise` or greater license */ export function hasEnterpriseLicense(license: ILicense | null | undefined): boolean { if (license === undefined || license === null) return false; - const qualifyingLicenses = ['enterprise', 'trial']; - return license.isActive && qualifyingLicenses.includes(license?.type ?? ''); + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('enterprise'); +} + +/* hasPlatinumLicense return if the given license is an active `platinum` or greater license + */ +export function hasPlatinumLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('platinum'); +} + +/* hasGoldLicense return if the given license is an active `gold` or greater license + */ +export function hasGoldLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('gold'); +} + +/* isTrialLicense returns if the given license is an active `trial` license + */ +export function isTrialLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license?.type === 'trial'; } diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts b/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts index 7c83b446a67c8..736cf1c5c5d48 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts @@ -10,6 +10,13 @@ import { Observable, Subscription } from 'rxjs'; import { ILicense } from '@kbn/licensing-plugin/public'; +import { + hasEnterpriseLicense, + hasGoldLicense, + hasPlatinumLicense, + isTrialLicense, +} from '../../../../common/utils/licensing'; + interface LicensingValues { license: ILicense | null; licenseSubscription: Subscription | null; @@ -50,29 +57,14 @@ export const LicensingLogic = kea [selectors.license], - (license) => { - const qualifyingLicenses = ['platinum', 'enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, + (license) => hasPlatinumLicense(license), ], hasEnterpriseLicense: [ (selectors) => [selectors.license], - (license) => { - const qualifyingLicenses = ['enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, - ], - hasGoldLicense: [ - (selectors) => [selectors.license], - (license) => { - const qualifyingLicenses = ['gold', 'platinum', 'enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, - ], - isTrial: [ - (selectors) => [selectors.license], - (license) => license?.isActive && license?.type === 'trial', + (license) => hasEnterpriseLicense(license), ], + hasGoldLicense: [(selectors) => [selectors.license], (license) => hasGoldLicense(license)], + isTrial: [(selectors) => [selectors.license], (license) => isTrialLicense(license)], }, events: ({ props, actions, values }) => ({ afterMount: () => { diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index 647a8b917d0c0..f7ce99b7f6708 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -49,6 +49,7 @@ export interface FleetConfigType { packages?: PreconfiguredPackage[]; outputs?: PreconfiguredOutput[]; agentIdVerificationEnabled?: boolean; + eventIngestedEnabled?: boolean; enableExperimental?: string[]; packageVerification?: { gpgKeyPath?: string; diff --git a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts index 935561426d7c2..30c01b1dfeb43 100644 --- a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts +++ b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts @@ -13,6 +13,7 @@ export const createConfigurationMock = (): FleetConfigType => { registryUrl: '', registryProxyUrl: '', agentIdVerificationEnabled: true, + eventIngestedEnabled: false, agents: { enabled: true, elasticsearch: { diff --git a/x-pack/plugins/fleet/server/config.ts b/x-pack/plugins/fleet/server/config.ts index ab5e06ef03716..b4f41562fd3ec 100644 --- a/x-pack/plugins/fleet/server/config.ts +++ b/x-pack/plugins/fleet/server/config.ts @@ -170,6 +170,7 @@ export const config: PluginConfigDescriptor = { proxies: PreconfiguredFleetProxiesSchema, spaceSettings: PreconfiguredSpaceSettingsSchema, agentIdVerificationEnabled: schema.boolean({ defaultValue: true }), + eventIngestedEnabled: schema.boolean({ defaultValue: false }), setup: schema.maybe( schema.object({ agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()), diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 55e6493c77891..621adc5b3b81c 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -17,6 +17,8 @@ export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; +export const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { @@ -46,6 +48,12 @@ export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { }; export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1'; +export const INGESTED_MAPPING = { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, +}; + export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, template: { @@ -58,11 +66,7 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { properties: { event: { properties: { - ingested: { - type: 'date', - format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', - ignore_malformed: false, - }, + ingested: INGESTED_MAPPING, agent_id_status: { ignore_above: 1024, type: 'keyword', @@ -74,12 +78,38 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { }, }; +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME = '.fleet_event_ingested-1'; + +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT = { + _meta: meta, + template: { + settings: { + index: { + final_pipeline: FLEET_EVENT_INGESTED_PIPELINE_ID, + }, + }, + mappings: { + properties: { + event: { + properties: { + ingested: INGESTED_MAPPING, + }, + }, + }, + }, + }, +}; + export const FLEET_COMPONENT_TEMPLATES = [ { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT }, { name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, }, + { + name: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + body: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT, + }, ]; export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`; @@ -96,6 +126,59 @@ export const STACK_COMPONENT_TEMPLATES = [ STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, ]; +export const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +// If the content is updated you probably need to update the FLEET_EVENT_INGESTED_PIPELINE_VERSION too to allow upgrade of the pipeline +export const FLEET_EVENT_INGESTED_PIPELINE_CONTENT = `--- +version: ${FLEET_EVENT_INGESTED_PIPELINE_VERSION} +_meta: + managed_by: ${meta.managed_by} + managed: ${meta.managed} +description: > + Pipeline for processing all incoming Fleet Agent documents that adds event.ingested. +processors: + - script: + description: Add time when event was ingested (and remove sub-seconds to improve storage efficiency) + tag: truncate-subseconds-event-ingested + ignore_failure: true + source: |- + if (ctx?.event == null) { + ctx.event = [:]; + } + + ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + - remove: + description: Remove any pre-existing untrusted values. + field: + - event.agent_id_status + - _security + ignore_missing: true + - remove: + description: Remove event.original unless the preserve_original_event tag is set + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - set_security_user: + field: _security + properties: + - authentication_type + - username + - realm + - api_key + - remove: + field: _security + ignore_missing: true +on_failure: + - remove: + field: _security + ignore_missing: true + ignore_failure: true + - append: + field: error.message + value: + - 'failed in Fleet agent event_ingested_pipeline: {{ _ingest.on_failure_message }}'`; + export const FLEET_FINAL_PIPELINE_VERSION = 4; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index fb7e27c8b0ef8..48de05c0b635d 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -111,6 +111,9 @@ export { FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, FLEET_INSTALL_FORMAT_VERSION, FLEET_AGENT_POLICIES_SCHEMA_VERSION, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index f032c1f7bb8c7..8d452b394dd18 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -81,6 +81,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, ...configOverrides, }; @@ -120,6 +121,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, }, config$, kibanaVersion: '8.99.0', // Fake version :) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 5a4672f67fe53..51162ac2c6335 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -20,6 +20,9 @@ import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, } from '../../../../constants'; import { getPipelineNameForDatastream } from '../../../../../common/services'; import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types'; @@ -302,6 +305,39 @@ export async function ensureFleetFinalPipelineIsInstalled( return { isCreated: false }; } +export async function ensureFleetEventIngestedPipelineIsInstalled( + esClient: ElasticsearchClient, + logger: Logger +) { + const esClientRequestOptions: TransportRequestOptions = { + ignore: [404], + }; + const res = await esClient.ingest.getPipeline( + { id: FLEET_EVENT_INGESTED_PIPELINE_ID }, + { ...esClientRequestOptions, meta: true } + ); + + const installedVersion = res?.body[FLEET_EVENT_INGESTED_PIPELINE_ID]?.version; + if ( + res.statusCode === 404 || + !installedVersion || + installedVersion < FLEET_EVENT_INGESTED_PIPELINE_VERSION + ) { + await installPipeline({ + esClient, + logger, + pipeline: { + nameForInstallation: FLEET_EVENT_INGESTED_PIPELINE_ID, + contentForInstallation: FLEET_EVENT_INGESTED_PIPELINE_CONTENT, + extension: 'yml', + }, + }); + return { isCreated: true }; + } + + return { isCreated: false }; +} + const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/'); const isDataStreamPipeline = (path: string, dataStreamDataset: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index c7d2e4eacb32a..c06d0cdbb6429 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -14,7 +14,11 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { errors } from '@elastic/elasticsearch'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; @@ -22,7 +26,6 @@ import type { RegistryDataStream } from '../../../../types'; import { processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { - FLEET_COMPONENT_TEMPLATES, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, @@ -36,10 +39,6 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATES_NAMES = FLEET_COMPONENT_TEMPLATES.map( - (componentTemplate) => componentTemplate.name -); - // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ print(val) { @@ -88,7 +87,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -108,7 +108,8 @@ describe('EPM template', () => { 'metrics@tsdb-settings', ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -138,6 +139,34 @@ describe('EPM template', () => { ]); }); + it('creates fleet event ingested component template if event ingested flag is enabled', () => { + appContextService.start( + createAppContextStartContractMock({ + agentIdVerificationEnabled: false, + eventIngestedEnabled: true, + }) + ); + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + templateIndexPattern: 'logs-*', + type: 'logs', + packageName: 'nginx', + composedOfTemplates, + templatePriority: 200, + mappings: { properties: [] }, + isIndexModeTimeSeries: false, + }); + expect(template.composed_of).toStrictEqual([ + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + ...composedOfTemplates, + STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + ]); + }); + it('adds empty composed_of correctly', () => { const composedOfTemplates: string[] = []; @@ -154,7 +183,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3709975c57a5e..b9c0846f3e4f2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -15,7 +15,10 @@ import type { import pMap from 'p-map'; import { isResponseError } from '@kbn/es-errors'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import type { Field, Fields } from '../../fields/field'; import type { @@ -27,6 +30,7 @@ import type { } from '../../../../types'; import { appContextService } from '../../..'; import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services'; +import type { FleetConfigType } from '../../../../../common/types'; import { STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, @@ -115,6 +119,9 @@ export function getTemplate({ const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries); + const isEventIngestedEnabled = (config?: FleetConfigType): boolean => + Boolean(!config?.agentIdVerificationEnabled && config?.eventIngestedEnabled); + template.composed_of = [ ...esBaseComponents, ...(template.composed_of || []), @@ -123,6 +130,9 @@ export function getTemplate({ ...(appContextService.getConfig()?.agentIdVerificationEnabled ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME] : []), + ...(isEventIngestedEnabled(appContextService.getConfig()) + ? [FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME] + : []), ]; template.ignore_missing_component_templates = template.composed_of.filter(isUserSettingsTemplate); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 0d6ec183531a4..ab882a013ebe3 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -36,7 +36,10 @@ import { downloadSourceService } from './download_source'; import { getRegistryUrl, settingsService } from '.'; import { awaitIfPending } from './setup_utils'; -import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; +import { + ensureFleetEventIngestedPipelineIsInstalled, + ensureFleetFinalPipelineIsInstalled, +} from './epm/elasticsearch/ingest_pipeline/install'; import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; @@ -336,6 +339,7 @@ export async function ensureFleetGlobalEsAssets( const globalAssetsRes = await Promise.all([ ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), + ensureFleetEventIngestedPipelineIsInstalled(esClient, logger), ]); const assetResults = globalAssetsRes.flat(); if (assetResults.some((asset) => asset.isCreated)) { diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts index 8bd8672b8fbba..79daba4c73867 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts @@ -147,6 +147,12 @@ const registerHttpRequestMockHelpers = ( const setSimulateTemplateResponse = (response?: HttpResponse, error?: ResponseError) => mockResponse('POST', `${API_BASE_PATH}/index_templates/simulate`, response, error); + const setSimulateTemplateByNameResponse = ( + name: string, + response?: HttpResponse, + error?: ResponseError + ) => mockResponse('POST', `${API_BASE_PATH}/index_templates/simulate/${name}`, response, error); + const setLoadComponentTemplatesResponse = (response?: HttpResponse, error?: ResponseError) => mockResponse('GET', `${API_BASE_PATH}/component_templates`, response, error); @@ -229,6 +235,7 @@ const registerHttpRequestMockHelpers = ( setLoadIndexStatsResponse, setUpdateIndexSettingsResponse, setSimulateTemplateResponse, + setSimulateTemplateByNameResponse, setLoadComponentTemplatesResponse, setLoadNodesPluginsResponse, setLoadTelemetryResponse, diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts index 615b8df18f905..ea536becfccac 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts @@ -617,7 +617,9 @@ describe('Index Templates tab', () => { const { find, actions, exists } = testBed; httpRequestsMockHelpers.setLoadTemplateResponse(templates[0].name, template); - httpRequestsMockHelpers.setSimulateTemplateResponse({ simulateTemplate: 'response' }); + httpRequestsMockHelpers.setSimulateTemplateByNameResponse(templates[0].name, { + simulateTemplate: 'response', + }); await actions.clickTemplateAt(0); diff --git a/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx b/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx index ed22baae580cc..fd1df7ba44697 100644 --- a/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx +++ b/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx @@ -23,22 +23,25 @@ export interface Filters { } interface Props { - template: { [key: string]: any }; + template?: { [key: string]: any }; filters?: Filters; + templateName?: string; } -export const SimulateTemplate = React.memo(({ template, filters }: Props) => { +export const SimulateTemplate = React.memo(({ template, filters, templateName }: Props) => { const [templatePreview, setTemplatePreview] = useState('{}'); const updatePreview = useCallback(async () => { - if (!template || Object.keys(template).length === 0) { + if (!templateName && (!template || Object.keys(template).length === 0)) { return; } - const indexTemplate = serializeTemplate( - stripEmptyFields(template, { types: ['string'] }) as TemplateDeserialized - ); - const { data, error } = await simulateIndexTemplate(indexTemplate); + const indexTemplate = templateName + ? undefined + : serializeTemplate( + stripEmptyFields(template, { types: ['string'] }) as TemplateDeserialized + ); + const { data, error } = await simulateIndexTemplate({ template: indexTemplate, templateName }); let filteredTemplate = data; if (data) { @@ -67,7 +70,7 @@ export const SimulateTemplate = React.memo(({ template, filters }: Props) => { } setTemplatePreview(JSON.stringify(filteredTemplate ?? error, null, 2)); - }, [template, filters]); + }, [template, filters, templateName]); useEffect(() => { updatePreview(); diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx index 38f4a8b4f787b..02df1f6e1c682 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx @@ -8,14 +8,13 @@ import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiText, EuiSpacer } from '@elastic/eui'; -import { TemplateDeserialized } from '../../../../../../../common'; import { SimulateTemplate } from '../../../../../components/index_templates'; interface Props { - templateDetails: TemplateDeserialized; + templateName: string; } -export const TabPreview = ({ templateDetails }: Props) => { +export const TabPreview = ({ templateName }: Props) => { return (
@@ -29,7 +28,7 @@ export const TabPreview = ({ templateDetails }: Props) => { - +
); }; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx index d2156d1aa958e..75446c0c05f2d 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx @@ -171,7 +171,7 @@ export const TemplateDetailsContent = ({ [SETTINGS_TAB_ID]: , [MAPPINGS_TAB_ID]: , [ALIASES_TAB_ID]: , - [PREVIEW_TAB_ID]: , + [PREVIEW_TAB_ID]: , }; const tabContent = tabToComponentMap[activeTab]; diff --git a/x-pack/plugins/index_management/public/application/services/api.ts b/x-pack/plugins/index_management/public/application/services/api.ts index 08baa49713573..9f03007014c4f 100644 --- a/x-pack/plugins/index_management/public/application/services/api.ts +++ b/x-pack/plugins/index_management/public/application/services/api.ts @@ -319,11 +319,23 @@ export async function updateTemplate(template: TemplateDeserialized) { return result; } -export function simulateIndexTemplate(template: { [key: string]: any }) { +export function simulateIndexTemplate({ + template, + templateName, +}: { + template?: { [key: string]: any }; + templateName?: string; +}) { + const path = templateName + ? `${API_BASE_PATH}/index_templates/simulate/${templateName}` + : `${API_BASE_PATH}/index_templates/simulate`; + + const body = templateName ? undefined : JSON.stringify(template); + return sendRequest({ - path: `${API_BASE_PATH}/index_templates/simulate`, + path, method: 'post', - body: JSON.stringify(template), + body, }).then((result) => { uiMetricService.trackMetric(METRIC_TYPE.COUNT, UIM_TEMPLATE_SIMULATE); return result; diff --git a/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts b/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts index 3dc6201f0831c..60e2cfbf8a53a 100644 --- a/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts @@ -15,23 +15,38 @@ const bodySchema = schema.object({}, { unknowns: 'allow' }); export function registerSimulateRoute({ router, lib: { handleEsError } }: RouteDependencies) { router.post( { - path: addBasePath('/index_templates/simulate'), - validate: { body: bodySchema }, + path: addBasePath('/index_templates/simulate/{templateName?}'), + validate: { + body: schema.nullable(bodySchema), + params: schema.object({ templateName: schema.maybe(schema.string()) }), + }, }, async (context, request, response) => { const { client } = (await context.core).elasticsearch; const template = request.body as TypeOf; + // Until ES fixes a bug on their side we need to send a fake index pattern + // that won't match any indices. + // Issue: https://github.com/elastic/elasticsearch/issues/59152 + // eslint-disable-next-line @typescript-eslint/naming-convention + const index_patterns = ['a_fake_index_pattern_that_wont_match_any_indices']; + const templateName = request.params.templateName; + + const params: estypes.IndicesSimulateTemplateRequest = templateName + ? { + name: templateName, + body: { + index_patterns, + }, + } + : { + body: { + ...template, + index_patterns, + }, + }; try { - const templatePreview = await client.asCurrentUser.indices.simulateTemplate({ - body: { - ...template, - // Until ES fixes a bug on their side we need to send a fake index pattern - // that won't match any indices. - // Issue: https://github.com/elastic/elasticsearch/issues/59152 - index_patterns: ['a_fake_index_pattern_that_wont_match_any_indices'], - }, - } as estypes.IndicesSimulateTemplateRequest); + const templatePreview = await client.asCurrentUser.indices.simulateTemplate(params); return response.ok({ body: templatePreview }); } catch (error) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 99fdc25382bf2..14767f4acd8f5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -276,7 +276,10 @@ export const getDatasetQualityTableColumns = ({ field: 'degradedDocs.percentage', sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( - + ), width: '140px', }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 752b224b6973a..897efb821ff64 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -19,6 +19,7 @@ import { overviewPanelTitleResources, } from '../../../../../common/translations'; import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; +import { DatasetQualityIndicator } from '../../../quality_indicator'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export @@ -31,6 +32,7 @@ export default function Summary() { totalServicesCount, totalHostsCount, totalDegradedDocsCount, + quality, } = useOverviewSummaryPanel(); return ( @@ -59,7 +61,16 @@ export default function Summary() { isLoading={isSummaryPanelLoading} /> - + + } + > { - const { quality } = dataStreamStat; - const translatedQuality = i18n.translate('xpack.datasetQuality.datasetQualityIdicator', { defaultMessage: '{quality}', values: { quality: capitalize(quality) }, @@ -29,7 +29,12 @@ export const DatasetQualityIndicator = ({ return ( - + ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx index 137c558dfbdd7..49ff342446071 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx @@ -7,16 +7,18 @@ import { EuiHealth, EuiText } from '@elastic/eui'; import React, { ReactNode } from 'react'; -import { QualityIndicators, InfoIndicators } from '../../../common/types'; +import type { QualityIndicators, InfoIndicators } from '../../../common/types'; export function QualityIndicator({ quality, description, isColoredDescription, + textSize = 's', }: { quality: QualityIndicators; description: string | ReactNode; isColoredDescription?: boolean; + textSize?: 'xs' | 's' | 'm'; }) { const qualityColors: Record = { poor: 'danger', @@ -25,8 +27,8 @@ export function QualityIndicator({ }; return ( - - + + {description} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts index 084210774f958..43cf6923075ee 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts @@ -7,6 +7,7 @@ import { useSelector } from '@xstate/react'; import { formatNumber } from '@elastic/eui'; +import { mapPercentageToQuality } from '../../common/utils'; import { BYTE_NUMBER_FORMAT, MAX_HOSTS_METRIC_VALUE, NUMBER_FORMAT } from '../../common/constants'; import { useDatasetQualityDetailsContext } from '../components/dataset_quality_details/context'; @@ -54,6 +55,13 @@ export const useOverviewSummaryPanel = () => { NUMBER_FORMAT ); + const degradedPercentage = + Number(totalDocsCount) > 0 + ? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100 + : 0; + + const quality = mapPercentageToQuality(degradedPercentage); + return { totalDocsCount, sizeInBytes, @@ -62,6 +70,7 @@ export const useOverviewSummaryPanel = () => { totalHostsCount, isSummaryPanelLoading, totalDegradedDocsCount, + quality, }; }; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts index f0d3eb96e4581..0b010d2d60099 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts @@ -41,7 +41,7 @@ const { describe( 'Blocklist', { - tags: ['@ess', '@serverless'], + tags: ['@ess', '@serverless', '@serverlessQA'], }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts index fb258bc04efa8..ee6ff0951d11b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts @@ -16,7 +16,7 @@ import { createAgentPolicyTask, getEndpointIntegrationVersion } from '../../task describe( 'Policy List', { - tags: ['@ess', '@serverless'], + tags: ['@ess', '@serverless', '@serverlessQA'], env: { ftrConfig: { kbnServerArgs: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts index 4b2b4dbf369d2..68f1cd4e5ea63 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts @@ -17,7 +17,7 @@ import { login } from '../../../../tasks/login'; describe( 'Agent policy settings API operations on Complete', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts index f099ac5da693b..34a09c490bcc3 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts @@ -17,7 +17,7 @@ import { describe( 'Agent Policy Settings - Complete', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts index 7fdd42ac196bd..8497806e176d2 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts @@ -17,7 +17,7 @@ import { describe( 'Agent Policy Settings - Essentials', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts index 5781fbea94880..09fd2cabd36e6 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts @@ -13,7 +13,7 @@ import { APP_POLICIES_PATH } from '../../../../../../../common/constants'; describe( 'When displaying the Policy Details in Endpoint Essentials PLI', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts index bae578e6c0490..21585d9f699ac 100644 --- a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts +++ b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts @@ -53,6 +53,12 @@ export function templatesApi(getService: FtrProviderContext['getService']) { .set('kbn-xsrf', 'xxx') .send(payload); + const simulateTemplateByName = (name: string) => + supertest + .post(`${API_BASE_PATH}/index_templates/simulate/${name}`) + .set('kbn-xsrf', 'xxx') + .send(); + return { getAllTemplates, getOneTemplate, @@ -61,5 +67,6 @@ export function templatesApi(getService: FtrProviderContext['getService']) { deleteTemplates, cleanUpTemplates, simulateTemplate, + simulateTemplateByName, }; } diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts index 66d6f34baa644..1fe7e022bfc9a 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.ts +++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts @@ -24,6 +24,7 @@ export default function ({ getService }: FtrProviderContext) { updateTemplate, cleanUpTemplates, simulateTemplate, + simulateTemplateByName, } = templatesApi(getService); describe('index templates', () => { @@ -452,6 +453,18 @@ export default function ({ getService }: FtrProviderContext) { const { body } = await simulateTemplate(payload).expect(200); expect(body.template).to.be.ok(); }); + + it('should simulate an index template by name', async () => { + const templateName = `template-${getRandomString()}`; + const payload = getTemplatePayload(templateName, [getRandomString()]); + + await createTemplate(payload).expect(200); + + await simulateTemplateByName(templateName).expect(200); + + // cleanup + await deleteTemplates([{ name: templateName }]); + }); }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/index.js b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js new file mode 100644 index 0000000000000..c6c76ca423b5f --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setupTestUsers } from '../test_users'; + +export default function loadTests({ loadTestFile, getService }) { + describe('Event Ingested', () => { + before(async () => { + await setupTestUsers(getService('security')); + }); + loadTestFile(require.resolve('./use_event_ingested')); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts new file mode 100644 index 0000000000000..7badbedbd77ba --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { testUsers } from '../test_users'; + +const TEST_INDEX = 'logs-log.log-test'; + +const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + +// TODO: Use test package or move to input package version github.com/elastic/kibana/issues/154243 +const LOG_INTEGRATION_VERSION = '1.1.2'; + +const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const es = getService('es'); + const esArchiver = getService('esArchiver'); + const fleetAndAgents = getService('fleetAndAgents'); + + describe('fleet_event_ingested_pipeline', () => { + skipIfNoDockerRegistry(providerContext); + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + await fleetAndAgents.setup(); + // Use the custom log package to test the fleet final pipeline + await supertestWithoutAuth + .post(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + }); + + after(async () => { + await supertestWithoutAuth + .delete(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + const res = await es.search({ + index: TEST_INDEX, + }); + + for (const hit of res.hits.hits) { + await es.delete({ + id: hit._id!, + index: hit._index, + }); + } + }); + + it('should correctly update the event ingested pipeline', async () => { + await es.ingest.putPipeline({ + id: FLEET_EVENT_INGESTED_PIPELINE_ID, + body: { + description: 'Test PIPELINE WITHOUT version', + processors: [ + { + set: { + field: 'my-keyword-field', + value: 'foo', + }, + }, + ], + }, + }); + await supertestWithoutAuth + .post(`/api/fleet/setup`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx'); + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + expect(pipelineRes[FLEET_EVENT_INGESTED_PIPELINE_ID].version).to.be(1); + }); + + it('should correctly setup the event ingested pipeline and apply to fleet managed index template', async () => { + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); + expect(res.index_templates.length).to.be(FLEET_EVENT_INGESTED_PIPELINE_VERSION); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('ecs@mappings'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('.fleet_globals-1'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain( + '.fleet_event_ingested-1' + ); + }); + + it('all docs should contain event.ingested without sub-seconds', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + '@timestamp': '2020-01-01T09:09:00', + message: 'hello', + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const ingestTimestamp = doc._source.event.ingested; + + // 2021-06-30T12:06:28Z + expect(ingestTimestamp).to.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + }); + + it('should remove agent_id_status', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + '@timestamp': '2020-01-01T09:09:00', + agent: { + id: 'agent1', + }, + event: { + agent_id_status: 'dummy', + }, + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const event = doc._source.event; + + expect(event.agent_id_status).to.be(undefined); + expect(event).to.have.property('ingested'); + }); + + it('removes event.original if preserve_original_event is not set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: [], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.be(undefined); + }); + + it('preserves event.original if preserve_original_event is set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: ['preserve_original_event'], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.eql(JSON.stringify({ foo: 'bar' })); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/config.event_ingested.ts b/x-pack/test/fleet_api_integration/config.event_ingested.ts new file mode 100644 index 0000000000000..cbdf4d501e1d2 --- /dev/null +++ b/x-pack/test/fleet_api_integration/config.event_ingested.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts')); + const serverArgs: string[] = [ + ...baseFleetApiConfig.get('kbnTestServer.serverArgs'), + // serverless oblt needs only event.ingested, without agent id verification + `--xpack.fleet.agentIdVerificationEnabled=false`, + `--xpack.fleet.eventIngestedEnabled=true`, + ]; + + return { + ...baseFleetApiConfig.getAll(), + kbnTestServer: { + ...baseFleetApiConfig.get('kbnTestServer'), + serverArgs, + }, + testFiles: [require.resolve('./apis/event_ingested')], + junit: { + reportName: 'X-Pack Event Ingested API Integration Tests', + }, + }; +} diff --git a/x-pack/test/functional/apps/index_management/index_template_wizard.ts b/x-pack/test/functional/apps/index_management/index_template_wizard.ts index cf6f1bf6a44a1..581a0b2761644 100644 --- a/x-pack/test/functional/apps/index_management/index_template_wizard.ts +++ b/x-pack/test/functional/apps/index_management/index_template_wizard.ts @@ -107,6 +107,76 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); }); + // https://github.com/elastic/kibana/pull/195174 + it('can preview index template that matches a_fake_index_pattern_that_wont_match_any_indices', async () => { + // Click Create Template button + await testSubjects.click('createTemplateButton'); + const pageTitleText = await testSubjects.getVisibleText('pageTitle'); + expect(pageTitleText).to.be('Create template'); + + const stepTitle1 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle1).to.be('Logistics'); + + // Fill out required fields + await testSubjects.setValue('nameField', 'a-star'); + await testSubjects.setValue('indexPatternsField', 'a*'); + await testSubjects.setValue('priorityField', '1000'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify empty prompt + const emptyPrompt = await testSubjects.exists('emptyPrompt'); + expect(emptyPrompt).to.be(true); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle2 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle2).to.be('Index settings (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle3 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle3).to.be('Mappings (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle4 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle4).to.be('Aliases (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle).to.be("Review details for 'a-star'"); + + // Verify that summary exists + const summaryTabContent = await testSubjects.exists('summaryTabContent'); + expect(summaryTabContent).to.be(true); + + // Verify that index mode is set to "Standard" + expect(await testSubjects.exists('indexModeTitle')).to.be(true); + expect(await testSubjects.getVisibleText('indexModeValue')).to.be('Standard'); + + // Click Create template + await pageObjects.indexManagement.clickNextButton(); + + // Click preview tab, we know its the last one + const tabs = await testSubjects.findAll('tab'); + await tabs[tabs.length - 1].click(); + const templatePreview = await testSubjects.getVisibleText('simulateTemplatePreview'); + expect(templatePreview).to.not.contain('error'); + + await testSubjects.click('closeDetailsButton'); + }); + describe('Mappings step', () => { beforeEach(async () => { await pageObjects.common.navigateToApp('indexManagement'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts index 680a439e326f5..141c9c0d864ef 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts @@ -15,7 +15,7 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const utils = getService('securitySolutionUtils'); - describe('@ess @serverless Resolver tests for the entity route', function () { + describe('@ess @serverless @serverlessQA Resolver tests for the entity route', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts index 05e7c72c98d4b..6ed414f6a0abd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts @@ -38,7 +38,7 @@ export default function ({ getService }: FtrProviderContext) { } }; - describe('@ess @serverless Resolver handling of entity ids', function () { + describe('@ess @serverless @serverlessQA Resolver handling of entity ids', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts index 91345c22ac82b..7e5848b5016a3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts @@ -52,7 +52,7 @@ export default function ({ getService }: FtrProviderContext) { ancestryArraySize: 2, }; - describe('@ess @serverless event route', function () { + describe('@ess @serverless @serverlessQA event route', function () { let entityIDFilterArray: JsonObject[] | undefined; let entityIDFilter: string | undefined; let adminSupertest: TestAgent; diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts index 8ecfab9335a79..aa68fbc692ae5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts @@ -56,7 +56,7 @@ export default function ({ getService }: FtrProviderContext) { alwaysGenMaxChildrenPerNode: true, ancestryArraySize: 2, }; - describe('@ess @serverless Resolver tree', function () { + describe('@ess @serverless @serverlessQA Resolver tree', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts index c006ecb88ad84..9fbc8e3f15507 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts @@ -10,7 +10,7 @@ import TestAgent from 'supertest/lib/agent'; import { FtrProviderContext } from '../../../../ftr_provider_context_edr_workflows'; export default function ({ getService }: FtrProviderContext) { - describe('@ess @serverless Response Actions support for sentinelOne agentType', function () { + describe('@ess @serverless @serverlessQA Response Actions support for sentinelOne agentType', function () { const utils = getService('securitySolutionUtils'); let adminSupertest: TestAgent; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 9fb1f74cbae0e..42a5a095ed6c4 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -194,7 +194,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Set Limit of 42 await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { - 'mapping.total_fields.limit': 43, + 'mapping.total_fields.limit': 42, }); await synthtrace.index([ @@ -262,13 +262,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Set Limit of 44 + // Set Limit of 43 await PageObjects.datasetQuality.setDataStreamSettings( PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ dataset: nginxAccessDatasetName, }) + '-000002', { - 'mapping.total_fields.limit': 44, + 'mapping.total_fields.limit': 43, } ); @@ -745,7 +745,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'disabled' ); - expect(currentFieldLimit).to.be(44); + expect(currentFieldLimit).to.be(43); expect(currentFieldLimitDisabledStatus).to.be('true'); // Should display new field limit