diff --git a/x-pack/plugins/fleet/common/services/agent_status.ts b/x-pack/plugins/fleet/common/services/agent_status.ts index e4b227b79536c..641da154e712d 100644 --- a/x-pack/plugins/fleet/common/services/agent_status.ts +++ b/x-pack/plugins/fleet/common/services/agent_status.ts @@ -8,7 +8,7 @@ import { AGENT_POLLING_THRESHOLD_MS } from '../constants'; import type { Agent, AgentStatus } from '../types'; -export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus { +export function getAgentStatus(agent: Agent): AgentStatus { const { last_checkin: lastCheckIn } = agent; if (!agent.active) { @@ -41,36 +41,42 @@ export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentSta return 'online'; } -export function buildKueryForEnrollingAgents() { - return 'not (last_checkin:*)'; +export function buildKueryForEnrollingAgents(path: string = '') { + return `not (${path}last_checkin:*)`; } -export function buildKueryForUnenrollingAgents() { - return 'unenrollment_started_at:*'; +export function buildKueryForUnenrollingAgents(path: string = '') { + return `${path}unenrollment_started_at:*`; } -export function buildKueryForOnlineAgents() { - return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()}) AND not (${buildKueryForUpdatingAgents()})`; +export function buildKueryForOnlineAgents(path: string = '') { + return `not (${buildKueryForOfflineAgents(path)}) AND not (${buildKueryForErrorAgents( + path + )}) AND not (${buildKueryForUpdatingAgents(path)})`; } -export function buildKueryForErrorAgents() { - return `(last_checkin_status:error or last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents()})`; +export function buildKueryForErrorAgents(path: string = '') { + return `(${path}last_checkin_status:error or ${path}last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents( + path + )})`; } -export function buildKueryForOfflineAgents() { - return `last_checkin < now-${ +export function buildKueryForOfflineAgents(path: string = '') { + return `${path}last_checkin < now-${ (4 * AGENT_POLLING_THRESHOLD_MS) / 1000 - }s AND not (${buildKueryForErrorAgents()}) AND not ( ${buildKueryForUpdatingAgents()} )`; + }s AND not (${buildKueryForErrorAgents(path)}) AND not ( ${buildKueryForUpdatingAgents(path)} )`; } -export function buildKueryForUpgradingAgents() { - return '(upgrade_started_at:*) and not (upgraded_at:*)'; +export function buildKueryForUpgradingAgents(path: string = '') { + return `(${path}upgrade_started_at:*) and not (${path}upgraded_at:*)`; } -export function buildKueryForUpdatingAgents() { - return `(${buildKueryForUpgradingAgents()}) or (${buildKueryForEnrollingAgents()}) or (${buildKueryForUnenrollingAgents()})`; +export function buildKueryForUpdatingAgents(path: string = '') { + return `(${buildKueryForUpgradingAgents(path)}) or (${buildKueryForEnrollingAgents( + path + )}) or (${buildKueryForUnenrollingAgents(path)})`; } -export function buildKueryForInactiveAgents() { - return `active:false`; +export function buildKueryForInactiveAgents(path: string = '') { + return `${path}active:false`; } diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts index 4aebae0b15c95..6566c2780d3d8 100644 --- a/x-pack/plugins/security_solution/common/endpoint/constants.ts +++ b/x-pack/plugins/security_solution/common/endpoint/constants.ts @@ -20,6 +20,9 @@ export const metadataTransformPrefix = 'endpoint.metadata_current-default'; /** The metadata Transform Name prefix with NO namespace and NO (package) version) */ export const metadataTransformPattern = 'endpoint.metadata_current-*'; +export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default'; +export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default'; + export const policyIndexPattern = 'metrics-endpoint.policy-*'; export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*'; export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency'; diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts index d92706d4e861a..6afd2de5b56b6 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts @@ -181,6 +181,14 @@ export async function indexEndpointHostDocs({ await indexFleetActionsForHost(client, hostMetadata); } + hostMetadata = { + ...hostMetadata, + // since the united transform uses latest metadata transform as a source + // there is an extra delay and fleet-agents gets populated much sooner. + // we manually add a delay to the time sync field so that the united transform + // will pick up the latest metadata doc. + '@timestamp': hostMetadata['@timestamp'] + 60000, + }; await client .index({ index: metadataIndex, diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index 9b899d9c1b887..297b1d2442c78 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -6,7 +6,7 @@ */ import { ApplicationStart } from 'kibana/public'; -import { PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common'; +import { Agent, PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common'; import { ManifestSchema } from '../schema/manifest'; export * from './actions'; @@ -546,6 +546,16 @@ export type HostMetadata = Immutable<{ data_stream: DataStream; }>; +export type UnitedAgentMetadata = Immutable<{ + agent: { + id: string; + }; + united: { + endpoint: HostMetadata; + agent: Agent; + }; +}>; + export interface LegacyEndpointEvent { '@timestamp': number; endgame: { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts index 83d3e62cf98f2..15d0684a2864b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts @@ -53,7 +53,7 @@ import { jest.mock('../../policy/store/services/ingest', () => ({ sendGetAgentConfigList: () => Promise.resolve({ items: [] }), sendGetAgentPolicyList: () => Promise.resolve({ items: [] }), - sendGetEndpointSecurityPackage: () => Promise.resolve({}), + sendGetEndpointSecurityPackage: () => Promise.resolve({ version: '1.1.1' }), sendGetFleetAgentsWithEndpoint: () => Promise.resolve({ total: 0 }), })); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts index f2f871f065a7b..84cf3513d5d3a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts @@ -6,6 +6,8 @@ */ import { Dispatch } from 'redux'; +import semverGte from 'semver/functions/gte'; + import { CoreStart, HttpStart } from 'kibana/public'; import { ActivityLog, @@ -40,6 +42,7 @@ import { getMetadataTransformStats, isMetadataTransformStatsLoading, getActivityLogIsUninitializedOrHasSubsequentAPIError, + endpointPackageVersion, } from './selectors'; import { AgentIdsPendingActions, @@ -61,6 +64,7 @@ import { HOST_METADATA_LIST_ROUTE, BASE_POLICY_RESPONSE_ROUTE, metadataCurrentIndexPattern, + METADATA_UNITED_INDEX, } from '../../../../../common/endpoint/constants'; import { IIndexPattern, Query } from '../../../../../../../../src/plugins/data/public'; import { @@ -85,13 +89,26 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory { - async function fetchIndexPatterns(): Promise { + // this needs to be called after endpointPackageVersion is loaded (getEndpointPackageInfo) + // or else wrong pattern might be loaded + async function fetchIndexPatterns( + state: ImmutableObject + ): Promise { + const packageVersion = endpointPackageVersion(state) ?? ''; + const parsedPackageVersion = packageVersion.includes('-') + ? packageVersion.substring(0, packageVersion.indexOf('-')) + : packageVersion; + const minUnitedIndexVersion = '1.2.0'; + const indexPatternToFetch = semverGte(parsedPackageVersion, minUnitedIndexVersion) + ? METADATA_UNITED_INDEX + : metadataCurrentIndexPattern; + const { indexPatterns } = depsStart.data; const fields = await indexPatterns.getFieldsForWildcard({ - pattern: metadataCurrentIndexPattern, + pattern: indexPatternToFetch, }); const indexPattern: IIndexPattern = { - title: metadataCurrentIndexPattern, + title: indexPatternToFetch, fields, }; return [indexPattern]; @@ -379,7 +396,7 @@ async function endpointDetailsListMiddleware({ }: { store: ImmutableMiddlewareAPI; coreStart: CoreStart; - fetchIndexPatterns: () => Promise; + fetchIndexPatterns: (state: ImmutableObject) => Promise; }) { const { getState, dispatch } = store; @@ -441,7 +458,7 @@ async function endpointDetailsListMiddleware({ // get index pattern and fields for search bar if (patterns(getState()).length === 0) { try { - const indexPatterns = await fetchIndexPatterns(); + const indexPatterns = await fetchIndexPatterns(getState()); if (indexPatterns !== undefined) { dispatch({ type: 'serverReturnedMetadataPatterns', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts index 0e25bb3561788..ed5dbbd09d79a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts @@ -42,7 +42,7 @@ import { HostMetadata, } from '../../../../common/endpoint/types'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; -import { createV2SearchResponse } from '../metadata/support/test_support'; +import { legacyMetadataSearchResponse } from '../metadata/support/test_support'; import { ElasticsearchAssetType } from '../../../../../fleet/common'; import { CasesClientMock } from '../../../../../cases/server/client/mocks'; @@ -188,7 +188,7 @@ describe('Host Isolation', () => { ctx.core.elasticsearch.client.asCurrentUser.search = jest .fn() .mockImplementation(() => - Promise.resolve({ body: createV2SearchResponse(searchResponse) }) + Promise.resolve({ body: legacyMetadataSearchResponse(searchResponse) }) ); const withLicense = license ? license : Platinum; licenseEmitter.next(withLicense); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts index af8c4d347c773..5c06dbd3db14c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts @@ -6,11 +6,14 @@ */ import Boom from '@hapi/boom'; +import { ApiResponse } from '@elastic/elasticsearch'; +import { SearchResponse, SearchTotalHits } from '@elastic/elasticsearch/api/types'; import { TypeOf } from '@kbn/config-schema'; import { IKibanaResponse, IScopedClusterClient, + KibanaRequest, KibanaResponseFactory, Logger, RequestHandler, @@ -19,18 +22,24 @@ import { import { HostInfo, HostMetadata, + UnitedAgentMetadata, HostResultList, HostStatus, } from '../../../../common/endpoint/types'; import type { SecuritySolutionRequestHandlerContext } from '../../../types'; -import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders'; -import { Agent, PackagePolicy } from '../../../../../fleet/common/types/models'; +import { + getESQueryHostMetadataByID, + kibanaRequestToMetadataListESQuery, + buildUnitedIndexQuery, +} from './query_builders'; +import { Agent, AgentPolicy, PackagePolicy } from '../../../../../fleet/common/types/models'; import { AgentNotFoundError } from '../../../../../fleet/server'; import { EndpointAppContext, HostListQueryResult } from '../../types'; import { GetMetadataListRequestSchema, GetMetadataRequestSchema } from './index'; import { findAllUnenrolledAgentIds } from './support/unenroll'; -import { findAgentIDsByStatus } from './support/agent_status'; +import { getAllEndpointPackagePolicies } from './support/endpoint_package_policies'; +import { findAgentIdsByStatus } from './support/agent_status'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { fleetAgentStatusToEndpointHostStatus } from '../../utils'; import { @@ -104,41 +113,32 @@ export const getMetadataListRequestHandler = function ( throw new Error('agentService not available'); } - const metadataRequestContext: MetadataRequestContext = { - esClient: context.core.elasticsearch.client, - endpointAppContextService: endpointAppContext.service, - logger, - requestHandlerContext: context, - savedObjectsClient: context.core.savedObjects.client, - }; - - const unenrolledAgentIds = await findAllUnenrolledAgentIds( - agentService, + const endpointPolicies = await getAllEndpointPackagePolicies( endpointAppContext.service.getPackagePolicyService()!, - context.core.savedObjects.client, - context.core.elasticsearch.client.asCurrentUser + context.core.savedObjects.client ); - const statusIDs = request?.body?.filters?.host_status?.length - ? await findAgentIDsByStatus( - agentService, - context.core.savedObjects.client, - context.core.elasticsearch.client.asCurrentUser, - request.body?.filters?.host_status - ) - : undefined; - - const queryParams = await kibanaRequestToMetadataListESQuery(request, endpointAppContext, { - unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS), - statusAgentIDs: statusIDs, - }); - - const result = await context.core.elasticsearch.client.asCurrentUser.search( - queryParams + const { unitedIndexExists, unitedQueryResponse } = await queryUnitedIndex( + context, + request, + endpointAppContext, + logger, + endpointPolicies ); - const hostListQueryResult = queryResponseToHostListResult(result.body); + if (unitedIndexExists) { + return response.ok({ + body: unitedQueryResponse, + }); + } + return response.ok({ - body: await mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext), + body: await legacyListMetadataQuery( + context, + request, + endpointAppContext, + logger, + endpointPolicies + ), }); }; }; @@ -395,3 +395,157 @@ export async function enrichHostMetadata( policy_info: policyInfo, }; } + +async function legacyListMetadataQuery( + context: SecuritySolutionRequestHandlerContext, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + request: KibanaRequest, + endpointAppContext: EndpointAppContext, + logger: Logger, + endpointPolicies: PackagePolicy[] +): Promise { + const agentService = endpointAppContext.service.getAgentService()!; + + const metadataRequestContext: MetadataRequestContext = { + esClient: context.core.elasticsearch.client, + endpointAppContextService: endpointAppContext.service, + logger, + requestHandlerContext: context, + savedObjectsClient: context.core.savedObjects.client, + }; + + const endpointPolicyIds = endpointPolicies.map((policy) => policy.policy_id); + const unenrolledAgentIds = await findAllUnenrolledAgentIds( + agentService, + context.core.elasticsearch.client.asCurrentUser, + endpointPolicyIds + ); + + const statusesToFilter = request?.body?.filters?.host_status ?? []; + const statusIds = await findAgentIdsByStatus( + agentService, + context.core.elasticsearch.client.asCurrentUser, + statusesToFilter + ); + + const queryParams = await kibanaRequestToMetadataListESQuery(request, endpointAppContext, { + unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS), + statusAgentIds: statusIds, + }); + + const result = await context.core.elasticsearch.client.asCurrentUser.search( + queryParams + ); + const hostListQueryResult = queryResponseToHostListResult(result.body); + return mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext); +} + +async function queryUnitedIndex( + context: SecuritySolutionRequestHandlerContext, + request: KibanaRequest, + endpointAppContext: EndpointAppContext, + logger: Logger, + endpointPolicies: PackagePolicy[] +): Promise<{ + unitedIndexExists: boolean; + unitedQueryResponse: HostResultList; +}> { + const endpointPolicyIds = endpointPolicies.map((policy) => policy.policy_id); + const unitedIndexQuery = await buildUnitedIndexQuery( + request, + endpointAppContext, + IGNORED_ELASTIC_AGENT_IDS, + endpointPolicyIds + ); + + let unitedMetadataQueryResponse: ApiResponse>; + try { + unitedMetadataQueryResponse = + await context.core.elasticsearch.client.asCurrentUser.search( + unitedIndexQuery + ); + } catch (error) { + const errorType = error?.meta?.body?.error?.type ?? ''; + + // no united index means that the endpoint package hasn't been upgraded yet + // this is expected so we fall back to the legacy query + // errors other than index_not_found_exception are unexpected + if (errorType !== 'index_not_found_exception') { + logger.error(error); + throw error; + } + return { + unitedIndexExists: false, + unitedQueryResponse: {} as HostResultList, + }; + } + + const { hits: docs, total: docsCount } = unitedMetadataQueryResponse?.body?.hits || {}; + const agentPolicyIds: string[] = docs.map((doc) => doc._source?.united?.agent?.policy_id ?? ''); + + const agentPolicies = + (await endpointAppContext.service + .getAgentPolicyService() + ?.getByIds(context.core.savedObjects.client, agentPolicyIds)) ?? []; + + const agentPoliciesMap: Record = agentPolicies.reduce( + (acc, agentPolicy) => ({ + ...acc, + [agentPolicy.id]: { + ...agentPolicy, + }, + }), + {} + ); + + const endpointPoliciesMap: Record = endpointPolicies.reduce( + (acc, packagePolicy) => ({ + ...acc, + [packagePolicy.policy_id]: packagePolicy, + }), + {} + ); + + const hosts = docs + .filter((doc) => { + const { endpoint: metadata, agent } = doc?._source?.united ?? {}; + return metadata && agent; + }) + .map((doc) => { + const { endpoint: metadata, agent } = doc!._source!.united!; + const agentPolicy = agentPoliciesMap[agent.policy_id!]; + const endpointPolicy = endpointPoliciesMap[agent.policy_id!]; + return { + metadata, + host_status: fleetAgentStatusToEndpointHostStatus(agent.last_checkin_status!), + policy_info: { + agent: { + applied: { + id: agent.policy_id || '', + revision: agent.policy_revision || 0, + }, + configured: { + id: agentPolicy?.id || '', + revision: agentPolicy?.revision || 0, + }, + }, + endpoint: { + id: endpointPolicy?.id || '', + revision: endpointPolicy?.revision || 0, + }, + }, + } as HostInfo; + }); + + const unitedQueryResponse: HostResultList = { + request_page_size: unitedIndexQuery.size, + request_page_index: unitedIndexQuery.from, + total: (docsCount as SearchTotalHits).value, + hosts, + }; + + return { + unitedIndexExists: true, + unitedQueryResponse, + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index bb1917ba12323..3fa90ad6d27a5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -33,11 +33,13 @@ import { import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; import { Agent, ElasticsearchAssetType } from '../../../../../fleet/common/types/models'; -import { createV2SearchResponse } from './support/test_support'; +import { legacyMetadataSearchResponse, unitedMetadataSearchResponse } from './support/test_support'; import { PackageService } from '../../../../../fleet/server/services'; import { HOST_METADATA_LIST_ROUTE, + metadataCurrentIndexPattern, metadataTransformPrefix, + METADATA_UNITED_INDEX, } from '../../../../common/endpoint/constants'; import type { SecuritySolutionPluginRouter } from '../../../types'; import { AgentNotFoundError, PackagePolicyServiceInterface } from '../../../../../fleet/server'; @@ -49,6 +51,15 @@ import { import { EndpointHostNotFoundError } from '../../services/metadata'; import { FleetAgentGenerator } from '../../../../common/endpoint/data_generators/fleet_agent_generator'; +class IndexNotFoundException extends Error { + meta: { body: { error: { type: string } } }; + + constructor() { + super(); + this.meta = { body: { error: { type: 'index_not_found_exception' } } }; + } +} + describe('test endpoint route', () => { let routerMock: jest.Mocked; let mockResponse: jest.Mocked; @@ -95,7 +106,7 @@ describe('test endpoint route', () => { }); }); - describe('with new transform package', () => { + describe('with .metrics-endpoint.metadata_united_default index', () => { beforeEach(() => { endpointAppContextService = new EndpointAppContextService(); mockPackageService = createMockPackageService(); @@ -135,12 +146,16 @@ describe('test endpoint route', () => { afterEach(() => endpointAppContextService.stop()); - it('test find the latest of all endpoints', async () => { + it('should fallback to legacy index if index not found', async () => { const mockRequest = httpServerMock.createKibanaRequest({}); - const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); - (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => - Promise.resolve({ body: response }) + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() ); + (mockScopedClient.asCurrentUser.search as jest.Mock) + .mockImplementationOnce(() => { + throw new IndexNotFoundException(); + }) + .mockImplementationOnce(() => Promise.resolve({ body: response })); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${HOST_METADATA_LIST_ROUTE}`) )!; @@ -152,7 +167,11 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); + const esSearchMock = mockScopedClient.asCurrentUser.search; + // should be called twice, united index first, then legacy index + expect(esSearchMock).toHaveBeenCalledTimes(2); + expect(esSearchMock.mock.calls[0][0]!.index).toEqual(METADATA_UNITED_INDEX); + expect(esSearchMock.mock.calls[1][0]!.index).toEqual(metadataCurrentIndexPattern); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -165,7 +184,7 @@ describe('test endpoint route', () => { expect(endpointResultList.request_page_size).toEqual(10); }); - it('test find the latest of all endpoints with paging properties', async () => { + it('should return expected metadata', async () => { const mockRequest = httpServerMock.createKibanaRequest({ body: { paging_properties: [ @@ -176,14 +195,21 @@ describe('test endpoint route', () => { page_index: 1, }, ], + + filters: { + kql: 'not host.ip:10.140.73.246', + host_status: ['updating'], + }, }, }); mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + const metadata = new EndpointDocGenerator().generateHostMetadata(); + const esSearchMock = mockScopedClient.asCurrentUser.search as jest.Mock; + esSearchMock.mockImplementationOnce(() => Promise.resolve({ - body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + body: unitedMetadataSearchResponse(metadata), }) ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => @@ -195,9 +221,261 @@ describe('test endpoint route', () => { mockRequest, mockResponse ); - expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); + + expect(esSearchMock).toHaveBeenCalledTimes(1); + expect(esSearchMock.mock.calls[0][0]!.index).toEqual(METADATA_UNITED_INDEX); + expect(esSearchMock.mock.calls[0][0]?.body?.query).toEqual({ + bool: { + must: [ + { + bool: { + filter: [ + { + terms: { + 'united.agent.policy_id': [], + }, + }, + { + exists: { + field: 'united.endpoint.agent.id', + }, + }, + { + exists: { + field: 'united.agent.agent.id', + }, + }, + { + term: { + 'united.agent.active': { + value: true, + }, + }, + }, + ], + must_not: { + terms: { + 'agent.id': [ + '00000000-0000-0000-0000-000000000000', + '11111111-1111-1111-1111-111111111111', + ], + }, + }, + }, + }, + { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgrade_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgraded_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.last_checkin', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.agent.unenrollment_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + match: { + 'host.ip': '10.140.73.246', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); + expect(mockResponse.ok).toBeCalled(); + const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; + expect(endpointResultList.hosts.length).toEqual(1); + expect(endpointResultList.hosts[0].metadata).toEqual(metadata); + expect(endpointResultList.total).toEqual(1); + expect(endpointResultList.request_page_index).toEqual(10); + expect(endpointResultList.request_page_size).toEqual(10); + }); + }); + + describe('with metrics-endpoint.metadata_current_default index', () => { + beforeEach(() => { + endpointAppContextService = new EndpointAppContextService(); + mockPackageService = createMockPackageService(); + mockPackageService.getInstallation.mockReturnValue( + Promise.resolve({ + installed_kibana: [], + package_assets: [], + es_index_patterns: {}, + name: '', + version: '', + install_status: 'installed', + install_version: '', + install_started_at: '', + install_source: 'registry', + installed_es: [ + { + id: 'logs-endpoint.events.security', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: `${metadataTransformPrefix}-0.16.0-dev.0`, + type: ElasticsearchAssetType.transform, + }, + ], + }) + ); + endpointAppContextService.start({ ...startContract, packageService: mockPackageService }); + mockAgentService = startContract.agentService!; + + registerEndpointRoutes(routerMock, { + logFactory: loggingSystemMock.create(), + service: endpointAppContextService, + config: () => Promise.resolve(createMockConfig()), + experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental), + }); + }); + + afterEach(() => endpointAppContextService.stop()); + + it('test find the latest of all endpoints', async () => { + const mockRequest = httpServerMock.createKibanaRequest({}); + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() + ); + (mockScopedClient.asCurrentUser.search as jest.Mock) + .mockImplementationOnce(() => { + throw new IndexNotFoundException(); + }) + .mockImplementationOnce(() => Promise.resolve({ body: response })); + [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => + path.startsWith(`${HOST_METADATA_LIST_ROUTE}`) + )!; + mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); + mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); + await routeHandler( + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), + mockRequest, + mockResponse + ); + + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(2); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); + expect(mockResponse.ok).toBeCalled(); + const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; + expect(endpointResultList.hosts.length).toEqual(1); + expect(endpointResultList.total).toEqual(1); + expect(endpointResultList.request_page_index).toEqual(0); + expect(endpointResultList.request_page_size).toEqual(10); + }); + + it('test find the latest of all endpoints with paging properties', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + body: { + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 1, + }, + ], + }, + }); + + mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); + mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); + (mockScopedClient.asCurrentUser.search as jest.Mock) + .mockImplementationOnce(() => { + throw new IndexNotFoundException(); + }) + .mockImplementationOnce(() => + Promise.resolve({ + body: legacyMetadataSearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) + ); + [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => + path.startsWith(`${HOST_METADATA_LIST_ROUTE}`) + )!; + + await routeHandler( + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), + mockRequest, + mockResponse + ); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(2); expect( - (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool .must_not ).toContainEqual({ terms: { @@ -237,11 +515,15 @@ describe('test endpoint route', () => { mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => - Promise.resolve({ - body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + (mockScopedClient.asCurrentUser.search as jest.Mock) + .mockImplementationOnce(() => { + throw new IndexNotFoundException(); }) - ); + .mockImplementationOnce(() => + Promise.resolve({ + body: legacyMetadataSearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) + ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${HOST_METADATA_LIST_ROUTE}`) )!; @@ -255,7 +537,7 @@ describe('test endpoint route', () => { expect(mockScopedClient.asCurrentUser.search).toBeCalled(); expect( // KQL filter to be passed through - (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: { @@ -273,7 +555,7 @@ describe('test endpoint route', () => { }, }); expect( - (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: [ @@ -315,7 +597,7 @@ describe('test endpoint route', () => { const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } }); (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => - Promise.resolve({ body: createV2SearchResponse() }) + Promise.resolve({ body: legacyMetadataSearchResponse() }) ); mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); @@ -343,7 +625,9 @@ describe('test endpoint route', () => { }); it('should return a single endpoint with status healthy', async () => { - const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() + ); const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, }); @@ -377,7 +661,9 @@ describe('test endpoint route', () => { }); it('should return a single endpoint with status unhealthy when AgentService throw 404', async () => { - const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() + ); const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, @@ -412,7 +698,9 @@ describe('test endpoint route', () => { }); it('should return a single endpoint with status unhealthy when status is not offline, online or enrolling', async () => { - const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() + ); const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, @@ -448,7 +736,9 @@ describe('test endpoint route', () => { }); it('should throw error when endpoint agent is not active', async () => { - const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); + const response = legacyMetadataSearchResponse( + new EndpointDocGenerator().generateHostMetadata() + ); const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.fixtures.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.fixtures.ts new file mode 100644 index 0000000000000..d2ad9831748b3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.fixtures.ts @@ -0,0 +1,461 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export const expectedCompleteUnitedIndexQuery = { + bool: { + must: [ + { + bool: { + must_not: { + terms: { + 'agent.id': ['test-agent-id'], + }, + }, + filter: [ + { + terms: { + 'united.agent.policy_id': ['test-endpoint-policy-id'], + }, + }, + { + exists: { + field: 'united.endpoint.agent.id', + }, + }, + { + exists: { + field: 'united.agent.agent.id', + }, + }, + { + term: { + 'united.agent.active': { + value: true, + }, + }, + }, + ], + }, + }, + { + bool: { + filter: [ + { + bool: { + must_not: { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + 'united.agent.last_checkin': { + lt: 'now-120s', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + filter: [ + { + bool: { + should: [ + { + bool: { + should: [ + { + match: { + 'united.agent.last_checkin_status': 'error', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + match: { + 'united.agent.last_checkin_status': 'degraded', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgrade_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgraded_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.last_checkin', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.agent.unenrollment_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgrade_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgraded_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.last_checkin', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.agent.unenrollment_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + must_not: { + bool: { + filter: [ + { + bool: { + should: [ + { + bool: { + should: [ + { + match: { + 'united.agent.last_checkin_status': 'error', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + match: { + 'united.agent.last_checkin_status': 'degraded', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgrade_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgraded_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.last_checkin', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.agent.unenrollment_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgrade_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.upgraded_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + must_not: { + bool: { + should: [ + { + exists: { + field: 'united.agent.last_checkin', + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.agent.unenrollment_started_at', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + }, + ], + }, + }, + { + bool: { + should: [ + { + exists: { + field: 'united.endpoint.host.os.name', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts index 87de5a540ea99..46a16e48c7edf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts @@ -6,12 +6,19 @@ */ import { httpServerMock, loggingSystemMock } from '../../../../../../../src/core/server/mocks'; -import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; +import { + kibanaRequestToMetadataListESQuery, + getESQueryHostMetadataByID, + buildUnitedIndexQuery, +} from './query_builders'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants'; import { parseExperimentalConfigValue } from '../../../../common/experimental_features'; import { get } from 'lodash'; +import { KibanaRequest } from 'kibana/server'; +import { EndpointAppContext } from '../../types'; +import { expectedCompleteUnitedIndexQuery } from './query_builders.fixtures'; describe('query builder', () => { describe('MetadataListESQuery', () => { @@ -200,4 +207,68 @@ describe('query builder', () => { }); }); }); + + describe('buildUnitedIndexQuery', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let mockRequest: KibanaRequest; + let mockEndpointAppContext: EndpointAppContext; + const filters = { kql: '', host_status: [] }; + beforeEach(() => { + mockRequest = httpServerMock.createKibanaRequest({ body: { filters } }); + mockEndpointAppContext = { + logFactory: loggingSystemMock.create(), + service: new EndpointAppContextService(), + config: () => Promise.resolve(createMockConfig()), + experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental), + }; + }); + + it('correctly builds empty query', async () => { + const query = await buildUnitedIndexQuery(mockRequest, mockEndpointAppContext, [], []); + const expected = { + bool: { + filter: [ + { + terms: { + 'united.agent.policy_id': [], + }, + }, + { + exists: { + field: 'united.endpoint.agent.id', + }, + }, + { + exists: { + field: 'united.agent.agent.id', + }, + }, + { + term: { + 'united.agent.active': { + value: true, + }, + }, + }, + ], + }, + }; + expect(query.body.query).toEqual(expected); + }); + + it('correctly builds query', async () => { + mockRequest.body.filters.kql = 'united.endpoint.host.os.name : *'; + mockRequest.body.filters.host_status = ['healthy']; + const ignoredAgentIds: string[] = ['test-agent-id']; + const endpointPolicyIds: string[] = ['test-endpoint-policy-id']; + const query = await buildUnitedIndexQuery( + mockRequest, + mockEndpointAppContext, + ignoredAgentIds, + endpointPolicyIds + ); + const expected = expectedCompleteUnitedIndexQuery; + expect(query.body.query).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts index 10db3372e0891..448496671b4f9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts @@ -7,13 +7,17 @@ import type { estypes } from '@elastic/elasticsearch'; import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; -import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants'; +import { + metadataCurrentIndexPattern, + METADATA_UNITED_INDEX, +} from '../../../../common/endpoint/constants'; import { KibanaRequest } from '../../../../../../../src/core/server'; import { EndpointAppContext } from '../../types'; +import { buildStatusesKuery } from './support/agent_status'; export interface QueryBuilderOptions { unenrolledAgentIds?: string[]; - statusAgentIDs?: string[]; + statusAgentIds?: string[]; } // sort using either event.created, or HostDetails.event.created, @@ -21,7 +25,7 @@ export interface QueryBuilderOptions { // using unmapped_type avoids errors when the given field doesn't exist, and sets to the 0-value for that type // effectively ignoring it // https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#_ignoring_unmapped_fields -const MetadataSortMethod: estypes.SearchSortContainer[] = [ +export const MetadataSortMethod: estypes.SearchSortContainer[] = [ { 'event.created': { order: 'desc', @@ -50,7 +54,7 @@ export async function kibanaRequestToMetadataListESQuery( query: buildQueryBody( request, queryBuilderOptions?.unenrolledAgentIds!, - queryBuilderOptions?.statusAgentIDs! + queryBuilderOptions?.statusAgentIds! ), track_total_hits: true, sort: MetadataSortMethod, @@ -86,7 +90,7 @@ function buildQueryBody( // eslint-disable-next-line @typescript-eslint/no-explicit-any request: KibanaRequest, unerolledAgentIds: string[] | undefined, - statusAgentIDs: string[] | undefined + statusAgentIds: string[] | undefined // eslint-disable-next-line @typescript-eslint/no-explicit-any ): Record { // the filtered properties may be preceded by 'HostDetails' under an older index mapping @@ -99,21 +103,22 @@ function buildQueryBody( ], } : null; - const filterStatusAgents = statusAgentIDs - ? { - filter: [ - { - bool: { - // OR's the two together - should: [ - { terms: { 'elastic.agent.id': statusAgentIDs } }, - { terms: { 'HostDetails.elastic.agent.id': statusAgentIDs } }, - ], + const filterStatusAgents = + statusAgentIds && statusAgentIds.length + ? { + filter: [ + { + bool: { + // OR's the two together + should: [ + { terms: { 'elastic.agent.id': statusAgentIds } }, + { terms: { 'HostDetails.elastic.agent.id': statusAgentIds } }, + ], + }, }, - }, - ], - } - : null; + ], + } + : null; const idFilter = { bool: { @@ -208,3 +213,83 @@ export function getESQueryHostMetadataByIDs(agentIDs: string[]) { index: metadataCurrentIndexPattern, }; } + +export async function buildUnitedIndexQuery( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + request: KibanaRequest, + endpointAppContext: EndpointAppContext, + ignoredAgentIds: string[] | undefined, + endpointPolicyIds: string[] = [] + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): Promise> { + const pagingProperties = await getPagingProperties(request, endpointAppContext); + const statusesToFilter = request?.body?.filters?.host_status ?? []; + const statusesKuery = buildStatusesKuery(statusesToFilter); + + const filterIgnoredAgents = + ignoredAgentIds && ignoredAgentIds.length > 0 + ? { + must_not: { terms: { 'agent.id': ignoredAgentIds } }, + } + : null; + const filterEndpointPolicyAgents = { + filter: [ + // must contain an endpoint policy id + { + terms: { 'united.agent.policy_id': endpointPolicyIds }, + }, + // doc contains both agent and metadata + { exists: { field: 'united.endpoint.agent.id' } }, + { exists: { field: 'united.agent.agent.id' } }, + // agent is enrolled + { + term: { + 'united.agent.active': { + value: true, + }, + }, + }, + ], + }; + + const idFilter = { + bool: { + ...filterIgnoredAgents, + ...filterEndpointPolicyAgents, + }, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let query: Record = + filterIgnoredAgents || filterEndpointPolicyAgents + ? idFilter + : { + match_all: {}, + }; + + if (statusesKuery || request?.body?.filters?.kql) { + const kqlQuery = toElasticsearchQuery(fromKueryExpression(request.body.filters.kql)); + const q = []; + if (filterIgnoredAgents || filterEndpointPolicyAgents) { + q.push(idFilter); + } + if (statusesKuery) { + q.push(toElasticsearchQuery(fromKueryExpression(statusesKuery))); + } + q.push({ ...kqlQuery }); + query = { + bool: { must: q }, + }; + } + + return { + body: { + query, + track_total_hits: true, + sort: MetadataSortMethod, + }, + from: pagingProperties.pageIndex * pagingProperties.pageSize, + size: pagingProperties.pageSize, + index: METADATA_UNITED_INDEX, + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.test.ts index 32893f3bfbc34..1eb9cfaf109a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.test.ts @@ -5,23 +5,18 @@ * 2.0. */ -import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; -import { findAgentIDsByStatus } from './agent_status'; -import { - elasticsearchServiceMock, - savedObjectsClientMock, -} from '../../../../../../../../src/core/server/mocks'; +import { ElasticsearchClient } from 'kibana/server'; +import { buildStatusesKuery, findAgentIdsByStatus } from './agent_status'; +import { elasticsearchServiceMock } from '../../../../../../../../src/core/server/mocks'; import { AgentService } from '../../../../../../fleet/server/services'; import { createMockAgentService } from '../../../../../../fleet/server/mocks'; import { Agent } from '../../../../../../fleet/common/types/models'; import { AgentStatusKueryHelper } from '../../../../../../fleet/common/services'; describe('test filtering endpoint hosts by agent status', () => { - let mockSavedObjectClient: jest.Mocked; let mockElasticsearchClient: jest.Mocked; let mockAgentService: jest.Mocked; beforeEach(() => { - mockSavedObjectClient = savedObjectsClientMock.create(); mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; mockAgentService = createMockAgentService(); }); @@ -36,12 +31,9 @@ describe('test filtering endpoint hosts by agent status', () => { }) ); - const result = await findAgentIDsByStatus( - mockAgentService, - mockSavedObjectClient, - mockElasticsearchClient, - ['healthy'] - ); + const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [ + 'healthy', + ]); expect(result).toBeDefined(); }); @@ -64,12 +56,9 @@ describe('test filtering endpoint hosts by agent status', () => { }) ); - const result = await findAgentIDsByStatus( - mockAgentService, - mockSavedObjectClient, - mockElasticsearchClient, - ['offline'] - ); + const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [ + 'offline', + ]); const offlineKuery = AgentStatusKueryHelper.buildKueryForOfflineAgents(); expect(mockAgentService.listAgents.mock.calls[0][1].kuery).toEqual( expect.stringContaining(offlineKuery) @@ -97,12 +86,10 @@ describe('test filtering endpoint hosts by agent status', () => { }) ); - const result = await findAgentIDsByStatus( - mockAgentService, - mockSavedObjectClient, - mockElasticsearchClient, - ['updating', 'unhealthy'] - ); + const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [ + 'updating', + 'unhealthy', + ]); const unenrollKuery = AgentStatusKueryHelper.buildKueryForUpdatingAgents(); const errorKuery = AgentStatusKueryHelper.buildKueryForErrorAgents(); expect(mockAgentService.listAgents.mock.calls[0][1].kuery).toEqual( @@ -111,4 +98,53 @@ describe('test filtering endpoint hosts by agent status', () => { expect(result).toBeDefined(); expect(result).toEqual(['A', 'B']); }); + + describe('buildStatusesKuery', () => { + it('correctly builds kuery for healthy status', () => { + const status = ['healthy']; + const kuery = buildStatusesKuery(status); + const expected = + '(not (united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) )) AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))'; + expect(kuery).toEqual(expected); + }); + + it('correctly builds kuery for offline status', () => { + const status = ['offline']; + const kuery = buildStatusesKuery(status); + const expected = + '(united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) ))'; + expect(kuery).toEqual(expected); + }); + + it('correctly builds kuery for unhealthy status', () => { + const status = ['unhealthy']; + const kuery = buildStatusesKuery(status); + const expected = + '((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))'; + expect(kuery).toEqual(expected); + }); + + it('correctly builds kuery for updating status', () => { + const status = ['updating']; + const kuery = buildStatusesKuery(status); + const expected = + '(((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))'; + expect(kuery).toEqual(expected); + }); + + it('correctly builds kuery for inactive status', () => { + const status = ['inactive']; + const kuery = buildStatusesKuery(status); + const expected = '(united.agent.active:false)'; + expect(kuery).toEqual(expected); + }); + + it('correctly builds kuery for multiple statuses', () => { + const statuses = ['offline', 'unhealthy']; + const kuery = buildStatusesKuery(statuses); + const expected = + '(united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) ) OR (united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))'; + expect(kuery).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.ts index 7b08dc1488e78..f9e04f4edebee 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.ts @@ -5,28 +5,45 @@ * 2.0. */ -import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { AgentService } from '../../../../../../fleet/server'; import { AgentStatusKueryHelper } from '../../../../../../fleet/common/services'; import { Agent } from '../../../../../../fleet/common/types/models'; import { HostStatus } from '../../../../../common/endpoint/types'; -const STATUS_QUERY_MAP = new Map([ - [HostStatus.HEALTHY.toString(), AgentStatusKueryHelper.buildKueryForOnlineAgents()], - [HostStatus.OFFLINE.toString(), AgentStatusKueryHelper.buildKueryForOfflineAgents()], - [HostStatus.UNHEALTHY.toString(), AgentStatusKueryHelper.buildKueryForErrorAgents()], - [HostStatus.UPDATING.toString(), AgentStatusKueryHelper.buildKueryForUpdatingAgents()], - [HostStatus.INACTIVE.toString(), AgentStatusKueryHelper.buildKueryForInactiveAgents()], -]); +const getStatusQueryMap = (path: string = '') => + new Map([ + [HostStatus.HEALTHY.toString(), AgentStatusKueryHelper.buildKueryForOnlineAgents(path)], + [HostStatus.OFFLINE.toString(), AgentStatusKueryHelper.buildKueryForOfflineAgents(path)], + [HostStatus.UNHEALTHY.toString(), AgentStatusKueryHelper.buildKueryForErrorAgents(path)], + [HostStatus.UPDATING.toString(), AgentStatusKueryHelper.buildKueryForUpdatingAgents(path)], + [HostStatus.INACTIVE.toString(), AgentStatusKueryHelper.buildKueryForInactiveAgents(path)], + ]); -export async function findAgentIDsByStatus( +export function buildStatusesKuery(statusesToFilter: string[]): string | undefined { + if (!statusesToFilter.length) { + return; + } + const STATUS_QUERY_MAP = getStatusQueryMap('united.agent.'); + const statusQueries = statusesToFilter.map((status) => STATUS_QUERY_MAP.get(status)); + if (!statusQueries.length) { + return; + } + + return `(${statusQueries.join(' OR ')})`; +} + +export async function findAgentIdsByStatus( agentService: AgentService, - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - status: string[], + statuses: string[], pageSize: number = 1000 ): Promise { - const helpers = status.map((s) => STATUS_QUERY_MAP.get(s)); + if (!statuses.length) { + return []; + } + const STATUS_QUERY_MAP = getStatusQueryMap(); + const helpers = statuses.map((s) => STATUS_QUERY_MAP.get(s)); const searchOptions = (pageNum: number) => { return { page: pageNum, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.test.ts new file mode 100644 index 0000000000000..6302bb94b1cbd --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from 'kibana/server'; +import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks'; +import { createPackagePolicyServiceMock } from '../../../../../../fleet/server/mocks'; +import { PackagePolicy } from '../../../../../../fleet/common/types/models'; +import { PackagePolicyServiceInterface } from '../../../../../../fleet/server'; +import { getAllEndpointPackagePolicies } from './endpoint_package_policies'; + +describe('endpoint_package_policies', () => { + describe('getAllEndpointPackagePolicies', () => { + let mockSavedObjectClient: jest.Mocked; + let mockPackagePolicyService: jest.Mocked; + + beforeEach(() => { + mockSavedObjectClient = savedObjectsClientMock.create(); + mockPackagePolicyService = createPackagePolicyServiceMock(); + }); + + it('gets all endpoint package policies', async () => { + const mockPolicy: PackagePolicy = { + id: '1', + policy_id: 'test-id-1', + } as PackagePolicy; + mockPackagePolicyService.list + .mockResolvedValueOnce({ + items: [mockPolicy], + total: 1, + perPage: 10, + page: 1, + }) + .mockResolvedValueOnce({ + items: [], + total: 1, + perPage: 10, + page: 1, + }); + + const endpointPackagePolicies = await getAllEndpointPackagePolicies( + mockPackagePolicyService, + mockSavedObjectClient + ); + const expected: PackagePolicy[] = [mockPolicy]; + expect(endpointPackagePolicies).toEqual(expected); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.ts new file mode 100644 index 0000000000000..ce8f7bcac510c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from 'kibana/server'; +import { PackagePolicyServiceInterface } from '../../../../../../fleet/server'; +import { PackagePolicy } from '../../../../../../fleet/common/types/models'; + +export const getAllEndpointPackagePolicies = async ( + packagePolicyService: PackagePolicyServiceInterface, + soClient: SavedObjectsClientContract +): Promise => { + const result: PackagePolicy[] = []; + const perPage = 1000; + let page = 1; + let hasMore = true; + + while (hasMore) { + const endpointPoliciesResponse = await packagePolicyService.list(soClient, { + perPage, + page: page++, + kuery: 'ingest-package-policies.package.name:endpoint', + }); + if (endpointPoliciesResponse.items.length > 0) { + result.push(...endpointPoliciesResponse.items); + } else { + hasMore = false; + } + } + + return result; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts index 307fbb7cbf7a4..0207d59137eb3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts @@ -6,9 +6,10 @@ */ import type { estypes } from '@elastic/elasticsearch'; -import { HostMetadata } from '../../../../../common/endpoint/types'; +import { METADATA_UNITED_INDEX } from '../../../../../common/endpoint/constants'; +import { HostMetadata, UnitedAgentMetadata } from '../../../../../common/endpoint/types'; -export function createV2SearchResponse( +export function legacyMetadataSearchResponse( hostMetadata?: HostMetadata ): estypes.SearchResponse { return { @@ -42,3 +43,46 @@ export function createV2SearchResponse( }, } as unknown as estypes.SearchResponse; } + +export function unitedMetadataSearchResponse( + hostMetadata?: HostMetadata +): estypes.SearchResponse { + return { + took: 15, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: null, + hits: hostMetadata + ? [ + { + _index: METADATA_UNITED_INDEX, + _id: '8FhM0HEBYyRTvb6lOQnw', + _score: null, + _source: { + agent: { + id: 'test-agent-id', + }, + united: { + agent: {}, + endpoint: { + ...hostMetadata, + }, + }, + }, + sort: [1588337587997], + }, + ] + : [], + }, + } as unknown as estypes.SearchResponse; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.test.ts index a9b39fb16d38b..6efac10b94fef 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.test.ts @@ -5,12 +5,9 @@ * 2.0. */ -import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { findAllUnenrolledAgentIds } from './unenroll'; -import { - elasticsearchServiceMock, - savedObjectsClientMock, -} from '../../../../../../../../src/core/server/mocks'; +import { elasticsearchServiceMock } from '../../../../../../../../src/core/server/mocks'; import { AgentService } from '../../../../../../fleet/server/services'; import { createMockAgentService, @@ -20,13 +17,11 @@ import { Agent, PackagePolicy } from '../../../../../../fleet/common/types/model import { PackagePolicyServiceInterface } from '../../../../../../fleet/server'; describe('test find all unenrolled Agent id', () => { - let mockSavedObjectClient: jest.Mocked; let mockElasticsearchClient: jest.Mocked; let mockAgentService: jest.Mocked; let mockPackagePolicyService: jest.Mocked; beforeEach(() => { - mockSavedObjectClient = savedObjectsClientMock.create(); mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; mockAgentService = createMockAgentService(); mockPackagePolicyService = createPackagePolicyServiceMock(); @@ -84,26 +79,21 @@ describe('test find all unenrolled Agent id', () => { perPage: 1, }) ); + const endpointPolicyIds = ['test-endpoint-policy-id']; const agentIds = await findAllUnenrolledAgentIds( mockAgentService, - mockPackagePolicyService, - mockSavedObjectClient, - mockElasticsearchClient + mockElasticsearchClient, + endpointPolicyIds ); expect(agentIds).toBeTruthy(); expect(agentIds).toEqual(['id1', 'id2']); - expect(mockPackagePolicyService.list).toHaveBeenNthCalledWith(1, mockSavedObjectClient, { - kuery: 'ingest-package-policies.package.name:endpoint', - page: 1, - perPage: 1000, - }); expect(mockAgentService.listAgents).toHaveBeenNthCalledWith(1, mockElasticsearchClient, { page: 1, perPage: 1000, showInactive: true, - kuery: '(active : false) OR (active: true AND NOT policy_id:("abc123"))', + kuery: `(active : false) OR (active: true AND NOT policy_id:("${endpointPolicyIds[0]}"))`, }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.ts index 9b61d52c268a6..2af1c9a597ebb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.ts @@ -5,56 +5,23 @@ * 2.0. */ -import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; -import { AgentService, PackagePolicyServiceInterface } from '../../../../../../fleet/server'; +import { ElasticsearchClient } from 'kibana/server'; +import { AgentService } from '../../../../../../fleet/server'; import { Agent } from '../../../../../../fleet/common/types/models'; -const getAllAgentPolicyIdsWithEndpoint = async ( - packagePolicyService: PackagePolicyServiceInterface, - soClient: SavedObjectsClientContract -): Promise => { - const result: string[] = []; - const perPage = 1000; - let page = 1; - let hasMore = true; - - while (hasMore) { - const endpointPoliciesResponse = await packagePolicyService.list(soClient, { - perPage, - page: page++, - kuery: 'ingest-package-policies.package.name:endpoint', - }); - if (endpointPoliciesResponse.items.length > 0) { - result.push( - ...endpointPoliciesResponse.items.map((endpointPolicy) => endpointPolicy.policy_id) - ); - } else { - hasMore = false; - } - } - - return result; -}; - export async function findAllUnenrolledAgentIds( agentService: AgentService, - packagePolicyService: PackagePolicyServiceInterface, - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, + endpointPolicyIds: string[], pageSize: number = 1000 ): Promise { - const agentPoliciesWithEndpoint = await getAllAgentPolicyIdsWithEndpoint( - packagePolicyService, - soClient - ); - // We want: // 1. if no endpoint policies exist, then get all Agents // 2. if we have a list of agent policies, then Agents that are Active and that are // NOT enrolled with an Agent Policy that has endpoint const kuery = - agentPoliciesWithEndpoint.length > 0 - ? `(active : false) OR (active: true AND NOT policy_id:("${agentPoliciesWithEndpoint.join( + endpointPolicyIds.length > 0 + ? `(active : false) OR (active: true AND NOT policy_id:("${endpointPolicyIds.join( '" OR "' )}"))` : undefined; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.test.ts index 05c7c618f58c1..c175fedda388e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.test.ts @@ -12,7 +12,7 @@ import { import { elasticsearchServiceMock } from '../../../../../../../src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { ElasticsearchClientMock } from '../../../../../../../src/core/server/elasticsearch/client/mocks'; -import { createV2SearchResponse } from '../../routes/metadata/support/test_support'; +import { legacyMetadataSearchResponse } from '../../routes/metadata/support/test_support'; import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data'; import { getESQueryHostMetadataByFleetAgentIds } from '../../routes/metadata/query_builders'; import { EndpointError } from '../../errors'; @@ -38,7 +38,7 @@ describe('EndpointMetadataService', () => { endpointMetadataDoc = new EndpointDocGenerator().generateHostMetadata(); esClient.search.mockReturnValue( elasticsearchServiceMock.createSuccessTransportRequestPromise( - createV2SearchResponse(endpointMetadataDoc) + legacyMetadataSearchResponse(endpointMetadataDoc) ) ); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts b/x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts index e60d54721d5e7..74a3b3c0b08f0 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts @@ -15,9 +15,9 @@ import { telemetryIndexPattern, } from '../../../plugins/security_solution/common/endpoint/constants'; -export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) { +export function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) { const client = getService('es'); - await client.transport.request( + return client.transport.request( { method: 'DELETE', path: `_data_stream/${index}`, @@ -41,6 +41,8 @@ export async function deleteAllDocsFromIndex( }, }, index: `${index}`, + wait_for_completion: true, + refresh: true, }, { ignore: [404], @@ -48,6 +50,11 @@ export async function deleteAllDocsFromIndex( ); } +export async function deleteIndex(getService: (serviceName: 'es') => Client, index: string) { + const client = getService('es'); + await client.indices.delete({ index, ignore_unavailable: true }); +} + export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) { await deleteDataStream(getService, metadataIndexPattern); } @@ -77,3 +84,14 @@ export async function deletePolicyStream(getService: (serviceName: 'es') => Clie export async function deleteTelemetryStream(getService: (serviceName: 'es') => Client) { await deleteDataStream(getService, telemetryIndexPattern); } + +export function stopTransform(getService: (serviceName: 'es') => Client, transformId: string) { + const client = getService('es'); + const stopRequest = { + transform_id: transformId, + force: true, + wait_for_completion: true, + allow_no_match: true, + }; + return client.transform.stopTransform(stopRequest); +} diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts index cadb9a420708a..35fe0cdd6da25 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts @@ -11,279 +11,300 @@ import { deleteAllDocsFromMetadataCurrentIndex, deleteAllDocsFromMetadataIndex, deleteMetadataStream, + deleteIndex, + stopTransform, } from './data_stream_helper'; -import { HOST_METADATA_LIST_ROUTE } from '../../../plugins/security_solution/common/endpoint/constants'; - -/** - * The number of host documents in the es archive. - */ -const numberOfHostsInFixture = 3; +import { + HOST_METADATA_LIST_ROUTE, + METADATA_UNITED_INDEX, + METADATA_UNITED_TRANSFORM, +} from '../../../plugins/security_solution/common/endpoint/constants'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); describe('test metadata api', () => { - describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is empty`, () => { - it('metadata api should return empty result when index is empty', async () => { - await deleteMetadataStream(getService); - await deleteAllDocsFromMetadataIndex(getService); - await deleteAllDocsFromMetadataCurrentIndex(getService); - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send() - .expect(200); - expect(body.total).to.eql(0); - expect(body.hosts.length).to.eql(0); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); - }); + // TODO add this after endpoint package changes are merged and in snapshot + // describe('with .metrics-endpoint.metadata_united_default index', () => { + // }); - describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is not empty`, () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/endpoint/metadata/api_feature', { - useCreate: true, + describe('with metrics-endpoint.metadata_current_default index', () => { + /** + * The number of host documents in the es archive. + */ + const numberOfHostsInFixture = 3; + + describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is empty`, () => { + it('metadata api should return empty result when index is empty', async () => { + await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`); + await deleteIndex(getService, METADATA_UNITED_INDEX); + await deleteMetadataStream(getService); + await deleteAllDocsFromMetadataIndex(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send() + .expect(200); + expect(body.total).to.eql(0); + expect(body.hosts.length).to.eql(0); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); }); - // wait for transform - await new Promise((r) => setTimeout(r, 120000)); - }); - // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need - // to do it manually - after(async () => { - await deleteMetadataStream(getService); - await deleteAllDocsFromMetadataIndex(getService); - await deleteAllDocsFromMetadataCurrentIndex(getService); - }); - it('metadata api should return one entry for each host with default paging', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send() - .expect(200); - expect(body.total).to.eql(numberOfHostsInFixture); - expect(body.hosts.length).to.eql(numberOfHostsInFixture); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); }); - it('metadata api should return page based on paging properties passed.', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - paging_properties: [ - { - page_size: 1, - }, - { - page_index: 1, - }, - ], - }) - .expect(200); - expect(body.total).to.eql(numberOfHostsInFixture); - expect(body.hosts.length).to.eql(1); - expect(body.request_page_size).to.eql(1); - expect(body.request_page_index).to.eql(1); - }); + describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is not empty`, () => { + before(async () => { + // stop the united transform and delete the index + // otherwise it won't hit metrics-endpoint.metadata_current_default index + await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`); + await deleteIndex(getService, METADATA_UNITED_INDEX); + await esArchiver.load( + 'x-pack/test/functional/es_archives/endpoint/metadata/api_feature', + { + useCreate: true, + } + ); + // wait for transform + await new Promise((r) => setTimeout(r, 120000)); + }); + // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need + // to do it manually + after(async () => { + await deleteMetadataStream(getService); + await deleteAllDocsFromMetadataIndex(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); + }); + it('metadata api should return one entry for each host with default paging', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send() + .expect(200); + expect(body.total).to.eql(numberOfHostsInFixture); + expect(body.hosts.length).to.eql(numberOfHostsInFixture); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); - /* test that when paging properties produces no result, the total should reflect the actual number of metadata - in the index. - */ - it('metadata api should return accurate total metadata if page index produces no result', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - paging_properties: [ - { - page_size: 10, - }, - { - page_index: 3, - }, - ], - }) - .expect(200); - expect(body.total).to.eql(numberOfHostsInFixture); - expect(body.hosts.length).to.eql(0); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(30); - }); + it('metadata api should return page based on paging properties passed.', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 1, + }, + { + page_index: 1, + }, + ], + }) + .expect(200); + expect(body.total).to.eql(numberOfHostsInFixture); + expect(body.hosts.length).to.eql(1); + expect(body.request_page_size).to.eql(1); + expect(body.request_page_index).to.eql(1); + }); - it('metadata api should return 400 when pagingProperties is below boundaries.', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - paging_properties: [ - { - page_size: 0, - }, - { - page_index: 1, - }, - ], - }) - .expect(400); - expect(body.message).to.contain('Value must be equal to or greater than [1]'); - }); + /* test that when paging properties produces no result, the total should reflect the actual number of metadata + in the index. + */ + it('metadata api should return accurate total metadata if page index produces no result', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 3, + }, + ], + }) + .expect(200); + expect(body.total).to.eql(numberOfHostsInFixture); + expect(body.hosts.length).to.eql(0); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(30); + }); - it('metadata api should return page based on filters passed.', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: 'not (HostDetails.host.ip:10.46.229.234 or host.ip:10.46.229.234)', - }, - }) - .expect(200); - expect(body.total).to.eql(2); - expect(body.hosts.length).to.eql(2); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); + it('metadata api should return 400 when pagingProperties is below boundaries.', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 0, + }, + { + page_index: 1, + }, + ], + }) + .expect(400); + expect(body.message).to.contain('Value must be equal to or greater than [1]'); + }); - it('metadata api should return page based on filters and paging passed.', async () => { - const notIncludedIp = '10.46.229.234'; - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - paging_properties: [ - { - page_size: 10, + it('metadata api should return page based on filters passed.', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: 'not (HostDetails.host.ip:10.46.229.234 or host.ip:10.46.229.234)', }, - { - page_index: 0, + }) + .expect(200); + expect(body.total).to.eql(2); + expect(body.hosts.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); + + it('metadata api should return page based on filters and paging passed.', async () => { + const notIncludedIp = '10.46.229.234'; + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + paging_properties: [ + { + page_size: 10, + }, + { + page_index: 0, + }, + ], + filters: { + kql: `not (HostDetails.host.ip:${notIncludedIp} or host.ip:${notIncludedIp})`, }, - ], - filters: { - kql: `not (HostDetails.host.ip:${notIncludedIp} or host.ip:${notIncludedIp})`, - }, - }) - .expect(200); - expect(body.total).to.eql(2); - const resultIps: string[] = [].concat( - ...body.hosts.map((hostInfo: Record) => hostInfo.metadata.host.ip) - ); - expect(resultIps.sort()).to.eql( - [ - '10.192.213.130', - '10.70.28.129', - '10.101.149.26', - '2606:a000:ffc0:39:11ef:37b9:3371:578c', - ].sort() - ); - expect(resultIps).not.include.eql(notIncludedIp); - expect(body.hosts.length).to.eql(2); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); + }) + .expect(200); + expect(body.total).to.eql(2); + const resultIps: string[] = [].concat( + ...body.hosts.map((hostInfo: Record) => hostInfo.metadata.host.ip) + ); + expect(resultIps.sort()).to.eql( + [ + '10.192.213.130', + '10.70.28.129', + '10.101.149.26', + '2606:a000:ffc0:39:11ef:37b9:3371:578c', + ].sort() + ); + expect(resultIps).not.include.eql(notIncludedIp); + expect(body.hosts.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); - it('metadata api should return page based on host.os.Ext.variant filter.', async () => { - const variantValue = 'Windows Pro'; - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: `HostDetails.host.os.Ext.variant:${variantValue} or host.os.Ext.variant:${variantValue}`, - }, - }) - .expect(200); - expect(body.total).to.eql(2); - const resultOsVariantValue: Set = new Set( - body.hosts.map((hostInfo: Record) => hostInfo.metadata.host.os.Ext.variant) - ); - expect(Array.from(resultOsVariantValue)).to.eql([variantValue]); - expect(body.hosts.length).to.eql(2); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); + it('metadata api should return page based on host.os.Ext.variant filter.', async () => { + const variantValue = 'Windows Pro'; + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: `HostDetails.host.os.Ext.variant:${variantValue} or host.os.Ext.variant:${variantValue}`, + }, + }) + .expect(200); + expect(body.total).to.eql(2); + const resultOsVariantValue: Set = new Set( + body.hosts.map((hostInfo: Record) => hostInfo.metadata.host.os.Ext.variant) + ); + expect(Array.from(resultOsVariantValue)).to.eql([variantValue]); + expect(body.hosts.length).to.eql(2); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); - it('metadata api should return the latest event for all the events for an endpoint', async () => { - const targetEndpointIp = '10.46.229.234'; - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: `HostDetails.host.ip:${targetEndpointIp} or host.ip:${targetEndpointIp}`, - }, - }) - .expect(200); - expect(body.total).to.eql(1); - const resultIp: string = body.hosts[0].metadata.host.ip.filter( - (ip: string) => ip === targetEndpointIp - ); - expect(resultIp).to.eql([targetEndpointIp]); - expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); - expect(body.hosts.length).to.eql(1); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); + it('metadata api should return the latest event for all the events for an endpoint', async () => { + const targetEndpointIp = '10.46.229.234'; + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: `HostDetails.host.ip:${targetEndpointIp} or host.ip:${targetEndpointIp}`, + }, + }) + .expect(200); + expect(body.total).to.eql(1); + const resultIp: string = body.hosts[0].metadata.host.ip.filter( + (ip: string) => ip === targetEndpointIp + ); + expect(resultIp).to.eql([targetEndpointIp]); + expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); + expect(body.hosts.length).to.eql(1); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); - it('metadata api should return the latest event for all the events where policy status is not success', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: `not (HostDetails.Endpoint.policy.applied.status:success or Endpoint.policy.applied.status:success)`, - }, - }) - .expect(200); - const statuses: Set = new Set( - body.hosts.map( - (hostInfo: Record) => hostInfo.metadata.Endpoint.policy.applied.status - ) - ); - expect(statuses.size).to.eql(1); - expect(Array.from(statuses)).to.eql(['failure']); - }); + it('metadata api should return the latest event for all the events where policy status is not success', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: `not (HostDetails.Endpoint.policy.applied.status:success or Endpoint.policy.applied.status:success)`, + }, + }) + .expect(200); + const statuses: Set = new Set( + body.hosts.map( + (hostInfo: Record) => hostInfo.metadata.Endpoint.policy.applied.status + ) + ); + expect(statuses.size).to.eql(1); + expect(Array.from(statuses)).to.eql(['failure']); + }); - it('metadata api should return the endpoint based on the elastic agent id, and status should be unhealthy', async () => { - const targetEndpointId = 'fc0ff548-feba-41b6-8367-65e8790d0eaf'; - const targetElasticAgentId = '023fa40c-411d-4188-a941-4147bfadd095'; - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: `HostDetails.elastic.agent.id:${targetElasticAgentId} or elastic.agent.id:${targetElasticAgentId}`, - }, - }) - .expect(200); - expect(body.total).to.eql(1); - const resultHostId: string = body.hosts[0].metadata.host.id; - const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id; - expect(resultHostId).to.eql(targetEndpointId); - expect(resultElasticAgentId).to.eql(targetElasticAgentId); - expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); - expect(body.hosts[0].host_status).to.eql('unhealthy'); - expect(body.hosts.length).to.eql(1); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); - }); + it('metadata api should return the endpoint based on the elastic agent id, and status should be unhealthy', async () => { + const targetEndpointId = 'fc0ff548-feba-41b6-8367-65e8790d0eaf'; + const targetElasticAgentId = '023fa40c-411d-4188-a941-4147bfadd095'; + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: `HostDetails.elastic.agent.id:${targetElasticAgentId} or elastic.agent.id:${targetElasticAgentId}`, + }, + }) + .expect(200); + expect(body.total).to.eql(1); + const resultHostId: string = body.hosts[0].metadata.host.id; + const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id; + expect(resultHostId).to.eql(targetEndpointId); + expect(resultElasticAgentId).to.eql(targetElasticAgentId); + expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); + expect(body.hosts[0].host_status).to.eql('unhealthy'); + expect(body.hosts.length).to.eql(1); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); - it('metadata api should return all hosts when filter is empty string', async () => { - const { body } = await supertest - .post(`${HOST_METADATA_LIST_ROUTE}`) - .set('kbn-xsrf', 'xxx') - .send({ - filters: { - kql: '', - }, - }) - .expect(200); - expect(body.total).to.eql(numberOfHostsInFixture); - expect(body.hosts.length).to.eql(numberOfHostsInFixture); - expect(body.request_page_size).to.eql(10); - expect(body.request_page_index).to.eql(0); + it('metadata api should return all hosts when filter is empty string', async () => { + const { body } = await supertest + .post(`${HOST_METADATA_LIST_ROUTE}`) + .set('kbn-xsrf', 'xxx') + .send({ + filters: { + kql: '', + }, + }) + .expect(200); + expect(body.total).to.eql(numberOfHostsInFixture); + expect(body.hosts.length).to.eql(numberOfHostsInFixture); + expect(body.request_page_size).to.eql(10); + expect(body.request_page_index).to.eql(0); + }); }); }); });