diff --git a/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_action_generator.ts b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_action_generator.ts index d0baf3fbe2a7d..44da774a415d9 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_action_generator.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_action_generator.ts @@ -59,7 +59,7 @@ export class FleetActionGenerator extends BaseDataGenerator { return merge(this.generate({ data: { command: 'unisolate' } }), overrides); } - /** Generates an endpoint action response */ + /** Generates an endpoint Fleet action response */ generateResponse(overrides: DeepPartial = {}): EndpointActionResponse { const timeStamp = overrides['@timestamp'] ? new Date(overrides['@timestamp']) : new Date(); diff --git a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts index c2279ec7b8be6..21ca7780597b0 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts @@ -225,6 +225,12 @@ export interface EndpointActionData< comment?: string; parameters?: TParameters; output?: ActionResponseOutput; + /** + * If `alert_id` is defined, then action request is of type `automated` + * + * **IMPORTANT**: should be used only when response actions are created from a Rule (automated response actions) + * as this property is used to determine if an action is of type `automated` + */ alert_id?: string[]; hosts?: Record; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts index a8bd5d6ce3d34..ac6243735fdba 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts @@ -10,8 +10,8 @@ import type { EndpointActionLogRequestParams, EndpointActionLogRequestQuery, } from '../../../../common/api/endpoint'; -import { getAuditLogResponse } from '../../services'; import type { SecuritySolutionRequestHandlerContext } from '../../../types'; +import { getAuditLogResponse } from '../../services/actions/actions_audit_log'; import type { EndpointAppContext } from '../../types'; export const auditLogRequestHandler = ( diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.ts index 7a4246e7b1a2f..c0b82b42102ce 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.ts @@ -7,20 +7,18 @@ import type { ElasticsearchClient } from '@kbn/core/server'; +import type { FetchActionResponsesResult } from './utils/fetch_action_responses'; import { fetchActionResponses } from './utils/fetch_action_responses'; import { ENDPOINT_ACTIONS_INDEX } from '../../../../common/endpoint/constants'; import { formatEndpointActionResults, - categorizeResponseResults, mapToNormalizedActionRequest, getAgentHostNamesWithIds, createActionDetailsRecord, } from './utils'; import type { ActionDetails, - ActivityLogActionResponse, EndpointActivityLogAction, - EndpointActivityLogActionResponse, LogsEndpointAction, } from '../../../../common/endpoint/types'; import { catchAndWrapError } from '../../utils'; @@ -42,11 +40,11 @@ export const getActionDetailsById = async | undefined; - let actionResponses: Array; + let actionResponses: FetchActionResponsesResult; try { // Get both the Action Request(s) and action Response(s) - const [actionRequestEsSearchResults, allResponseEsHits] = await Promise.all([ + const [actionRequestEsSearchResults, actionResponseResult] = await Promise.all([ // Get the action request(s) esClient .search( @@ -66,9 +64,10 @@ export const getActionDetailsById = async response.data), + fetchActionResponses({ esClient, actionIds: [actionId] }), ]); + actionResponses = actionResponseResult; actionRequestsLogEntries = formatEndpointActionResults( actionRequestEsSearchResults?.hits?.hits ?? [] ); @@ -80,10 +79,6 @@ export const getActionDetailsById = async { expect(esClient.search).toHaveBeenNthCalledWith( 1, { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-10d', - }, + query: { + bool: { + must: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-10d', }, }, - { - range: { - '@timestamp': { - lte: 'now', - }, + }, + { + range: { + '@timestamp': { + lte: 'now', }, }, - { - terms: { - 'data.command': ['isolate', 'unisolate', 'get-file'], - }, + }, + { + terms: { + 'data.command': ['isolate', 'unisolate', 'get-file'], }, - { - terms: { - input_type: ['endpoint'], - }, + }, + { + terms: { + input_type: ['endpoint'], }, - { - terms: { - agents: ['123'], - }, + }, + { + terms: { + agents: ['123'], }, - ], - }, + }, + ], }, - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*elastic*', - }, + }, + { + bool: { + should: [ + { + query_string: { + fields: ['user_id'], + query: '*elastic*', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', }, - }, - ], + ], + }, }, + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], from: 0, index: '.logs-endpoint.actions-default', size: 20, }, - { ignore: [404], meta: true } + { ignore: [404] } ); }); @@ -430,77 +428,75 @@ describe('When using `getActionList()', () => { expect(esClient.search).toHaveBeenNthCalledWith( 1, { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-1d', - }, + query: { + bool: { + must: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-1d', }, }, - { - range: { - '@timestamp': { - lte: 'now', - }, + }, + { + range: { + '@timestamp': { + lte: 'now', }, }, - ], - }, + }, + ], }, - { - bool: { - should: [ - { - bool: { - should: [ - { - match: { - user_id: 'elastic', - }, + }, + { + bool: { + should: [ + { + bool: { + should: [ + { + match: { + user_id: 'elastic', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - { - bool: { - should: [ - { - match: { - user_id: 'kibana', - }, + }, + { + bool: { + should: [ + { + match: { + user_id: 'kibana', }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - ], - minimum_should_match: 1, - }, + }, + ], + minimum_should_match: 1, }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', }, - }, - ], + ], + }, }, + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], from: 0, index: '.logs-endpoint.actions-default', size: 10, }, - { ignore: [404], meta: true } + { ignore: [404] } ); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts index 1ebe20eb393b6..b82e7852955c9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts @@ -6,23 +6,23 @@ */ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; +import { fetchActionRequests } from './utils/fetch_action_requests'; +import type { FetchActionResponsesResult } from './utils/fetch_action_responses'; import { fetchActionResponses } from './utils/fetch_action_responses'; import { ENDPOINT_DEFAULT_PAGE_SIZE } from '../../../../common/endpoint/constants'; import { CustomHttpRequestError } from '../../../utils/custom_http_request_error'; -import type { ActionListApiResponse } from '../../../../common/endpoint/types'; +import type { ActionListApiResponse, LogsEndpointAction } from '../../../../common/endpoint/types'; import type { ResponseActionAgentType, ResponseActionStatus, + ResponseActionsApiCommandNames, + ResponseActionType, } from '../../../../common/endpoint/service/response_actions/constants'; -import { getActions } from '../../utils/action_list_helpers'; - import { - categorizeResponseResults, createActionDetailsRecord, - formatEndpointActionResults, getAgentHostNamesWithIds, + mapResponsesByActionId, mapToNormalizedActionRequest, } from './utils'; import type { EndpointMetadataService } from '../metadata'; @@ -200,16 +200,13 @@ const getActionDetailsList = async ({ actionDetails: ActionListApiResponse['data']; totalRecords: number; }> => { - let actionRequests; - let actionReqIds; - let actionResponses; - let agentsHostInfo: { [id: string]: string }; + let actionRequests: LogsEndpointAction[] = []; + let totalRecords: number = 0; try { - // fetch actions with matching agent_ids if any - const { actionIds, actionRequests: _actionRequests } = await getActions({ + const { data, total } = await fetchActionRequests({ agentTypes, - commands, + commands: commands as ResponseActionsApiCommandNames[], esClient, elasticAgentIds, startDate, @@ -218,10 +215,12 @@ const getActionDetailsList = async ({ size, userIds, unExpiredOnly, - types, + types: types as ResponseActionType[], + logger, }); - actionRequests = _actionRequests; - actionReqIds = actionIds; + + actionRequests = data; + totalRecords = total; } catch (error) { // all other errors const err = new CustomHttpRequestError( @@ -234,18 +233,18 @@ const getActionDetailsList = async ({ throw err; } - if (!actionRequests?.body?.hits?.hits) { - // return empty details array + if (!totalRecords) { return { actionDetails: [], totalRecords: 0 }; } - // format endpoint actions into { type, item } structure - const formattedActionRequests = formatEndpointActionResults(actionRequests?.body?.hits?.hits); - const totalRecords = (actionRequests?.body?.hits?.total as unknown as SearchTotalHits).value; - - // normalized actions with a flat structure to access relevant values - const normalizedActionRequests: Array> = - formattedActionRequests.map((action) => mapToNormalizedActionRequest(action.item.data)); + const normalizedActionRequests = actionRequests.map(mapToNormalizedActionRequest); + const agentIds: string[] = []; + const actionReqIds = normalizedActionRequests.map((actionReq) => { + agentIds.push(...actionReq.agents); + return actionReq.id; + }); + let actionResponses: FetchActionResponsesResult; + let agentsHostInfo: { [id: string]: string }; try { // get all responses for given action Ids and agent Ids @@ -253,10 +252,10 @@ const getActionDetailsList = async ({ [actionResponses, agentsHostInfo] = await Promise.all([ fetchActionResponses({ esClient, agentIds: elasticAgentIds, actionIds: actionReqIds }), - await getAgentHostNamesWithIds({ + getAgentHostNamesWithIds({ esClient, metadataService, - agentIds: normalizedActionRequests.map((action) => action.agents).flat(), + agentIds, }), ]); } catch (error) { @@ -271,22 +270,14 @@ const getActionDetailsList = async ({ throw err; } - // categorize responses as fleet and endpoint responses - const categorizedResponses = categorizeResponseResults({ - results: actionResponses.data, - }); - - // compute action details list for each action id + const responsesByActionId = mapResponsesByActionId(actionResponses); const actionDetails: ActionListApiResponse['data'] = normalizedActionRequests.map((action) => { - // pick only those responses that match the current action id - const matchedResponses = categorizedResponses.filter((categorizedResponse) => - categorizedResponse.type === 'response' - ? categorizedResponse.item.data.EndpointActions.action_id === action.id - : categorizedResponse.item.data.action_id === action.id + const actionRecord = createActionDetailsRecord( + action, + responsesByActionId[action.id] ?? { fleetResponses: [], endpointResponses: [] }, + agentsHostInfo ); - const actionRecord = createActionDetailsRecord(action, matchedResponses, agentsHostInfo); - if (withOutputs && !withOutputs.includes(action.id)) { delete actionRecord.outputs; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/actions.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/actions.ts deleted file mode 100644 index f86090a778d85..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/actions.ts +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { TransportResult } from '@elastic/elasticsearch'; -import type { SecuritySolutionRequestHandlerContext } from '../../../types'; -import type { - ActivityLog, - ActivityLogEntry, - EndpointAction, - EndpointActionResponse, - LogsEndpointAction, - LogsEndpointActionResponse, -} from '../../../../common/endpoint/types'; -import { getActionRequestsResult, getActionResponsesResult, getTimeSortedData } from '../../utils'; - -import { categorizeActionResults, categorizeResponseResults, getUniqueLogData } from './utils'; - -export const getAuditLogResponse = async ({ - elasticAgentId, - page, - pageSize, - startDate, - endDate, - context, - logger, -}: { - elasticAgentId: string; - page: number; - pageSize: number; - startDate: string; - endDate: string; - context: SecuritySolutionRequestHandlerContext; - logger: Logger; -}): Promise => { - const size = Math.floor(pageSize / 2); - const from = page <= 1 ? 0 : page * size - size + 1; - - const data = await getActivityLog({ - context, - from, - size, - startDate, - endDate, - elasticAgentId, - logger, - }); - - return { - page, - pageSize, - startDate, - endDate, - data, - }; -}; - -const getActivityLog = async ({ - context, - size, - from, - startDate, - endDate, - elasticAgentId, - logger, -}: { - context: SecuritySolutionRequestHandlerContext; - elasticAgentId: string; - size: number; - from: number; - startDate: string; - endDate: string; - logger: Logger; -}): Promise => { - let actionsResult: TransportResult, unknown>; - let responsesResult: TransportResult, unknown>; - - try { - // fetch actions with matching agent_id - const { actionIds, actionRequests } = await getActionRequestsResult({ - context, - logger, - elasticAgentId, - startDate, - endDate, - size, - from, - }); - actionsResult = actionRequests; - - // fetch responses with matching unique set of `action_id`s - responsesResult = await getActionResponsesResult({ - actionIds: [...new Set(actionIds)], // de-dupe `action_id`s - context, - logger, - elasticAgentId, - startDate, - endDate, - }); - } catch (error) { - logger.error(error); - throw error; - } - if (actionsResult?.statusCode !== 200) { - logger.error(`Error fetching actions log for agent_id ${elasticAgentId}`); - throw new Error(`Error fetching actions log for agent_id ${elasticAgentId}`); - } - - // label record as `action`, `fleetAction` - const responses = categorizeResponseResults({ - results: responsesResult?.body?.hits?.hits as Array< - estypes.SearchHit - >, - }); - - // label record as `response`, `fleetResponse` - const actions = categorizeActionResults({ - results: actionsResult?.body?.hits?.hits as Array< - estypes.SearchHit - >, - }); - - // filter out the duplicate endpoint actions that also have fleetActions - // include endpoint actions that have no fleet actions - const uniqueLogData = getUniqueLogData([...responses, ...actions]); - - // sort by @timestamp in desc order, newest first - const sortedData = getTimeSortedData(uniqueLogData); - - return sortedData; -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/audit_log_helpers.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts similarity index 54% rename from x-pack/plugins/security_solution/server/endpoint/utils/audit_log_helpers.ts rename to x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts index 93f71d9f690a2..e53156cee78ea 100644 --- a/x-pack/plugins/security_solution/server/endpoint/utils/audit_log_helpers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/actions_audit_log.ts @@ -6,24 +6,33 @@ */ import type { Logger } from '@kbn/core/server'; -import type { SearchRequest } from '@kbn/data-plugin/public'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportResult } from '@elastic/elasticsearch'; +import type { SearchRequest } from '@kbn/data-plugin/common'; import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common'; import { + ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, ENDPOINT_ACTIONS_DS, ENDPOINT_ACTIONS_INDEX, - ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, -} from '../../../common/endpoint/constants'; -import type { SecuritySolutionRequestHandlerContext } from '../../types'; +} from '../../../../common/endpoint/constants'; +import type { SecuritySolutionRequestHandlerContext } from '../../../types'; import type { ActivityLog, + ActivityLogEntry, EndpointAction, + EndpointActionResponse, LogsEndpointAction, -} from '../../../common/endpoint/types'; -import { doesLogsEndpointActionsIndexExist } from './yes_no_data_stream'; -import { getDateFilters } from '../services/actions/utils'; -import { ACTION_REQUEST_INDICES, ACTION_RESPONSE_INDICES } from '../services/actions/constants'; + LogsEndpointActionResponse, +} from '../../../../common/endpoint/types'; +import { doesLogsEndpointActionsIndexExist } from '../../utils'; + +import { + categorizeActionResults, + categorizeResponseResults, + getDateFilters, + getUniqueLogData, +} from './utils'; +import { ACTION_REQUEST_INDICES, ACTION_RESPONSE_INDICES } from './constants'; const queryOptions = { headers: { @@ -32,13 +41,134 @@ const queryOptions = { ignore: [404], }; -export const getTimeSortedData = (data: ActivityLog['data']): ActivityLog['data'] => { +/** + * Used only for the deprecated `/api/endpoint/action_log/{agent_id}` legacy API route + * + * Use newer response action services instead + * + * @deprecated + */ +export const getAuditLogResponse = async ({ + elasticAgentId, + page, + pageSize, + startDate, + endDate, + context, + logger, +}: { + elasticAgentId: string; + page: number; + pageSize: number; + startDate: string; + endDate: string; + context: SecuritySolutionRequestHandlerContext; + logger: Logger; +}): Promise => { + const size = Math.floor(pageSize / 2); + const from = page <= 1 ? 0 : page * size - size + 1; + + const data = await getActivityLog({ + context, + from, + size, + startDate, + endDate, + elasticAgentId, + logger, + }); + + return { + page, + pageSize, + startDate, + endDate, + data, + }; +}; + +const getActivityLog = async ({ + context, + size, + from, + startDate, + endDate, + elasticAgentId, + logger, +}: { + context: SecuritySolutionRequestHandlerContext; + elasticAgentId: string; + size: number; + from: number; + startDate: string; + endDate: string; + logger: Logger; +}): Promise => { + let actionsResult: TransportResult, unknown>; + let responsesResult: TransportResult, unknown>; + + try { + // fetch actions with matching agent_id + const { actionIds, actionRequests } = await getActionRequestsResult({ + context, + logger, + elasticAgentId, + startDate, + endDate, + size, + from, + }); + actionsResult = actionRequests; + + // fetch responses with matching unique set of `action_id`s + responsesResult = await getActionResponsesResult({ + actionIds: [...new Set(actionIds)], // de-dupe `action_id`s + context, + logger, + elasticAgentId, + startDate, + endDate, + }); + } catch (error) { + logger.error(error); + throw error; + } + if (actionsResult?.statusCode !== 200) { + logger.error(`Error fetching actions log for agent_id ${elasticAgentId}`); + throw new Error(`Error fetching actions log for agent_id ${elasticAgentId}`); + } + + // label record as `action`, `fleetAction` + const responses = categorizeResponseResults({ + results: responsesResult?.body?.hits?.hits as Array< + estypes.SearchHit + >, + }); + + // label record as `response`, `fleetResponse` + const actions = categorizeActionResults({ + results: actionsResult?.body?.hits?.hits as Array< + estypes.SearchHit + >, + }); + + // filter out the duplicate endpoint actions that also have fleetActions + // include endpoint actions that have no fleet actions + const uniqueLogData = getUniqueLogData([...responses, ...actions]); + + // sort by @timestamp in desc order, newest first + const sortedData = getTimeSortedData(uniqueLogData); + + return sortedData; +}; + +const getTimeSortedData = (data: ActivityLog['data']): ActivityLog['data'] => { return data.sort((a, b) => new Date(b.item.data['@timestamp']) > new Date(a.item.data['@timestamp']) ? 1 : -1 ); }; -export const getActionRequestsResult = async ({ +const getActionRequestsResult = async ({ context, logger, elasticAgentId, @@ -109,7 +239,7 @@ export const getActionRequestsResult = async ({ } }; -export const getActionResponsesResult = async ({ +const getActionResponsesResult = async ({ context, logger, elasticAgentId, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.test.ts index a027f8e662c85..67633e3badcc9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.test.ts @@ -11,6 +11,7 @@ import type { ResponseActionsClientUpdateCasesOptions, ResponseActionsClientWriteActionRequestToEndpointIndexOptions, ResponseActionsClientWriteActionResponseToEndpointIndexOptions, + ResponseActionsClientPendingAction, } from './base_response_actions_client'; import { HOST_NOT_ENROLLED, ResponseActionsClientImpl } from './base_response_actions_client'; import type { @@ -677,11 +678,7 @@ describe('ResponseActionsClientImpl base class', () => { }); it('should provide an array of pending actions', async () => { - const iterationData: Array< - Array< - LogsEndpointAction - > - > = []; + const iterationData: ResponseActionsClientPendingAction[][] = []; for await (const pendingActions of baseClassMock.fetchAllPendingActions()) { iterationData.push(pendingActions); @@ -690,12 +687,15 @@ describe('ResponseActionsClientImpl base class', () => { expect(iterationData.length).toBe(2); expect(iterationData[0]).toEqual([]); // First page of results should be empty due to how the mock was setup expect(iterationData[1]).toEqual([ - expect.objectContaining({ - EndpointActions: expect.objectContaining({ - action_id: 'action-id-2', + { + action: expect.objectContaining({ + EndpointActions: expect.objectContaining({ + action_id: 'action-id-2', + }), + agent: { id: 'agent-b' }, }), - agent: { id: 'agent-b' }, - }), + pendingAgentIds: ['agent-b'], + }, ]); }); }); @@ -738,7 +738,7 @@ class MockClassWithExposedProtectedMembers extends ResponseActionsClientImpl { return super.writeActionResponseToEndpointIndex(options); } - public fetchAllPendingActions(): AsyncIterable { + public fetchAllPendingActions(): AsyncIterable { return super.fetchAllPendingActions(); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts index 148f04a587990..ddd2eef4aa65e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/lib/base_response_actions_client.ts @@ -19,7 +19,12 @@ import { fetchEndpointActionResponses, } from '../../utils/fetch_action_responses'; import { createEsSearchIterable } from '../../../../utils/create_es_search_iterable'; -import { categorizeResponseResults, getActionRequestExpiration } from '../../utils'; +import { + getActionCompletionInfo, + getActionRequestExpiration, + mapResponsesByActionId, + mapToNormalizedActionRequest, +} from '../../utils'; import { isActionSupportedByAgentType } from '../../../../../../common/endpoint/service/response_actions/is_response_action_supported'; import type { EndpointAppContextService } from '../../../../endpoint_app_context_services'; import { APP_ID } from '../../../../../../common'; @@ -70,7 +75,6 @@ import type { import { stringify } from '../../../../utils/stringify'; import { CASE_ATTACHMENT_ENDPOINT_TYPE_ID } from '../../../../../../common/constants'; import { EMPTY_COMMENT } from '../../../../utils/translations'; -import { ActivityLogItemTypes } from '../../../../../../common/endpoint/types'; const ENTERPRISE_LICENSE_REQUIRED_MSG = i18n.translate( 'xpack.securitySolution.responseActionsList.error.licenseTooLow', @@ -154,6 +158,15 @@ export interface FetchActionResponseEsDocsResponse< [agentId: string]: LogsEndpointActionResponse; } +export interface ResponseActionsClientPendingAction< + TParameters extends EndpointActionDataParameterTypes = EndpointActionDataParameterTypes, + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TMeta extends {} = {} +> { + action: LogsEndpointAction; + pendingAgentIds: string[]; +} + /** * Base class for a Response Actions client */ @@ -548,7 +561,7 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient return validateActionId(this.options.esClient, actionId, this.agentType); } - protected fetchAllPendingActions(): AsyncIterable { + protected fetchAllPendingActions(): AsyncIterable { const esClient = this.options.esClient; const query: QueryDslQueryContainer = { bool: { @@ -574,62 +587,42 @@ export abstract class ResponseActionsClientImpl implements ResponseActionsClient sort: '@timestamp', query, }, - resultsMapper: async (data): Promise => { + resultsMapper: async (data): Promise => { const actionRequests = data.hits.hits.map((hit) => hit._source as LogsEndpointAction); - const pendingRequests: LogsEndpointAction[] = []; + const pendingRequests: ResponseActionsClientPendingAction[] = []; if (actionRequests.length > 0) { - const actionResults = ( - await fetchActionResponses({ - esClient, - actionIds: actionRequests.map((action) => action.EndpointActions.action_id), - }) - ).data; - const categorizedResults = categorizeResponseResults({ results: actionResults }); - - // An object whose keys are the Action ID and values are an array of agent IDs that have sent their responses - // ex: { uuid-1: [ agentA, agentB ] } - const agentResponsesForActionId = categorizedResults.reduce((acc, categoriezedResult) => { - let actionId = ''; - let agentId = ''; - - if (categoriezedResult.type === ActivityLogItemTypes.RESPONSE) { - actionId = categoriezedResult.item.data.EndpointActions.action_id; - agentId = Array.isArray(categoriezedResult.item.data.agent.id) - ? categoriezedResult.item.data.agent.id[0] - : categoriezedResult.item.data.agent.id; - } else { - actionId = categoriezedResult.item.data.action_id; - agentId = categoriezedResult.item.data.agent_id; - } - - if (!acc[actionId]) { - acc[actionId] = []; - } - - acc[actionId].push(agentId); - - return acc; - }, {} as Record); + const actionResults = await fetchActionResponses({ + esClient, + actionIds: actionRequests.map((action) => action.EndpointActions.action_id), + }); + const responsesByActionId = mapResponsesByActionId(actionResults); // Determine what actions are still pending for (const actionRequest of actionRequests) { - const thisActionAgentResponses = - agentResponsesForActionId[actionRequest.EndpointActions.action_id]; - - if (!thisActionAgentResponses) { - pendingRequests.push(actionRequest); - } else { - const thisActionAgentIds = Array.isArray(actionRequest.agent.id) - ? actionRequest.agent.id - : [actionRequest.agent.id]; - - // If at least one Agent has not yet sent a response, then this action is still pending - if ( - !thisActionAgentIds.every((agentId) => thisActionAgentResponses.includes(agentId)) - ) { - pendingRequests.push(actionRequest); + const actionCompleteInfo = getActionCompletionInfo( + mapToNormalizedActionRequest(actionRequest), + responsesByActionId[actionRequest.EndpointActions.action_id] ?? { + endpointResponses: [], + fleetResponses: [], } + ); + + // If not completed, add action to the pending list and calculate the list of agent IDs + // whose response we are still waiting on + if (!actionCompleteInfo.isCompleted) { + const pendingActionData: ResponseActionsClientPendingAction = { + action: actionRequest, + pendingAgentIds: [], + }; + + for (const [agentId, agentIdState] of Object.entries(actionCompleteInfo.agentState)) { + if (!agentIdState.isCompleted) { + pendingActionData.pendingAgentIds.push(agentId); + } + } + + pendingRequests.push(pendingActionData); } } } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/sentinelone/sentinel_one_actions_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/sentinelone/sentinel_one_actions_client.ts index a757eb16b63bd..be612c3f2864d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/sentinelone/sentinel_one_actions_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/clients/sentinelone/sentinel_one_actions_client.ts @@ -67,6 +67,7 @@ import type { ResponseActionsClientOptions, ResponseActionsClientValidateRequestResponse, ResponseActionsClientWriteActionRequestToEndpointIndexOptions, + ResponseActionsClientPendingAction, } from '../lib/base_response_actions_client'; import { ResponseActionsClientImpl } from '../lib/base_response_actions_client'; import { RESPONSE_ACTIONS_ZIP_PASSCODE } from '../../../../../../common/endpoint/service/response_actions/constants'; @@ -569,7 +570,7 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { return; } - const pendingActionsByType = groupBy(pendingActions, 'EndpointActions.data.command'); + const pendingActionsByType = groupBy(pendingActions, 'action.EndpointActions.data.command'); for (const [actionType, typePendingActions] of Object.entries(pendingActionsByType)) { if (abortSignal.aborted) { @@ -582,7 +583,7 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { { const isolationResponseDocs = await this.checkPendingIsolateOrReleaseActions( typePendingActions as Array< - LogsEndpointAction + ResponseActionsClientPendingAction >, actionType as 'isolate' | 'unisolate' ); @@ -596,7 +597,7 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { { const responseDocsForGetFile = await this.checkPendingGetFileActions( typePendingActions as Array< - LogsEndpointAction< + ResponseActionsClientPendingAction< ResponseActionGetFileParameters, ResponseActionGetFileOutputContent, SentinelOneGetFileRequestMeta @@ -651,7 +652,9 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { * @private */ private async checkPendingIsolateOrReleaseActions( - actionRequests: Array>, + actionRequests: Array< + ResponseActionsClientPendingAction + >, command: ResponseActionsApiCommandNames & ('isolate' | 'unisolate') ): Promise { const completedResponses: LogsEndpointActionResponse[] = []; @@ -664,44 +667,48 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { // Create the `OR` clause that filters for each agent id and an updated date of greater than the date when // the isolate request was created - const agentListQuery: QueryDslQueryContainer[] = actionRequests.reduce((acc, action) => { - const s1AgentId = action.meta?.agentId; - - if (s1AgentId) { - if (!actionsByAgentId[s1AgentId]) { - actionsByAgentId[s1AgentId] = []; - } + const agentListQuery: QueryDslQueryContainer[] = actionRequests.reduce( + (acc, pendingActionData) => { + const action = pendingActionData.action; + const s1AgentId = action.meta?.agentId; + + if (s1AgentId) { + if (!actionsByAgentId[s1AgentId]) { + actionsByAgentId[s1AgentId] = []; + } - actionsByAgentId[s1AgentId].push(action); + actionsByAgentId[s1AgentId].push(action); - acc.push({ - bool: { - filter: [ - { term: { 'sentinel_one.activity.agent.id': s1AgentId } }, - { range: { 'sentinel_one.activity.updated_at': { gt: action['@timestamp'] } } }, - ], - }, - }); - } else { - // This is an edge case and should never happen. But just in case :-) - warnings.push( - `${command} response action ID [${action.EndpointActions.action_id}] missing SentinelOne agent ID, thus unable to check on it's status. Forcing it to complete as failure.` - ); - - completedResponses.push( - this.buildActionResponseEsDoc<{}, SentinelOneIsolationResponseMeta>({ - actionId: action.EndpointActions.action_id, - agentId: Array.isArray(action.agent.id) ? action.agent.id[0] : action.agent.id, - data: { command }, - error: { - message: `Unable to very if action completed. SentinelOne agent id ('meta.agentId') missing on action request document!`, + acc.push({ + bool: { + filter: [ + { term: { 'sentinel_one.activity.agent.id': s1AgentId } }, + { range: { 'sentinel_one.activity.updated_at': { gt: action['@timestamp'] } } }, + ], }, - }) - ); - } + }); + } else { + // This is an edge case and should never happen. But just in case :-) + warnings.push( + `${command} response action ID [${action.EndpointActions.action_id}] missing SentinelOne agent ID, thus unable to check on it's status. Forcing it to complete as failure.` + ); - return acc; - }, [] as QueryDslQueryContainer[]); + completedResponses.push( + this.buildActionResponseEsDoc<{}, SentinelOneIsolationResponseMeta>({ + actionId: action.EndpointActions.action_id, + agentId: Array.isArray(action.agent.id) ? action.agent.id[0] : action.agent.id, + data: { command }, + error: { + message: `Unable to very if action completed. SentinelOne agent id ('meta.agentId') missing on action request document!`, + }, + }) + ); + } + + return acc; + }, + [] as QueryDslQueryContainer[] + ); if (agentListQuery.length > 0) { const query: QueryDslQueryContainer = { @@ -837,7 +844,7 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { private async checkPendingGetFileActions( actionRequests: Array< - LogsEndpointAction< + ResponseActionsClientPendingAction< ResponseActionGetFileParameters, ResponseActionGetFileOutputContent, SentinelOneGetFileRequestMeta @@ -874,7 +881,7 @@ export class SentinelOneActionsClient extends ResponseActionsClientImpl { }, }, ], - should: actionRequests.reduce((acc, action) => { + should: actionRequests.reduce((acc, { action }) => { const s1AgentId = action.meta?.agentId; const s1CommandBatchUUID = action.meta?.commandBatchUuid; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/index.ts index 1490da7b018a0..8d942c945a274 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -export * from './actions'; export { getActionDetailsById } from './action_details_by_id'; export { getActionList, getActionListByStatus } from './action_list'; export { getPendingActionsSummary } from './pending_actions_summary'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.test.ts new file mode 100644 index 0000000000000..f56afd92e7484 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.test.ts @@ -0,0 +1,365 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import type { FetchActionRequestsOptions } from './fetch_action_requests'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { applyActionListEsSearchMock } from '../mocks'; +import { fetchActionRequests } from './fetch_action_requests'; +import { ENDPOINT_ACTIONS_INDEX } from '../../../../../common/endpoint/constants'; + +describe('fetchActionRequests()', () => { + let esClientMock: ElasticsearchClientMock; + let fetchOptions: FetchActionRequestsOptions; + + beforeEach(() => { + esClientMock = elasticsearchServiceMock.createScopedClusterClient().asInternalUser; + + fetchOptions = { + logger: loggingSystemMock.create().get(), + esClient: esClientMock, + from: 0, + size: 10, + }; + + applyActionListEsSearchMock(esClientMock); + }); + + it('should return an array of items', async () => { + await expect(fetchActionRequests(fetchOptions)).resolves.toEqual({ + data: [ + { + '@timestamp': '2022-04-27T16:08:47.449Z', + EndpointActions: { + action_id: '123', + data: { + command: 'kill-process', + comment: '5wb6pu6kh2xix5i', + }, + expiration: '2022-05-10T16:08:47.449Z', + input_type: 'endpoint', + type: 'INPUT_ACTION', + }, + agent: { + id: 'agent-a', + }, + user: { + id: 'Shanel', + }, + }, + ], + total: 1, + from: 0, + size: 10, + }); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { + bool: { + filter: [], + }, + }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter using `from` and `size` provided on input', async () => { + fetchOptions.size = 101; + fetchOptions.from = 50; + const response = await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { + bool: { + filter: [], + }, + }, + ], + }, + }, + from: 50, + size: 101, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + + expect(response).toMatchObject({ size: 101, from: 50 }); + }); + + it('should filter by commands', async () => { + fetchOptions.commands = ['isolate', 'upload']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { + bool: { + filter: [{ terms: { 'data.command': ['isolate', 'upload'] } }], + }, + }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by agent types', async () => { + fetchOptions.agentTypes = ['crowdstrike']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [{ bool: { filter: [{ terms: { input_type: ['crowdstrike'] } }] } }], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by agent ids', async () => { + fetchOptions.elasticAgentIds = ['agent-1', 'agent-2']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [{ bool: { filter: [{ terms: { agents: ['agent-1', 'agent-2'] } }] } }], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter for un-expired', async () => { + fetchOptions.unExpiredOnly = true; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [{ bool: { filter: [{ range: { expiration: { gte: 'now' } } }] } }], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by start date', async () => { + fetchOptions.startDate = '2024-05-20T14:56:27.352Z'; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { bool: { filter: [{ range: { '@timestamp': { gte: fetchOptions.startDate } } }] } }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by end date', async () => { + fetchOptions.endDate = '2024-05-20T19:56:27.352Z'; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { bool: { filter: [{ range: { '@timestamp': { lte: fetchOptions.endDate } } }] } }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by user ids', async () => { + fetchOptions.userIds = ['user-1', 'user-2']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [ + { bool: { filter: [] } }, + { + bool: { + minimum_should_match: 1, + should: [ + { + bool: { minimum_should_match: 1, should: [{ match: { user_id: 'user-1' } }] }, + }, + { + bool: { minimum_should_match: 1, should: [{ match: { user_id: 'user-2' } }] }, + }, + ], + }, + }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by `manual` action type', async () => { + fetchOptions.types = ['manual']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [{ bool: { filter: [] } }], + must_not: { exists: { field: 'data.alert_id' } }, + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should filter by `automated` action type', async () => { + fetchOptions.types = ['manual']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + must: [{ bool: { filter: [] } }], + must_not: { exists: { field: 'data.alert_id' } }, + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); + + it('should auery using all available filters', async () => { + fetchOptions.types = ['automated']; + fetchOptions.userIds = ['user-1']; + fetchOptions.startDate = '2023-05-20T19:56:27.352Z'; + fetchOptions.endDate = '2024-05-20T19:56:27.352Z'; + fetchOptions.unExpiredOnly = true; + fetchOptions.elasticAgentIds = ['agent-1', 'agent-2']; + fetchOptions.agentTypes = ['sentinel_one']; + fetchOptions.commands = ['kill-process']; + await fetchActionRequests(fetchOptions); + + expect(esClientMock.search).toHaveBeenCalledWith( + { + index: ENDPOINT_ACTIONS_INDEX, + query: { + bool: { + filter: { exists: { field: 'data.alert_id' } }, + must: [ + { + bool: { + filter: [ + { range: { '@timestamp': { gte: '2023-05-20T19:56:27.352Z' } } }, + { range: { '@timestamp': { lte: '2024-05-20T19:56:27.352Z' } } }, + { terms: { 'data.command': ['kill-process'] } }, + { terms: { input_type: ['sentinel_one'] } }, + { terms: { agents: ['agent-1', 'agent-2'] } }, + { range: { expiration: { gte: 'now' } } }, + ], + }, + }, + { + bool: { + minimum_should_match: 1, + should: [{ match: { user_id: 'user-1' } }], + }, + }, + ], + }, + }, + from: 0, + size: 10, + sort: [{ '@timestamp': { order: 'desc' } }], + }, + { ignore: [404] } + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.ts new file mode 100644 index 0000000000000..6bfbf752c8d45 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_requests.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SearchRequest, + QueryDslQueryContainer, + QueryDslBoolQuery, + SearchTotalHits, +} from '@elastic/elasticsearch/lib/api/types'; +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { Logger } from '@kbn/logging'; +import { stringify } from '../../../utils/stringify'; +import { getDateFilters } from '../..'; +import { ENDPOINT_ACTIONS_INDEX } from '../../../../../common/endpoint/constants'; +import { catchAndWrapError } from '../../../utils'; +import type { LogsEndpointAction } from '../../../../../common/endpoint/types'; +import type { + ResponseActionAgentType, + ResponseActionsApiCommandNames, + ResponseActionType, +} from '../../../../../common/endpoint/service/response_actions/constants'; + +export interface FetchActionRequestsOptions { + esClient: ElasticsearchClient; + logger: Logger; + from?: number; + size?: number; + startDate?: string; + endDate?: string; + agentTypes?: ResponseActionAgentType[]; + commands?: ResponseActionsApiCommandNames[]; + elasticAgentIds?: string[]; + userIds?: string[]; + unExpiredOnly?: boolean; + types?: ResponseActionType[]; +} + +interface FetchActionRequestsResponse { + data: LogsEndpointAction[]; + total: number; + from: number; + size: number; +} + +/** + * Fetches a list of Action Requests from the Endpoint action request index (not fleet) + * @param logger + * @param agentTypes + * @param commands + * @param elasticAgentIds + * @param esClient + * @param endDate + * @param from + * @param size + * @param startDate + * @param userIds + * @param unExpiredOnly + * @param types + */ +export const fetchActionRequests = async ({ + logger, + esClient, + from = 0, + size = 10, + agentTypes, + commands, + elasticAgentIds, + endDate, + startDate, + userIds, + unExpiredOnly = false, + types, +}: FetchActionRequestsOptions): Promise => { + const additionalFilters = []; + + if (commands?.length) { + additionalFilters.push({ terms: { 'data.command': commands } }); + } + + if (agentTypes?.length) { + additionalFilters.push({ terms: { input_type: agentTypes } }); + } + + if (elasticAgentIds?.length) { + additionalFilters.push({ terms: { agents: elasticAgentIds } }); + } + + if (unExpiredOnly) { + additionalFilters.push({ range: { expiration: { gte: 'now' } } }); + } + + const must: QueryDslQueryContainer[] = [ + { + bool: { + filter: [...getDateFilters({ startDate, endDate }), ...additionalFilters], + }, + }, + ]; + + if (userIds?.length) { + const userIdsKql = userIds.map((userId) => `user_id:${userId}`).join(' or '); + const mustClause = toElasticsearchQuery(fromKueryExpression(userIdsKql)); + must.push(mustClause); + } + + const isNotASingleActionType = !types || (types && types.length > 1); + + const actionsSearchQuery: SearchRequest = { + index: ENDPOINT_ACTIONS_INDEX, + size, + from, + query: { + bool: { + must, + ...(isNotASingleActionType ? {} : getActionTypeFilter(types[0])), + }, + }, + sort: [{ '@timestamp': { order: 'desc' } }], + }; + + const actionRequests = await esClient + .search(actionsSearchQuery, { ignore: [404] }) + .catch(catchAndWrapError); + + const total = (actionRequests.hits?.total as SearchTotalHits)?.value; + + logger.debug( + `Searching for action requests found a total of [${total}] records using search query:\n${stringify( + actionsSearchQuery, + 15 + )}` + ); + + return { + data: (actionRequests?.hits?.hits ?? []).map((esHit) => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return esHit._source!; + }), + size, + from, + total, + }; +}; + +/** @private */ +const getActionTypeFilter = (actionType: string): QueryDslBoolQuery => { + return actionType === 'manual' + ? { + must_not: { + exists: { + field: 'data.alert_id', + }, + }, + } + : actionType === 'automated' + ? { + filter: { + exists: { + field: 'data.alert_id', + }, + }, + } + : {}; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.test.ts index 6c366142adfb9..5f84107a85135 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.test.ts @@ -24,116 +24,94 @@ describe('fetchActionResponses()', () => { it('should return results', async () => { await expect(fetchActionResponses({ esClient: esClientMock })).resolves.toEqual({ - data: [ + endpointResponses: [ { - _id: 'ef278144-d8b9-45c6-9c3c-484c86b57d0b', - _index: '.fleet-actions-results', - _score: 1, - _source: { - '@timestamp': '2022-04-30T16:08:47.449Z', - action_data: { - command: 'get-file', - comment: '', - }, - action_id: '123', - agent_id: 'agent-a', - completed_at: '2022-04-30T10:53:59.449Z', - error: '', - started_at: '2022-04-30T12:56:00.449Z', + '@timestamp': '2022-04-30T16:08:47.449Z', + action_data: { + command: 'get-file', + comment: '', }, - sort: ['abc'], + action_id: '123', + agent_id: 'agent-a', + completed_at: '2022-04-30T10:53:59.449Z', + error: '', + started_at: '2022-04-30T12:56:00.449Z', }, { - _id: 'ef278144-d8b9-45c6-9c3c-484c86b57d0b', - _index: '.ds-.logs-endpoint.action.responses-some_namespace-something', - _score: 1, - _source: { - '@timestamp': '2022-04-30T16:08:47.449Z', - EndpointActions: { - action_id: '123', - completed_at: '2022-04-30T10:53:59.449Z', - data: { - command: 'get-file', - comment: '', - output: { - content: { - code: 'ra_get-file_success_done', - contents: [ - { - file_name: 'bad_file.txt', - path: '/some/path/bad_file.txt', - sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f', - size: 1234, - type: 'file', - }, - ], - zip_size: 123, - }, - type: 'json', + '@timestamp': '2022-04-30T16:08:47.449Z', + EndpointActions: { + action_id: '123', + completed_at: '2022-04-30T10:53:59.449Z', + data: { + command: 'get-file', + comment: '', + output: { + content: { + code: 'ra_get-file_success_done', + contents: [ + { + file_name: 'bad_file.txt', + path: '/some/path/bad_file.txt', + sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f', + size: 1234, + type: 'file', + }, + ], + zip_size: 123, }, + type: 'json', }, - started_at: '2022-04-30T12:56:00.449Z', - }, - agent: { - id: 'agent-a', }, + started_at: '2022-04-30T12:56:00.449Z', + }, + agent: { + id: 'agent-a', }, - sort: ['abc'], }, + ], + fleetResponses: [ { - _id: 'ef278144-d8b9-45c6-9c3c-484c86b57d0b', - _index: '.fleet-actions-results', - _score: 1, - _source: { - '@timestamp': '2022-04-30T16:08:47.449Z', - action_data: { - command: 'get-file', - comment: '', - }, - action_id: '123', - agent_id: 'agent-a', - completed_at: '2022-04-30T10:53:59.449Z', - error: '', - started_at: '2022-04-30T12:56:00.449Z', + '@timestamp': '2022-04-30T16:08:47.449Z', + action_data: { + command: 'get-file', + comment: '', }, - sort: ['abc'], + action_id: '123', + agent_id: 'agent-a', + completed_at: '2022-04-30T10:53:59.449Z', + error: '', + started_at: '2022-04-30T12:56:00.449Z', }, { - _id: 'ef278144-d8b9-45c6-9c3c-484c86b57d0b', - _index: '.ds-.logs-endpoint.action.responses-some_namespace-something', - _score: 1, - _source: { - '@timestamp': '2022-04-30T16:08:47.449Z', - EndpointActions: { - action_id: '123', - completed_at: '2022-04-30T10:53:59.449Z', - data: { - command: 'get-file', - comment: '', - output: { - content: { - code: 'ra_get-file_success_done', - contents: [ - { - file_name: 'bad_file.txt', - path: '/some/path/bad_file.txt', - sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f', - size: 1234, - type: 'file', - }, - ], - zip_size: 123, - }, - type: 'json', + '@timestamp': '2022-04-30T16:08:47.449Z', + EndpointActions: { + action_id: '123', + completed_at: '2022-04-30T10:53:59.449Z', + data: { + command: 'get-file', + comment: '', + output: { + content: { + code: 'ra_get-file_success_done', + contents: [ + { + file_name: 'bad_file.txt', + path: '/some/path/bad_file.txt', + sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f', + size: 1234, + type: 'file', + }, + ], + zip_size: 123, }, + type: 'json', }, - started_at: '2022-04-30T12:56:00.449Z', - }, - agent: { - id: 'agent-a', }, + started_at: '2022-04-30T12:56:00.449Z', + }, + agent: { + id: 'agent-a', }, - sort: ['abc'], }, ], }); @@ -142,7 +120,10 @@ describe('fetchActionResponses()', () => { it('should return empty array with no responses exist', async () => { applyActionListEsSearchMock(esClientMock, undefined, BaseDataGenerator.toEsSearchResponse([])); - await expect(fetchActionResponses({ esClient: esClientMock })).resolves.toEqual({ data: [] }); + await expect(fetchActionResponses({ esClient: esClientMock })).resolves.toEqual({ + endpointResponses: [], + fleetResponses: [], + }); }); it('should query both fleet and endpoint indexes', async () => { @@ -156,14 +137,14 @@ describe('fetchActionResponses()', () => { }; expect(esClientMock.search).toHaveBeenCalledWith( - { index: AGENT_ACTIONS_RESULTS_INDEX, size: ACTIONS_SEARCH_PAGE_SIZE, body: expectedQuery }, + { index: AGENT_ACTIONS_RESULTS_INDEX, size: ACTIONS_SEARCH_PAGE_SIZE, ...expectedQuery }, { ignore: [404] } ); expect(esClientMock.search).toHaveBeenCalledWith( { index: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, size: ACTIONS_SEARCH_PAGE_SIZE, - body: expectedQuery, + ...expectedQuery, }, { ignore: [404] } ); @@ -176,13 +157,13 @@ describe('fetchActionResponses()', () => { }; expect(esClientMock.search).toHaveBeenCalledWith( - expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, body: expectedQuery }), + expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, ...expectedQuery }), { ignore: [404] } ); expect(esClientMock.search).toHaveBeenCalledWith( expect.objectContaining({ index: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, - body: expectedQuery, + ...expectedQuery, }), { ignore: [404] } ); @@ -195,13 +176,13 @@ describe('fetchActionResponses()', () => { }; expect(esClientMock.search).toHaveBeenCalledWith( - expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, body: expectedQuery }), + expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, ...expectedQuery }), { ignore: [404] } ); expect(esClientMock.search).toHaveBeenCalledWith( expect.objectContaining({ index: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, - body: expectedQuery, + ...expectedQuery, }), { ignore: [404] } ); @@ -222,13 +203,13 @@ describe('fetchActionResponses()', () => { }; expect(esClientMock.search).toHaveBeenCalledWith( - expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, body: expectedQuery }), + expect.objectContaining({ index: AGENT_ACTIONS_RESULTS_INDEX, ...expectedQuery }), { ignore: [404] } ); expect(esClientMock.search).toHaveBeenCalledWith( expect.objectContaining({ index: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, - body: expectedQuery, + ...expectedQuery, }), { ignore: [404] } ); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.ts index eb49c6c67216e..54396ecb9e86b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/fetch_action_responses.ts @@ -17,18 +17,6 @@ import { ACTIONS_SEARCH_PAGE_SIZE } from '../constants'; import { catchAndWrapError } from '../../../utils'; import { ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN } from '../../../../../common/endpoint/constants'; -interface FetchActionResponsesOptions { - esClient: ElasticsearchClient; - /** List of specific action ids to filter for */ - actionIds?: string[]; - /** List of specific agent ids to filter for */ - agentIds?: string[]; -} - -interface FetchActionResponsesResult { - data: Array>; -} - /** @private */ const buildSearchQuery = ( actionIds: string[] = [], @@ -47,53 +35,39 @@ const buildSearchQuery = ( return query; }; +interface FetchActionResponsesOptions { + esClient: ElasticsearchClient; + /** List of specific action ids to filter for */ + actionIds?: string[]; + /** List of specific agent ids to filter for */ + agentIds?: string[]; +} + +export interface FetchActionResponsesResult< + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TResponseMeta extends {} = {} +> { + /** Response (aka: the `ack`) sent to the fleet index */ + fleetResponses: EndpointActionResponse[]; + /** Responses sent by Endpoint directly to the endpoint index */ + endpointResponses: Array>; +} + /** * Fetch Response Action responses from both the Endpoint and the Fleet indexes */ -export const fetchActionResponses = async ({ - esClient, - actionIds = [], - agentIds = [], -}: FetchActionResponsesOptions): Promise => { - const query = buildSearchQuery(actionIds, agentIds); - - // TODO:PT refactor this method to use new `fetchFleetActionResponses()` and `fetchEndpointActionResponses()` - - // Get the Action Response(s) from both the Fleet action response index and the Endpoint - // action response index. - // We query both indexes separately in order to ensure they are both queried - example if the - // Fleet actions responses index does not exist yet, ES would generate a `404` and would - // never actually query the Endpoint Actions index. With support for 3rd party response - // actions, we need to ensure that both indexes are queried. +export const fetchActionResponses = async < + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TResponseMeta extends {} = {} +>( + options: FetchActionResponsesOptions +): Promise> => { const [fleetResponses, endpointResponses] = await Promise.all([ - // Responses in Fleet index - esClient - .search( - { - index: AGENT_ACTIONS_RESULTS_INDEX, - size: ACTIONS_SEARCH_PAGE_SIZE, - body: { query }, - }, - { ignore: [404] } - ) - .catch(catchAndWrapError), - - // Responses in Endpoint index - esClient - .search( - { - index: ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN, - size: ACTIONS_SEARCH_PAGE_SIZE, - body: { query }, - }, - { ignore: [404] } - ) - .catch(catchAndWrapError), + fetchFleetActionResponses(options), + fetchEndpointActionResponses(options), ]); - return { - data: [...(fleetResponses?.hits?.hits ?? []), ...(endpointResponses?.hits?.hits ?? [])], - }; + return { fleetResponses, endpointResponses }; }; /** @@ -123,7 +97,7 @@ export const fetchEndpointActionResponses = async < ) .catch(catchAndWrapError); - return searchResponse.hits.hits.map((esHit) => { + return (searchResponse?.hits?.hits ?? []).map((esHit) => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion return esHit._source!; }); @@ -151,7 +125,7 @@ export const fetchFleetActionResponses = async ({ ) .catch(catchAndWrapError); - return searchResponse.hits.hits.map((esHit) => { + return (searchResponse?.hits?.hits ?? []).map((esHit) => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion return esHit._source!; }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.test.ts index a2e69696b557c..59e99c91a1afd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.test.ts @@ -30,12 +30,11 @@ import type { EndpointActivityLogActionResponse, LogsEndpointAction, LogsEndpointActionResponse, - EndpointActionResponseDataOutput, } from '../../../../../common/endpoint/types'; import { v4 as uuidv4 } from 'uuid'; import type { Results } from '../../../routes/actions/mocks'; import { mockAuditLogSearchResult } from '../../../routes/actions/mocks'; -import { ActivityLogItemTypes } from '../../../../../common/endpoint/types'; +import type { FetchActionResponsesResult } from '../..'; describe('When using Actions service utilities', () => { let fleetActionGenerator: FleetActionGenerator; @@ -138,7 +137,7 @@ describe('When using Actions service utilities', () => { agents: [], }) ), - [] + { fleetResponses: [], endpointResponses: [] } ) ).toEqual({ ...NOT_COMPLETED_OUTPUT, @@ -154,7 +153,7 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [] + { fleetResponses: [], endpointResponses: [] } ) ).toEqual(NOT_COMPLETED_OUTPUT); }); @@ -167,28 +166,19 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [ - fleetActionGenerator.generateActivityLogActionResponse({ - item: { data: { action_id: '123' } }, - }), - ] + { + fleetResponses: [ + fleetActionGenerator.generateResponse({ + action_id: '123', + }), + ], + endpointResponses: [], + } ) ).toEqual(NOT_COMPLETED_OUTPUT); }); it('should show complete as `true` with completion date if Endpoint Response received', () => { - const endpointResponse = endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': COMPLETED_AT, - agent: { id: '123' }, - EndpointActions: { - completed_at: COMPLETED_AT, - data: { output: { type: 'json', content: { code: 'aaa' } } }, - }, - }, - }, - }); expect( getActionCompletionInfo( mapToNormalizedActionRequest( @@ -196,7 +186,19 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [endpointResponse] + { + fleetResponses: [], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + '@timestamp': COMPLETED_AT, + agent: { id: '123' }, + EndpointActions: { + completed_at: COMPLETED_AT, + data: { output: { type: 'json', content: { code: 'aaa' } } }, + }, + }), + ], + } ) ).toEqual({ isCompleted: true, @@ -224,25 +226,6 @@ describe('When using Actions service utilities', () => { it('should return action outputs (if any) per agent id', () => { const processes = endpointActionGenerator.randomResponseActionProcesses(3); - const endpointResponse = endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': COMPLETED_AT, - agent: { id: '123' }, - EndpointActions: { - completed_at: COMPLETED_AT, - data: { - output: { - type: 'json', - content: { - entries: processes, - }, - }, - }, - }, - }, - }, - }); expect( getActionCompletionInfo( mapToNormalizedActionRequest( @@ -250,7 +233,26 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [endpointResponse] + { + fleetResponses: [], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + '@timestamp': COMPLETED_AT, + agent: { id: '123' }, + EndpointActions: { + completed_at: COMPLETED_AT, + data: { + output: { + type: 'json', + content: { + entries: processes, + }, + }, + }, + }, + }), + ], + } ) ).toEqual({ isCompleted: true, @@ -277,30 +279,26 @@ describe('When using Actions service utilities', () => { }); describe('and action failed', () => { - let fleetResponseAtError: ActivityLogActionResponse; - let endpointResponseAtError: EndpointActivityLogActionResponse; + let fleetResponseAtError: EndpointActionResponse; + let endpointResponseAtError: LogsEndpointActionResponse; beforeEach(() => { const actionId = uuidv4(); - fleetResponseAtError = fleetActionGenerator.generateActivityLogActionResponse({ - item: { - data: { agent_id: '123', action_id: actionId, error: 'agent failed to deliver' }, - }, + fleetResponseAtError = fleetActionGenerator.generateResponse({ + agent_id: '123', + action_id: actionId, + error: 'agent failed to deliver', }); - endpointResponseAtError = endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': '2022-05-18T13:03:54.756Z', - agent: { id: '123' }, - error: { - message: 'endpoint failed to apply', - }, - EndpointActions: { - action_id: actionId, - completed_at: '2022-05-18T13:03:54.756Z', - }, - }, + endpointResponseAtError = endpointActionGenerator.generateResponse({ + '@timestamp': '2022-05-18T13:03:54.756Z', + agent: { id: '123' }, + error: { + message: 'endpoint failed to apply', + }, + EndpointActions: { + action_id: actionId, + completed_at: '2022-05-18T13:03:54.756Z', }, }); }); @@ -313,17 +311,17 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [endpointResponseAtError] + { fleetResponses: [], endpointResponses: [endpointResponseAtError] } ) ).toEqual({ - completedAt: endpointResponseAtError.item.data['@timestamp'], + completedAt: endpointResponseAtError['@timestamp'], errors: ['Endpoint action response error: endpoint failed to apply'], isCompleted: true, wasSuccessful: false, outputs: expect.anything(), agentState: { '123': { - completedAt: endpointResponseAtError.item.data['@timestamp'], + completedAt: endpointResponseAtError['@timestamp'], errors: ['Endpoint action response error: endpoint failed to apply'], isCompleted: true, wasSuccessful: false, @@ -340,17 +338,17 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [fleetResponseAtError] + { fleetResponses: [fleetResponseAtError], endpointResponses: [] } ) ).toEqual({ - completedAt: fleetResponseAtError.item.data.completed_at, + completedAt: fleetResponseAtError.completed_at, errors: ['Fleet action response error: agent failed to deliver'], isCompleted: true, wasSuccessful: false, outputs: {}, agentState: { '123': { - completedAt: fleetResponseAtError.item.data.completed_at, + completedAt: fleetResponseAtError.completed_at, errors: ['Fleet action response error: agent failed to deliver'], isCompleted: true, wasSuccessful: false, @@ -367,10 +365,10 @@ describe('When using Actions service utilities', () => { agents: ['123'], }) ), - [fleetResponseAtError, endpointResponseAtError] + { fleetResponses: [fleetResponseAtError], endpointResponses: [endpointResponseAtError] } ) ).toEqual({ - completedAt: endpointResponseAtError.item.data['@timestamp'], + completedAt: endpointResponseAtError['@timestamp'], errors: [ 'Endpoint action response error: endpoint failed to apply', 'Fleet action response error: agent failed to deliver', @@ -380,7 +378,7 @@ describe('When using Actions service utilities', () => { outputs: expect.anything(), agentState: { '123': { - completedAt: endpointResponseAtError.item.data['@timestamp'], + completedAt: endpointResponseAtError['@timestamp'], errors: [ 'Endpoint action response error: endpoint failed to apply', 'Fleet action response error: agent failed to deliver', @@ -396,66 +394,63 @@ describe('When using Actions service utilities', () => { describe('with multiple agent ids', () => { let agentIds: string[]; let actionId: string; - let action123Responses: Array< - | ActivityLogActionResponse - | EndpointActivityLogActionResponse - >; - let action456Responses: Array< - | ActivityLogActionResponse - | EndpointActivityLogActionResponse - >; - let action789Responses: Array< - | ActivityLogActionResponse - | EndpointActivityLogActionResponse - >; + let action123Responses: FetchActionResponsesResult; + let action456Responses: FetchActionResponsesResult; + let action789Responses: FetchActionResponsesResult; beforeEach(() => { agentIds = ['123', '456', '789']; actionId = uuidv4(); - action123Responses = [ - fleetActionGenerator.generateActivityLogActionResponse({ - item: { data: { agent_id: '123', error: '', action_id: actionId } }, - }), - endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': '2022-01-05T19:27:23.816Z', - agent: { id: '123' }, - EndpointActions: { action_id: actionId, completed_at: '2022-01-05T19:27:23.816Z' }, - }, - }, - }), - ]; + action123Responses = { + fleetResponses: [ + fleetActionGenerator.generateResponse({ + agent_id: '123', + error: '', + action_id: actionId, + }), + ], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + '@timestamp': '2022-01-05T19:27:23.816Z', + agent: { id: '123' }, + EndpointActions: { action_id: actionId, completed_at: '2022-01-05T19:27:23.816Z' }, + }), + ], + }; - action456Responses = [ - fleetActionGenerator.generateActivityLogActionResponse({ - item: { data: { action_id: actionId, agent_id: '456', error: '' } }, - }), - endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': COMPLETED_AT, - agent: { id: '456' }, - EndpointActions: { action_id: actionId, completed_at: COMPLETED_AT }, - }, - }, - }), - ]; + action456Responses = { + fleetResponses: [ + fleetActionGenerator.generateResponse({ + action_id: actionId, + agent_id: '456', + error: '', + }), + ], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + '@timestamp': COMPLETED_AT, + agent: { id: '456' }, + EndpointActions: { action_id: actionId, completed_at: COMPLETED_AT }, + }), + ], + }; - action789Responses = [ - fleetActionGenerator.generateActivityLogActionResponse({ - item: { data: { action_id: actionId, agent_id: '789', error: '' } }, - }), - endpointActionGenerator.generateActivityLogActionResponse({ - item: { - data: { - '@timestamp': '2022-03-05T19:27:23.816Z', - agent: { id: '789' }, - EndpointActions: { action_id: actionId, completed_at: '2022-03-05T19:27:23.816Z' }, - }, - }, - }), - ]; + action789Responses = { + fleetResponses: [ + fleetActionGenerator.generateResponse({ + action_id: actionId, + agent_id: '789', + error: '', + }), + ], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + '@timestamp': '2022-03-05T19:27:23.816Z', + agent: { id: '789' }, + EndpointActions: { action_id: actionId, completed_at: '2022-03-05T19:27:23.816Z' }, + }), + ], + }; }); it('should show complete as `false` if no responses', () => { @@ -466,7 +461,7 @@ describe('When using Actions service utilities', () => { agents: agentIds, }) ), - [] + { fleetResponses: [], endpointResponses: [] } ) ).toEqual({ ...NOT_COMPLETED_OUTPUT, @@ -496,14 +491,18 @@ describe('When using Actions service utilities', () => { agents: agentIds, }) ), - [ - ...action123Responses, - - // Action id: 456 === Not complete (only fleet response) - action456Responses[0], - - ...action789Responses, - ] + { + fleetResponses: [ + ...action123Responses.fleetResponses, + ...action456Responses.fleetResponses, + ...action789Responses.fleetResponses, + ], + endpointResponses: [ + ...action123Responses.endpointResponses, + ...action789Responses.endpointResponses, + // Action id: 456 === Not complete (only fleet response) + ], + } ) ).toEqual({ ...NOT_COMPLETED_OUTPUT, @@ -539,7 +538,18 @@ describe('When using Actions service utilities', () => { agents: agentIds, }) ), - [...action123Responses, ...action456Responses, ...action789Responses] + { + fleetResponses: [ + ...action123Responses.fleetResponses, + ...action456Responses.fleetResponses, + ...action789Responses.fleetResponses, + ], + endpointResponses: [ + ...action123Responses.endpointResponses, + ...action456Responses.endpointResponses, + ...action789Responses.endpointResponses, + ], + } ) ).toEqual({ isCompleted: true, @@ -571,8 +581,8 @@ describe('When using Actions service utilities', () => { }); it('should complete as `true` if one agent only received a fleet response with error on it', () => { - action456Responses[0].item.data.error = 'something is no good'; - action456Responses[0].item.data['@timestamp'] = '2022-05-06T12:50:19.747Z'; + action456Responses.fleetResponses[0].error = 'something is no good'; + action456Responses.fleetResponses[0]['@timestamp'] = '2022-05-06T12:50:19.747Z'; expect( getActionCompletionInfo( @@ -581,14 +591,18 @@ describe('When using Actions service utilities', () => { agents: agentIds, }) ), - [ - ...action123Responses, - - // Action id: 456 === is complete with only a fleet response that has `error` - action456Responses[0], - - ...action789Responses, - ] + { + fleetResponses: [ + ...action123Responses.fleetResponses, + ...action456Responses.fleetResponses, + ...action789Responses.fleetResponses, + ], + endpointResponses: [ + ...action123Responses.endpointResponses, + ...action789Responses.endpointResponses, + // Action id: 456 === is complete with only a fleet response that has `error` + ], + } ) ).toEqual({ completedAt: '2022-05-06T12:50:19.747Z', @@ -604,7 +618,7 @@ describe('When using Actions service utilities', () => { wasSuccessful: true, }, '456': { - completedAt: action456Responses[0].item.data['@timestamp'], + completedAt: action456Responses.fleetResponses[0]['@timestamp'], errors: ['Fleet action response error: something is no good'], isCompleted: true, wasSuccessful: false, @@ -621,18 +635,15 @@ describe('When using Actions service utilities', () => { it('should include output for agents for which the action was complete', () => { // Add output to the completed actions - ( - action123Responses[1] as EndpointActivityLogActionResponse - ).item.data.EndpointActions.data.output = { + + action123Responses.endpointResponses[0].EndpointActions.data.output = { type: 'json', content: { code: 'bar', }, }; - ( - action789Responses[1] as EndpointActivityLogActionResponse - ).item.data.EndpointActions.data.output = { + action789Responses.endpointResponses[0].EndpointActions.data.output = { type: 'text', // @ts-expect-error need to fix ActionResponseOutput type content: 'some endpoint output data', @@ -645,14 +656,18 @@ describe('When using Actions service utilities', () => { agents: agentIds, }) ), - [ - ...action123Responses, - - // Action id: 456 === Not complete (only fleet response) - action456Responses[0], - - ...action789Responses, - ] + { + fleetResponses: [ + ...action123Responses.fleetResponses, + ...action456Responses.fleetResponses, + ...action789Responses.fleetResponses, + ], + endpointResponses: [ + ...action123Responses.endpointResponses, + ...action789Responses.endpointResponses, + // Action id: 456 === Not complete (only fleet response) + ], + } ) ).toEqual({ ...NOT_COMPLETED_OUTPUT, @@ -975,7 +990,7 @@ describe('When using Actions service utilities', () => { describe('#createActionDetailsRecord()', () => { let actionRequest: NormalizedActionRequest; - let actionResponses: Array; + let actionResponses: FetchActionResponsesResult; let agentHostInfo: Record; beforeEach(() => { @@ -993,31 +1008,22 @@ describe('When using Actions service utilities', () => { hosts: {}, }; - actionResponses = [ - { - type: ActivityLogItemTypes.FLEET_RESPONSE, - item: { - id: actionRequest.id, - data: fleetActionGenerator.generateResponse({ + actionResponses = { + fleetResponses: [ + fleetActionGenerator.generateResponse({ + action_id: actionRequest.id, + agent_id: actionRequest.agents[0], + }), + ], + endpointResponses: [ + endpointActionGenerator.generateResponse({ + agent: { id: actionRequest.agents }, + EndpointActions: { action_id: actionRequest.id, - agent_id: actionRequest.agents[0], - }), - }, - }, - - { - type: ActivityLogItemTypes.RESPONSE, - item: { - id: actionRequest.id, - data: endpointActionGenerator.generateResponse({ - agent: { id: actionRequest.agents }, - EndpointActions: { - action_id: actionRequest.id, - }, - }), - }, - }, - ]; + }, + }), + ], + }; agentHostInfo = { [actionRequest.agents[0]]: 'host-a', diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.ts index 1c64d1f59a062..cd7680d3bd3ac 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/utils/utils.ts @@ -10,6 +10,8 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { EcsError } from '@elastic/ecs'; import moment from 'moment/moment'; import { i18n } from '@kbn/i18n'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { FetchActionResponsesResult } from '../..'; import type { ResponseActionAgentType, ResponseActionsApiCommandNames, @@ -52,8 +54,8 @@ export const isLogsEndpointAction = ( * @param item */ export const isLogsEndpointActionResponse = ( - item: EndpointActionResponse | LogsEndpointActionResponse -): item is LogsEndpointActionResponse => { + item: EndpointActionResponse | LogsEndpointActionResponse +): item is LogsEndpointActionResponse => { return 'EndpointActions' in item && 'agent' in item; }; @@ -124,20 +126,49 @@ export const mapToNormalizedActionRequest = ( }; }; +/** + * Maps the list of fetch action responses (from both Endpoint and Fleet indexes) to a Map + * whose keys are the action ID and value is the set of responses for that action id + * @param actionResponses + */ +export const mapResponsesByActionId = ( + actionResponses: FetchActionResponsesResult +): { [actionId: string]: FetchActionResponsesResult } => { + return [...actionResponses.endpointResponses, ...actionResponses.fleetResponses].reduce<{ + [actionId: string]: FetchActionResponsesResult; + }>((acc, response) => { + const actionId = getActionIdFromActionResponse(response); + + if (!acc[actionId]) { + acc[actionId] = { + endpointResponses: [], + fleetResponses: [], + }; + } + + if (isLogsEndpointActionResponse(response)) { + acc[actionId].endpointResponses.push(response); + } else { + acc[actionId].fleetResponses.push(response); + } + + return acc; + }, {}); +}; + type ActionCompletionInfo = Pick< Required, 'isCompleted' | 'completedAt' | 'wasSuccessful' | 'errors' | 'outputs' | 'agentState' >; export const getActionCompletionInfo = < - TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TResponseMeta extends {} = {} >( /** The normalized action request */ action: NormalizedActionRequest, - /** List of action Log responses received for the action */ - actionResponses: Array< - ActivityLogActionResponse | EndpointActivityLogActionResponse - > + /** List of responses (from both Endpoint and Fleet) */ + actionResponses: FetchActionResponsesResult ): ActionCompletionInfo => { const agentIds = action.agents; const completedInfo: ActionCompletionInfo = { @@ -169,7 +200,7 @@ export const getActionCompletionInfo = < completedAt: undefined, }; - // Store the outputs and agent state for any agent that has received a response + // Store the outputs and agent state for any agent that sent a response if (agentResponses) { completedInfo.agentState[agentId].isCompleted = agentResponses.isCompleted; completedInfo.agentState[agentId].wasSuccessful = agentResponses.wasSuccessful; @@ -178,10 +209,10 @@ export const getActionCompletionInfo = < if ( agentResponses.endpointResponse && - agentResponses.endpointResponse.item.data.EndpointActions.data.output + agentResponses.endpointResponse.EndpointActions.data.output ) { completedInfo.outputs[agentId] = - agentResponses.endpointResponse.item.data.EndpointActions.data.output; + agentResponses.endpointResponse.EndpointActions.data.output; } } } @@ -255,14 +286,15 @@ export const getActionStatus = ({ }; interface NormalizedAgentActionResponse< - TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TResponseMeta extends {} = {} > { isCompleted: boolean; completedAt: undefined | string; wasSuccessful: boolean; errors: undefined | string[]; - fleetResponse: undefined | ActivityLogActionResponse; - endpointResponse: undefined | EndpointActivityLogActionResponse; + fleetResponse: undefined | EndpointActionResponse; + endpointResponse: undefined | LogsEndpointActionResponse; } type ActionResponseByAgentId = Record; @@ -273,20 +305,19 @@ type ActionResponseByAgentId = Record; * @param actionResponses */ const mapActionResponsesByAgentId = < - TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput + TOutputContent extends EndpointActionResponseDataOutput = EndpointActionResponseDataOutput, + TResponseMeta extends {} = {} >( - actionResponses: Array< - ActivityLogActionResponse | EndpointActivityLogActionResponse - > + actionResponses: FetchActionResponsesResult ): ActionResponseByAgentId => { - const response: ActionResponseByAgentId = {}; - - for (const actionResponse of actionResponses) { - const agentId = getAgentIdFromActionResponse(actionResponse); - let thisAgentActionResponses = response[agentId]; - - if (!thisAgentActionResponses) { - response[agentId] = { + const response = [ + ...actionResponses.endpointResponses, + ...actionResponses.fleetResponses, + ].reduce((acc, actionResponseRecord) => { + const agentId = getAgentIdFromActionResponse(actionResponseRecord); + + if (!acc[agentId]) { + acc[agentId] = { isCompleted: false, completedAt: undefined, wasSuccessful: false, @@ -294,62 +325,61 @@ const mapActionResponsesByAgentId = < fleetResponse: undefined, endpointResponse: undefined, }; - - thisAgentActionResponses = response[agentId]; } - if (actionResponse.type === 'fleetResponse') { - thisAgentActionResponses.fleetResponse = actionResponse; + if (isLogsEndpointActionResponse(actionResponseRecord)) { + acc[agentId].endpointResponse = actionResponseRecord; } else { - thisAgentActionResponses.endpointResponse = actionResponse; + acc[agentId].fleetResponse = actionResponseRecord; } - thisAgentActionResponses.isCompleted = + return acc; + }, {}); + + for (const agentNormalizedResponse of Object.values(response)) { + agentNormalizedResponse.isCompleted = // Action is complete if an Endpoint Action Response was received - Boolean(thisAgentActionResponses.endpointResponse) || + Boolean(agentNormalizedResponse.endpointResponse) || // OR: // If we did not have an endpoint response and the Fleet response has `error`, then // action is complete. Elastic Agent was unable to deliver the action request to the // endpoint, so we are unlikely to ever receive an Endpoint Response. - Boolean(thisAgentActionResponses.fleetResponse?.item.data.error); + Boolean(agentNormalizedResponse.fleetResponse?.error); // When completed, calculate additional properties about the action - if (thisAgentActionResponses.isCompleted) { - if (thisAgentActionResponses.endpointResponse) { - thisAgentActionResponses.completedAt = - thisAgentActionResponses.endpointResponse?.item.data['@timestamp']; - thisAgentActionResponses.wasSuccessful = true; + if (agentNormalizedResponse.isCompleted) { + if (agentNormalizedResponse.endpointResponse) { + agentNormalizedResponse.completedAt = + agentNormalizedResponse.endpointResponse?.['@timestamp']; + agentNormalizedResponse.wasSuccessful = true; } else if ( // Check if perhaps the Fleet action response returned an error, in which case, the Fleet Agent // failed to deliver the Action to the Endpoint. If that's the case, we are not going to get // a Response from endpoint, thus mark the Action as completed and use the Fleet Message's // timestamp for the complete data/time. - thisAgentActionResponses.fleetResponse && - thisAgentActionResponses.fleetResponse.item.data.error + agentNormalizedResponse.fleetResponse && + agentNormalizedResponse.fleetResponse.error ) { - thisAgentActionResponses.isCompleted = true; - thisAgentActionResponses.completedAt = - thisAgentActionResponses.fleetResponse.item.data['@timestamp']; + agentNormalizedResponse.isCompleted = true; + agentNormalizedResponse.completedAt = agentNormalizedResponse.fleetResponse['@timestamp']; } const errors: NormalizedAgentActionResponse['errors'] = []; // only one of the errors should be in there - if (thisAgentActionResponses.endpointResponse?.item.data.error?.message) { + if (agentNormalizedResponse.endpointResponse?.error?.message) { errors.push( - `Endpoint action response error: ${thisAgentActionResponses.endpointResponse.item.data.error.message}` + `Endpoint action response error: ${agentNormalizedResponse.endpointResponse.error.message}` ); } - if (thisAgentActionResponses.fleetResponse?.item.data.error) { - errors.push( - `Fleet action response error: ${thisAgentActionResponses.fleetResponse?.item.data.error}` - ); + if (agentNormalizedResponse.fleetResponse?.error) { + errors.push(`Fleet action response error: ${agentNormalizedResponse.fleetResponse.error}`); } if (errors.length) { - thisAgentActionResponses.wasSuccessful = false; - thisAgentActionResponses.errors = errors; + agentNormalizedResponse.wasSuccessful = false; + agentNormalizedResponse.errors = errors; } } } @@ -361,18 +391,30 @@ const mapActionResponsesByAgentId = < * Given an Action response, this will return the Agent ID for that action response. * @param actionResponse */ -const getAgentIdFromActionResponse = ( - actionResponse: - | ActivityLogActionResponse - | EndpointActivityLogActionResponse +export const getAgentIdFromActionResponse = ( + actionResponse: EndpointActionResponse | LogsEndpointActionResponse ): string => { - const responseData = actionResponse.item.data; + if (isLogsEndpointActionResponse(actionResponse)) { + return Array.isArray(actionResponse.agent.id) + ? actionResponse.agent.id[0] + : actionResponse.agent.id; + } + + return actionResponse.agent_id; +}; - if (isLogsEndpointActionResponse(responseData)) { - return Array.isArray(responseData.agent.id) ? responseData.agent.id[0] : responseData.agent.id; +/** + * Given an Action response from either Endpoint or Fleet, utility will return its action id + * @param actionResponse + */ +export const getActionIdFromActionResponse = ( + actionResponse: EndpointActionResponse | LogsEndpointActionResponse +): string => { + if (isLogsEndpointActionResponse(actionResponse)) { + return actionResponse.EndpointActions.action_id; } - return responseData.agent_id; + return actionResponse.action_id; }; // common helpers used by old and new log API @@ -382,8 +424,8 @@ export const getDateFilters = ({ }: { startDate?: string; endDate?: string; -}) => { - const dateFilters = []; +}): QueryDslQueryContainer[] => { + const dateFilters: QueryDslQueryContainer[] = []; if (startDate) { dateFilters.push({ range: { '@timestamp': { gte: startDate } } }); } @@ -423,41 +465,6 @@ export const getUniqueLogData = (activityLogEntries: ActivityLogEntry[]): Activi return [...nonEndpointActionsDocs, ...onlyEndpointActionsDocWithoutFleetActions]; }; -export const hasAckInResponse = (response: EndpointActionResponse): boolean => { - return response.action_response?.endpoint?.ack ?? false; -}; - -// return TRUE if for given action_id/agent_id -// there is no doc in .logs-endpoint.action.response-default -export const hasNoEndpointResponse = ({ - action, - agentId, - indexedActionIds, -}: { - action: EndpointAction; - agentId: string; - indexedActionIds: string[]; -}): boolean => { - return action.agents.includes(agentId) && !indexedActionIds.includes(action.action_id); -}; - -// return TRUE if for given action_id/agent_id -// there is no doc in .fleet-actions-results -export const hasNoFleetResponse = ({ - action, - agentId, - agentResponses, -}: { - action: EndpointAction; - agentId: string; - agentResponses: EndpointActionResponse[]; -}): boolean => { - return ( - action.agents.includes(agentId) && - !agentResponses.map((e) => e.action_id).includes(action.action_id) - ); -}; - const matchesDsNamePattern = ({ dataStreamName, index, @@ -554,7 +561,7 @@ export const getAgentHostNamesWithIds = async ({ export const createActionDetailsRecord = ( actionRequest: NormalizedActionRequest, - actionResponses: Array, + actionResponses: FetchActionResponsesResult, agentHostInfo: Record ): T => { const { isCompleted, completedAt, wasSuccessful, errors, outputs, agentState } = diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.test.ts b/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.test.ts deleted file mode 100644 index fce8721f78391..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.test.ts +++ /dev/null @@ -1,601 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ScopedClusterClientMock } from '@kbn/core/server/mocks'; -import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; -import { - applyActionListEsSearchMock, - createActionRequestsEsSearchResultsMock, -} from '../services/actions/mocks'; -import { getActions } from './action_list_helpers'; - -describe('action helpers', () => { - let mockScopedEsClient: ScopedClusterClientMock; - - beforeEach(() => { - mockScopedEsClient = elasticsearchServiceMock.createScopedClusterClient(); - }); - - describe('#getActions', () => { - it('should call with base filter query correctly when no other filter options provided', async () => { - const esClient = mockScopedEsClient.asInternalUser; - applyActionListEsSearchMock(esClient); - await getActions({ esClient, size: 10, from: 0 }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [], - }, - }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 0, - index: '.logs-endpoint.actions-default', - size: 10, - }, - { - ignore: [404], - meta: true, - } - ); - }); - - it('should query with additional filter options provided', async () => { - const esClient = mockScopedEsClient.asInternalUser; - - applyActionListEsSearchMock(esClient); - await getActions({ - esClient, - size: 20, - from: 5, - startDate: 'now-10d', - agentTypes: ['endpoint'], - elasticAgentIds: ['agent-123', 'agent-456'], - endDate: 'now', - commands: ['isolate', 'unisolate', 'get-file'], - userIds: ['*elastic*', '*kibana*'], - }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-10d', - }, - }, - }, - { - range: { - '@timestamp': { - lte: 'now', - }, - }, - }, - { - terms: { - 'data.command': ['isolate', 'unisolate', 'get-file'], - }, - }, - { - terms: { - input_type: ['endpoint'], - }, - }, - { - terms: { - agents: ['agent-123', 'agent-456'], - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*elastic*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*kibana*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 5, - index: '.logs-endpoint.actions-default', - size: 20, - }, - { - ignore: [404], - meta: true, - } - ); - }); - - it('should search with exact usernames when given', async () => { - const esClient = mockScopedEsClient.asInternalUser; - - applyActionListEsSearchMock(esClient); - await getActions({ - esClient, - size: 10, - from: 1, - startDate: 'now-1d', - endDate: 'now', - userIds: ['elastic', 'kibana'], - }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-1d', - }, - }, - }, - { - range: { - '@timestamp': { - lte: 'now', - }, - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - should: [ - { - match: { - user_id: 'elastic', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - match: { - user_id: 'kibana', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 1, - index: '.logs-endpoint.actions-default', - size: 10, - }, - { - ignore: [404], - meta: true, - } - ); - }); - - it('should return expected output', async () => { - const esClient = mockScopedEsClient.asInternalUser; - const actionRequests = createActionRequestsEsSearchResultsMock(); - - applyActionListEsSearchMock(esClient, actionRequests); - - const actions = await getActions({ - esClient, - size: 10, - from: 0, - elasticAgentIds: ['agent-a'], - }); - - expect(actions.actionIds).toEqual(['123']); - expect(actions.actionRequests?.body?.hits?.hits[0]._source?.agent.id).toEqual('agent-a'); - }); - - describe('action `Types` filter', () => { - it('should correctly query with multiple action `types` filter options provided', async () => { - const esClient = mockScopedEsClient.asInternalUser; - - applyActionListEsSearchMock(esClient); - await getActions({ - esClient, - size: 20, - from: 5, - startDate: 'now-10d', - elasticAgentIds: ['agent-123', 'agent-456'], - endDate: 'now', - types: ['manual', 'automated'], - userIds: ['*elastic*', '*kibana*'], - }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-10d', - }, - }, - }, - { - range: { - '@timestamp': { - lte: 'now', - }, - }, - }, - - { - terms: { - agents: ['agent-123', 'agent-456'], - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*elastic*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*kibana*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 5, - index: '.logs-endpoint.actions-default', - size: 20, - }, - { - ignore: [404], - meta: true, - } - ); - }); - - it('should correctly query with single `manual` action `types` filter options provided', async () => { - const esClient = mockScopedEsClient.asInternalUser; - - applyActionListEsSearchMock(esClient); - await getActions({ - esClient, - size: 20, - from: 5, - startDate: 'now-10d', - elasticAgentIds: ['agent-123', 'agent-456'], - endDate: 'now', - types: ['manual'], - userIds: ['*elastic*', '*kibana*'], - }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-10d', - }, - }, - }, - { - range: { - '@timestamp': { - lte: 'now', - }, - }, - }, - - { - terms: { - agents: ['agent-123', 'agent-456'], - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*elastic*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*kibana*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - must_not: { - exists: { - field: 'data.alert_id', - }, - }, - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 5, - index: '.logs-endpoint.actions-default', - size: 20, - }, - { - ignore: [404], - meta: true, - } - ); - }); - - it('should correctly query with single `automated` action `types` filter options provided', async () => { - const esClient = mockScopedEsClient.asInternalUser; - - applyActionListEsSearchMock(esClient); - await getActions({ - esClient, - size: 20, - from: 5, - startDate: 'now-10d', - elasticAgentIds: ['agent-123', 'agent-456'], - endDate: 'now', - types: ['automated'], - userIds: ['*elastic*', '*kibana*'], - }); - - expect(esClient.search).toHaveBeenCalledWith( - { - body: { - query: { - bool: { - must: [ - { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-10d', - }, - }, - }, - { - range: { - '@timestamp': { - lte: 'now', - }, - }, - }, - - { - terms: { - agents: ['agent-123', 'agent-456'], - }, - }, - ], - }, - }, - { - bool: { - should: [ - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*elastic*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - { - bool: { - should: [ - { - query_string: { - fields: ['user_id'], - query: '*kibana*', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - filter: { - exists: { - field: 'data.alert_id', - }, - }, - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - from: 5, - index: '.logs-endpoint.actions-default', - size: 20, - }, - { - ignore: [404], - meta: true, - } - ); - }); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.ts b/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.ts deleted file mode 100644 index 056e01326f9a9..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SearchRequest } from '@kbn/data-plugin/public'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { TransportResult } from '@elastic/elasticsearch'; -import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; - -import { ENDPOINT_ACTIONS_INDEX } from '../../../common/endpoint/constants'; -import type { LogsEndpointAction } from '../../../common/endpoint/types'; -import { getDateFilters } from '../services/actions/utils'; -import { catchAndWrapError } from './wrap_errors'; -import type { GetActionDetailsListParam } from '../services/actions/action_list'; - -const queryOptions = Object.freeze({ - ignore: [404], -}); - -const getActionTypeFilter = (actionType: string): SearchRequest => { - return actionType === 'manual' - ? { - must_not: { - exists: { - field: 'data.alert_id', - }, - }, - } - : actionType === 'automated' - ? { - filter: { - exists: { - field: 'data.alert_id', - }, - }, - } - : {}; -}; -export const getActions = async ({ - agentTypes, - commands, - elasticAgentIds, - esClient, - endDate, - from, - size, - startDate, - userIds, - unExpiredOnly, - types, -}: Omit): Promise<{ - actionIds: string[]; - actionRequests: TransportResult, unknown>; -}> => { - const additionalFilters = []; - - if (commands?.length) { - additionalFilters.push({ - terms: { - 'data.command': commands, - }, - }); - } - - if (agentTypes?.length) { - additionalFilters.push({ terms: { input_type: agentTypes } }); - } - - if (elasticAgentIds?.length) { - additionalFilters.push({ terms: { agents: elasticAgentIds } }); - } - - if (unExpiredOnly) { - additionalFilters.push({ range: { expiration: { gte: 'now' } } }); - } - - const dateFilters = getDateFilters({ startDate, endDate }); - - const actionsFilters = [...dateFilters, ...additionalFilters]; - - const must: SearchRequest = [ - { - bool: { - filter: actionsFilters, - }, - }, - ]; - - if (userIds?.length) { - const userIdsKql = userIds.map((userId) => `user_id:${userId}`).join(' or '); - const mustClause = toElasticsearchQuery(fromKueryExpression(userIdsKql)); - must.push(mustClause); - } - - const isNotASingleActionType = !types || (types && types.length > 1); - - const actionsSearchQuery: SearchRequest = { - index: ENDPOINT_ACTIONS_INDEX, - size, - from, - body: { - query: { - bool: { - must, - ...(isNotASingleActionType ? {} : getActionTypeFilter(types[0])), - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - }; - - const actionRequests: TransportResult< - estypes.SearchResponse, - unknown - > = await esClient - .search(actionsSearchQuery, { - ...queryOptions, - meta: true, - }) - .catch(catchAndWrapError); - - // only one type of actions - const actionIds = actionRequests?.body?.hits?.hits.map((e) => { - return (e._source as LogsEndpointAction).EndpointActions.action_id; - }); - - return { actionIds, actionRequests }; -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/index.ts b/x-pack/plugins/security_solution/server/endpoint/utils/index.ts index ac9c0fdf8ab41..c8035fa0db3f5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/utils/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/utils/index.ts @@ -7,6 +7,4 @@ export * from './fleet_agent_status_to_endpoint_host_status'; export * from './wrap_errors'; -export * from './audit_log_helpers'; -export * from './action_list_helpers'; export * from './yes_no_data_stream'; diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.test.ts b/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.test.ts index 4fde3243d950c..8f196526bd116 100644 --- a/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.test.ts @@ -7,44 +7,7 @@ import type { ElasticsearchClientMock } from '@kbn/core/server/mocks'; import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { - doLogsEndpointActionDsExists, - doesLogsEndpointActionsIndexExist, -} from './yes_no_data_stream'; - -describe('Accurately answers if index template for data stream exists', () => { - let esClient: ElasticsearchClientMock; - - beforeEach(() => { - esClient = elasticsearchServiceMock.createScopedClusterClient().asInternalUser; - }); - - it('Returns FALSE for a non-existent data stream index template', async () => { - esClient.indices.existsIndexTemplate.mockResponseImplementation(() => ({ - body: false, - statusCode: 404, - })); - const doesItExist = await doLogsEndpointActionDsExists({ - esClient, - logger: loggingSystemMock.create().get('host-isolation'), - dataStreamName: '.test-stream.name', - }); - expect(doesItExist).toBeFalsy(); - }); - - it('Returns TRUE for an existing index', async () => { - esClient.indices.existsIndexTemplate.mockResponseImplementation(() => ({ - body: true, - statusCode: 200, - })); - const doesItExist = await doLogsEndpointActionDsExists({ - esClient, - logger: loggingSystemMock.create().get('host-isolation'), - dataStreamName: '.test-stream.name', - }); - expect(doesItExist).toBeTruthy(); - }); -}); +import { doesLogsEndpointActionsIndexExist } from './yes_no_data_stream'; describe('Accurately answers if index exists', () => { let esClient: ElasticsearchClientMock; diff --git a/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.ts b/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.ts index 3996a1a7ddee2..f4c61fefc54ad 100644 --- a/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.ts +++ b/x-pack/plugins/security_solution/server/endpoint/utils/yes_no_data_stream.ts @@ -7,33 +7,6 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -export const doLogsEndpointActionDsExists = async ({ - esClient, - logger, - dataStreamName, -}: { - esClient: ElasticsearchClient; - logger: Logger; - dataStreamName: string; -}): Promise => { - try { - const doesIndexTemplateExist = await esClient.indices.existsIndexTemplate( - { - name: dataStreamName, - }, - { meta: true } - ); - return doesIndexTemplateExist.statusCode !== 404; - } catch (error) { - const errorType = error?.type ?? ''; - if (errorType !== 'resource_not_found_exception') { - logger.error(error); - throw error; - } - return false; - } -}; - export const doesLogsEndpointActionsIndexExist = async ({ esClient, logger, diff --git a/x-pack/plugins/security_solution/server/utils/custom_http_request_error.ts b/x-pack/plugins/security_solution/server/utils/custom_http_request_error.ts index 9ce131a9182ef..a3a1387f2e700 100644 --- a/x-pack/plugins/security_solution/server/utils/custom_http_request_error.ts +++ b/x-pack/plugins/security_solution/server/utils/custom_http_request_error.ts @@ -13,5 +13,9 @@ export class CustomHttpRequestError extends Error { super(message); // For debugging - capture name of subclasses this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } } }