From 2f876bce2e4476ea08793515153a5b2256e9164f Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Tue, 15 Mar 2022 16:36:30 +0100 Subject: [PATCH] [Fleet] Add agent incoming data endpoint and presentational component (#127177) * [Fleet] Create endpoint to check if agent has incoming data * Document new endpoint * Improvements to component * Update endpoint schema * Remove button for now * Address review comments * Add dynamic button functionality * Add option to hide button and improve query Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/fleet/common/constants/routes.ts | 1 + .../plugins/fleet/common/openapi/bundled.json | 45 +++++++++++ .../plugins/fleet/common/openapi/bundled.yaml | 28 +++++++ .../fleet/common/openapi/entrypoint.yaml | 2 + .../openapi/paths/agent_status@data.yaml | 27 +++++++ .../plugins/fleet/common/services/routes.ts | 1 + .../fleet/common/types/rest_spec/agent.ts | 13 ++++ .../confirm_incoming_data.tsx | 75 ++++++++++++++++++ x-pack/plugins/fleet/public/hooks/index.ts | 1 + .../hooks/use_get_agent_incoming_data.tsx | 78 +++++++++++++++++++ .../fleet/public/hooks/use_request/agents.ts | 9 +++ x-pack/plugins/fleet/public/types/index.ts | 3 + .../fleet/server/routes/agent/handlers.ts | 27 +++++++ .../fleet/server/routes/agent/index.ts | 13 ++++ .../fleet/server/services/agents/status.ts | 77 ++++++++++++++++++ .../fleet/server/types/rest_spec/agent.ts | 6 ++ 16 files changed, 406 insertions(+) create mode 100644 x-pack/plugins/fleet/common/openapi/paths/agent_status@data.yaml create mode 100644 x-pack/plugins/fleet/public/components/agent_enrollment_flyout/confirm_incoming_data.tsx create mode 100644 x-pack/plugins/fleet/public/hooks/use_get_agent_incoming_data.tsx diff --git a/x-pack/plugins/fleet/common/constants/routes.ts b/x-pack/plugins/fleet/common/constants/routes.ts index 61df3b78282ca..f2d170a35b0a8 100644 --- a/x-pack/plugins/fleet/common/constants/routes.ts +++ b/x-pack/plugins/fleet/common/constants/routes.ts @@ -105,6 +105,7 @@ export const AGENT_API_ROUTES = { REASSIGN_PATTERN: `${API_ROOT}/agents/{agentId}/reassign`, BULK_REASSIGN_PATTERN: `${API_ROOT}/agents/bulk_reassign`, STATUS_PATTERN: `${API_ROOT}/agent_status`, + DATA_PATTERN: `${API_ROOT}/agent_status/data`, // deprecated since 8.0 STATUS_PATTERN_DEPRECATED: `${API_ROOT}/agent-status`, UPGRADE_PATTERN: `${API_ROOT}/agents/{agentId}/upgrade`, diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index f16ea62aa1b0c..87f75b86da485 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -1119,6 +1119,51 @@ ] } }, + "/agent_status/data": { + "get": { + "summary": "Agents - Get incoming data", + "tags": [], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "data": { + "type": "boolean" + } + } + } + } + } + } + } + } + } + } + }, + "operationId": "get-agent-data", + "parameters": [ + { + "schema": { + "type": "array" + }, + "name": "agentsIds", + "in": "query", + "required": true + } + ] + } + }, "/agents": { "get": { "summary": "Agents - List", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 28040efa5f41b..dd5569779ee19 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -686,6 +686,34 @@ paths: name: kuery in: query required: false + /agent_status/data: + get: + summary: Agents - Get incoming data + tags: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + additionalProperties: + type: object + properties: + data: + type: boolean + operationId: get-agent-data + parameters: + - schema: + type: array + name: agentsId + in: query + required: true /agents: get: summary: Agents - List diff --git a/x-pack/plugins/fleet/common/openapi/entrypoint.yaml b/x-pack/plugins/fleet/common/openapi/entrypoint.yaml index 4d51beeb40a2e..e54b270b94b7c 100644 --- a/x-pack/plugins/fleet/common/openapi/entrypoint.yaml +++ b/x-pack/plugins/fleet/common/openapi/entrypoint.yaml @@ -49,6 +49,8 @@ paths: $ref: paths/agent_status_deprecated.yaml /agent_status: $ref: paths/agent_status.yaml + /agent_status/data: + $ref: paths/agent_status@data.yaml /agents: $ref: paths/agents.yaml /agents/bulk_upgrade: diff --git a/x-pack/plugins/fleet/common/openapi/paths/agent_status@data.yaml b/x-pack/plugins/fleet/common/openapi/paths/agent_status@data.yaml new file mode 100644 index 0000000000000..cde8139383065 --- /dev/null +++ b/x-pack/plugins/fleet/common/openapi/paths/agent_status@data.yaml @@ -0,0 +1,27 @@ +get: + summary: Agents - Get incoming data + tags: [] + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + additionalProperties: + type: object + properties: + data: + type: boolean + operationId: get-agent-data + parameters: + - schema: + type: array + name: agentsId + in: query + required: true diff --git a/x-pack/plugins/fleet/common/services/routes.ts b/x-pack/plugins/fleet/common/services/routes.ts index 25c0680645f92..95218ded84a80 100644 --- a/x-pack/plugins/fleet/common/services/routes.ts +++ b/x-pack/plugins/fleet/common/services/routes.ts @@ -164,6 +164,7 @@ export const agentRouteService = { getBulkUpgradePath: () => AGENT_API_ROUTES.BULK_UPGRADE_PATTERN, getListPath: () => AGENT_API_ROUTES.LIST_PATTERN, getStatusPath: () => AGENT_API_ROUTES.STATUS_PATTERN, + getIncomingDataPath: () => AGENT_API_ROUTES.DATA_PATTERN, getCreateActionPath: (agentId: string) => AGENT_API_ROUTES.ACTIONS_PATTERN.replace('{agentId}', agentId), }; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/agent.ts b/x-pack/plugins/fleet/common/types/rest_spec/agent.ts index 5e091b9c543f2..40570bc599053 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/agent.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/agent.ts @@ -161,3 +161,16 @@ export interface GetAgentStatusResponse { updating: number; }; } + +export interface GetAgentIncomingDataRequest { + query: { + agentsIds: string[]; + }; +} + +export interface IncomingDataList { + [key: string]: { data: boolean }; +} +export interface GetAgentIncomingDataResponse { + items: IncomingDataList[]; +} diff --git a/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/confirm_incoming_data.tsx b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/confirm_incoming_data.tsx new file mode 100644 index 0000000000000..85817fa9850a0 --- /dev/null +++ b/x-pack/plugins/fleet/public/components/agent_enrollment_flyout/confirm_incoming_data.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiCallOut, EuiText, EuiSpacer, EuiButton } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import type { InstalledIntegrationPolicy } from '../../hooks'; +import { useGetAgentIncomingData } from '../../hooks'; +interface Props { + agentsIds: string[]; + installedPolicy?: InstalledIntegrationPolicy; +} + +export const ConfirmIncomingData: React.FunctionComponent = ({ + agentsIds, + installedPolicy, +}) => { + const { enrolledAgents, numAgentsWithData, isLoading, linkButton } = useGetAgentIncomingData( + agentsIds, + installedPolicy + ); + + return ( + <> + {isLoading ? ( + + {i18n.translate('xpack.fleet.confirmIncomingData.loading', { + defaultMessage: + 'It may take a few minutes for data to arrive in Elasticsearch. If the system is not generating data, it may help to generate some to ensure data is being collected correctly. If you’re having trouble, see our troubleshooting guide. You may close this dialog and check later by viewing our integration assets.', + })} + + ) : ( + <> + + + + {i18n.translate('xpack.fleet.confirmIncomingData.subtitle', { + defaultMessage: 'Your agent is enrolled successfully and your data is received.', + })} + + + )} + + + {installedPolicy && ( + + {linkButton.text} + + )} + + ); +}; diff --git a/x-pack/plugins/fleet/public/hooks/index.ts b/x-pack/plugins/fleet/public/hooks/index.ts index 5c995131396b4..c5dcdd78b9bb9 100644 --- a/x-pack/plugins/fleet/public/hooks/index.ts +++ b/x-pack/plugins/fleet/public/hooks/index.ts @@ -27,3 +27,4 @@ export * from './use_platform'; export * from './use_agent_policy_refresh'; export * from './use_package_installations'; export * from './use_agent_enrollment_flyout_data'; +export * from './use_get_agent_incoming_data'; diff --git a/x-pack/plugins/fleet/public/hooks/use_get_agent_incoming_data.tsx b/x-pack/plugins/fleet/public/hooks/use_get_agent_incoming_data.tsx new file mode 100644 index 0000000000000..a14dbd30aef45 --- /dev/null +++ b/x-pack/plugins/fleet/public/hooks/use_get_agent_incoming_data.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useEffect, useState, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; + +import type { IncomingDataList } from '../../common/types/rest_spec/agent'; + +import { sendGetAgentIncomingData, useLink } from './index'; + +export interface InstalledIntegrationPolicy { + name: string; + version: string; +} + +export const useGetAgentIncomingData = ( + agentsIds: string[], + installedPolicy?: InstalledIntegrationPolicy +) => { + const [isLoading, setIsLoading] = useState(true); + const [incomingData, setIncomingData] = useState([]); + + useEffect(() => { + const getIncomingData = async () => { + const { data } = await sendGetAgentIncomingData({ agentsIds }); + if (data?.items) { + setIncomingData(data?.items); + setIsLoading(false); + } + }; + if (agentsIds) { + getIncomingData(); + } + }, [agentsIds]); + + const enrolledAgents = useMemo(() => incomingData.length, [incomingData.length]); + const numAgentsWithData = useMemo( + () => + incomingData.reduce((acc, curr) => { + const agentData = Object.values(curr)[0]; + return !!agentData.data ? acc + 1 : acc; + }, 0), + [incomingData] + ); + const { getAbsolutePath, getHref } = useLink(); + + let href; + let text; + if (!installedPolicy) { + href = ''; + text = ''; + } + + if (installedPolicy?.name === 'apm') { + href = getAbsolutePath('/app/home#/tutorial/apm'); + text = i18n.translate('xpack.fleet.confirmIncomingData.installApmAgentButtonText', { + defaultMessage: 'Install APM Agent', + }); + } else { + href = getHref('integration_details_assets', { + pkgkey: `${installedPolicy?.name}-${installedPolicy?.version}`, + }); + text = i18n.translate('xpack.fleet.confirmIncomingData.viewDataAssetsButtonText', { + defaultMessage: 'View assets', + }); + } + const linkButton = { href, text }; + + return { + enrolledAgents, + numAgentsWithData, + isLoading, + linkButton, + }; +}; diff --git a/x-pack/plugins/fleet/public/hooks/use_request/agents.ts b/x-pack/plugins/fleet/public/hooks/use_request/agents.ts index 0fbe59f3f48ee..9bfba13052c35 100644 --- a/x-pack/plugins/fleet/public/hooks/use_request/agents.ts +++ b/x-pack/plugins/fleet/public/hooks/use_request/agents.ts @@ -21,6 +21,8 @@ import type { GetAgentsResponse, GetAgentStatusRequest, GetAgentStatusResponse, + GetAgentIncomingDataRequest, + GetAgentIncomingDataResponse, PostAgentUpgradeRequest, PostBulkAgentUpgradeRequest, PostAgentUpgradeResponse, @@ -68,6 +70,13 @@ export function useGetAgentStatus(query: GetAgentStatusRequest['query'], options ...options, }); } +export function sendGetAgentIncomingData(query: GetAgentIncomingDataRequest['query']) { + return sendRequest({ + method: 'get', + path: agentRouteService.getIncomingDataPath(), + query, + }); +} export function sendGetAgentStatus( query: GetAgentStatusRequest['query'], diff --git a/x-pack/plugins/fleet/public/types/index.ts b/x-pack/plugins/fleet/public/types/index.ts index 0a5d39c4a1ce9..fc29f046aac04 100644 --- a/x-pack/plugins/fleet/public/types/index.ts +++ b/x-pack/plugins/fleet/public/types/index.ts @@ -61,6 +61,9 @@ export type { PostBulkAgentUpgradeResponse, GetAgentStatusRequest, GetAgentStatusResponse, + GetAgentIncomingDataRequest, + IncomingDataList, + GetAgentIncomingDataResponse, PutAgentReassignRequest, PutAgentReassignResponse, PostBulkAgentReassignRequest, diff --git a/x-pack/plugins/fleet/server/routes/agent/handlers.ts b/x-pack/plugins/fleet/server/routes/agent/handlers.ts index 1473b508f1354..eb2fde7b5894b 100644 --- a/x-pack/plugins/fleet/server/routes/agent/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent/handlers.ts @@ -21,6 +21,7 @@ import type { UpdateAgentRequestSchema, DeleteAgentRequestSchema, GetAgentStatusRequestSchema, + GetAgentDataRequestSchema, PutAgentReassignRequestSchema, PostBulkAgentReassignRequestSchema, } from '../../types'; @@ -218,3 +219,29 @@ export const getAgentStatusForAgentPolicyHandler: RequestHandler< return defaultIngestErrorHandler({ error, response }); } }; + +export const getAgentDataHandler: RequestHandler< + undefined, + TypeOf +> = async (context, request, response) => { + const esClient = context.core.elasticsearch.client.asCurrentUser; + try { + let items; + + if (isStringArray(request.query.agentsIds)) { + items = await AgentService.getIncomingDataByAgentsId(esClient, request.query.agentsIds); + } else { + items = await AgentService.getIncomingDataByAgentsId(esClient, [request.query.agentsIds]); + } + + const body = { items }; + + return response.ok({ body }); + } catch (error) { + return defaultIngestErrorHandler({ error, response }); + } +}; + +function isStringArray(arr: unknown | string[]): arr is string[] { + return Array.isArray(arr) && arr.every((p) => typeof p === 'string'); +} diff --git a/x-pack/plugins/fleet/server/routes/agent/index.ts b/x-pack/plugins/fleet/server/routes/agent/index.ts index 83f0c17c82c37..535bb780abe57 100644 --- a/x-pack/plugins/fleet/server/routes/agent/index.ts +++ b/x-pack/plugins/fleet/server/routes/agent/index.ts @@ -14,6 +14,7 @@ import { PostAgentUnenrollRequestSchema, PostBulkAgentUnenrollRequestSchema, GetAgentStatusRequestSchema, + GetAgentDataRequestSchema, PostNewAgentActionRequestSchema, PutAgentReassignRequestSchema, PostBulkAgentReassignRequestSchema, @@ -32,6 +33,7 @@ import { getAgentStatusForAgentPolicyHandler, putAgentsReassignHandler, postBulkAgentsReassignHandler, + getAgentDataHandler, } from './handlers'; import { postNewAgentActionHandlerBuilder } from './actions_handlers'; import { postAgentUnenrollHandler, postBulkAgentsUnenrollHandler } from './unenroll_handler'; @@ -141,6 +143,17 @@ export const registerAPIRoutes = (router: FleetAuthzRouter, config: FleetConfigT }, getAgentStatusForAgentPolicyHandler ); + // Agent data + router.get( + { + path: AGENT_API_ROUTES.DATA_PATTERN, + validate: GetAgentDataRequestSchema, + fleetAuthz: { + fleet: { all: true }, + }, + }, + getAgentDataHandler + ); // upgrade agent router.post( diff --git a/x-pack/plugins/fleet/server/services/agents/status.ts b/x-pack/plugins/fleet/server/services/agents/status.ts index 5c5176ec41352..828a3b1622cef 100644 --- a/x-pack/plugins/fleet/server/services/agents/status.ts +++ b/x-pack/plugins/fleet/server/services/agents/status.ts @@ -14,9 +14,12 @@ import { fromKueryExpression } from '@kbn/es-query'; import { AGENTS_PREFIX } from '../../constants'; import type { AgentStatus } from '../../types'; import { AgentStatusKueryHelper } from '../../../common/services'; +import { FleetUnauthorizedError } from '../../errors'; import { getAgentById, getAgentsByKuery, removeSOAttributes } from './crud'; +const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*,synthetics-*-*'; + export async function getAgentStatusById( esClient: ElasticsearchClient, agentId: string @@ -92,3 +95,77 @@ export async function getAgentStatusForAgentPolicy( events: 0, }; } +export async function getIncomingDataByAgentsId( + esClient: ElasticsearchClient, + agentsIds: string[] +) { + try { + const { has_all_requested: hasAllPrivileges } = await esClient.security.hasPrivileges({ + body: { + index: [ + { + names: [DATA_STREAM_INDEX_PATTERN], + privileges: ['read'], + }, + ], + }, + }); + if (!hasAllPrivileges) { + throw new FleetUnauthorizedError('Missing permissions to read data streams indices'); + } + + const searchResult = await esClient.search({ + index: DATA_STREAM_INDEX_PATTERN, + allow_partial_search_results: true, + _source: false, + timeout: '5s', + size: 0, + body: { + query: { + bool: { + filter: [ + { + terms: { + 'agent.id': agentsIds, + }, + }, + { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + ], + }, + }, + aggs: { + agent_ids: { + terms: { + field: 'agent.id', + size: 10, + }, + }, + }, + }, + }); + + if (!searchResult.aggregations?.agent_ids) { + return agentsIds.map((id) => { + return { [id]: { data: false } }; + }); + } + + // @ts-expect-error aggregation type is not specified + const agentIdsWithData: string[] = searchResult.aggregations.agent_ids.buckets.map( + (bucket: any) => bucket.key as string + ); + + return agentsIds.map((id) => + agentIdsWithData.includes(id) ? { [id]: { data: true } } : { [id]: { data: false } } + ); + } catch (e) { + throw new Error(e); + } +} diff --git a/x-pack/plugins/fleet/server/types/rest_spec/agent.ts b/x-pack/plugins/fleet/server/types/rest_spec/agent.ts index 3f66c8159562a..ea11637119dc9 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/agent.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/agent.ts @@ -111,3 +111,9 @@ export const GetAgentStatusRequestSchema = { kuery: schema.maybe(schema.string()), }), }; + +export const GetAgentDataRequestSchema = { + query: schema.object({ + agentsIds: schema.oneOf([schema.arrayOf(schema.string()), schema.string()]), + }), +};