From 12401b2216ed3635117de8cfd38f09061f3ff9a0 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 23 May 2023 15:04:54 -0400 Subject: [PATCH] [Logs Onboarding] Adds install shipper step for custom logs (#157802) Closes #154937 This PR includes the steps to install standalone elastic agent + reporting the status from bash script back to kibana. ![Screenshot 2023-05-15 at 5 02 29 PM](https://github.com/elastic/kibana/assets/1967266/62484fb3-e02f-410d-aa7a-86bcc4dc0b03) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Yngrid Coello Co-authored-by: Yngrid Coello --- .../current_mappings.json | 12 + .../group2/check_registered_types.test.ts | 1 + .../group3/dot_kibana_split.test.ts | 1 + .../group3/type_registrations.test.ts | 1 + .../public/assets/standalone_agent_setup.sh | 28 ++ .../wizard/install_elastic_agent.tsx | 346 ++++++++++-------- .../public/context/create_wizard_context.tsx | 7 +- .../server/lib/get_authentication_api_key.ts | 23 ++ .../observability_onboarding/server/plugin.ts | 3 + .../custom_logs/create_shipper_api_key.ts | 32 ++ ...d_latest_observability_onboarding_state.ts | 36 ++ .../server/routes/custom_logs/get_es_hosts.ts | 39 ++ .../routes/custom_logs/get_kibana_url.ts | 14 + .../get_observability_onboarding_state.ts | 36 ++ .../server/routes/custom_logs/route.ts | 257 +++++++++++++ .../save_observability_onboarding_state.ts | 40 ++ .../routes/elastic_agent/generate_yml.ts | 59 +++ .../server/routes/elastic_agent/route.ts | 51 +++ .../server/routes/index.ts | 4 + .../server/routes/register_routes.ts | 7 + .../server/routes/types.ts | 6 +- .../observability_onboarding_status.ts | 39 ++ .../observability_onboarding/server/types.ts | 6 + .../observability_onboarding/tsconfig.json | 5 + 24 files changed, 896 insertions(+), 157 deletions(-) create mode 100644 x-pack/plugins/observability_onboarding/public/assets/standalone_agent_setup.sh create mode 100644 x-pack/plugins/observability_onboarding/server/lib/get_authentication_api_key.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/create_shipper_api_key.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_latest_observability_onboarding_state.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_es_hosts.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_kibana_url.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_observability_onboarding_state.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/custom_logs/save_observability_onboarding_state.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts create mode 100644 x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 2c4a574cb6b43..566ace3a9b543 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2097,6 +2097,18 @@ } } }, + "observability-onboarding-state": { + "properties": { + "state": { + "type": "object", + "dynamic": false + }, + "progress": { + "type": "object", + "dynamic": false + } + } + }, "ml-job": { "properties": { "job_id": { diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 90874fcc1699e..963a188f808bf 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -119,6 +119,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ml-module": "c88b6a012cfb7b7adb7629b1edeab6b83f1fd048", "ml-trained-model": "49a1685d79990ad05ea1d1d30e28456fe002f3b9", "monitoring-telemetry": "24f7393dfacb6c7b0f7ad7d242171a1c29feaa48", + "observability-onboarding-state": "9a55f01199158a68ea8a0123e99ff092cdcdb71c", "osquery-manager-usage-metric": "23a8f08a98dd0f58ab4e559daa35b06edc40ed4f", "osquery-pack": "edd84b2c59ef36214ece0676706da8f22175c660", "osquery-pack-asset": "18e08979d46ee7e5538f54c080aec4d8c58516ca", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts index a52512318fab3..e44db9f0c2201 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/dot_kibana_split.test.ts @@ -233,6 +233,7 @@ describe('split .kibana index into multiple system indices', () => { "ml-module", "ml-trained-model", "monitoring-telemetry", + "observability-onboarding-state", "osquery-manager-usage-metric", "osquery-pack", "osquery-pack-asset", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts index 490925a555c05..5d64d16eb1499 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts @@ -89,6 +89,7 @@ const previouslyRegisteredTypes = [ 'ml-module', 'ml-telemetry', 'monitoring-telemetry', + 'observability-onboarding-state', 'osquery-pack', 'osquery-pack-asset', 'osquery-saved-query', diff --git a/x-pack/plugins/observability_onboarding/public/assets/standalone_agent_setup.sh b/x-pack/plugins/observability_onboarding/public/assets/standalone_agent_setup.sh new file mode 100644 index 0000000000000..71a16c5f037c8 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/public/assets/standalone_agent_setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +API_KEY_ENCODED=$1 +API_ENDPOINT=$2 + +updateStepProgress() { + echo " GET $API_ENDPOINT/step/$1?status=$2" + curl --request GET \ + --url "$API_ENDPOINT/step/$1?status=$2" \ + --header "Authorization: ApiKey $API_KEY_ENCODED" \ + --header "Content-Type: application/json" \ + --header "kbn-xsrf: true" + echo "" +} + +echo "Downloading Elastic Agent" +# https://www.elastic.co/guide/en/fleet/8.7/install-standalone-elastic-agent.html +curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-8.7.1-linux-x86_64.tar.gz +updateStepProgress "ea-download" "success" +echo "Extracting Elastic Agent" +tar xzvf elastic-agent-8.7.1-linux-x86_64.tar.gz +updateStepProgress "ea-extract" "success" +echo "Installing Elastic Agent" +cd elastic-agent-8.7.1-linux-x86_64 +./elastic-agent install -f +updateStepProgress "ea-install" "success" +echo "Sending status to Kibana..." +updateStepProgress "ea-status" "active" diff --git a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx index 410cc084608a8..bf4363dea9c7f 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx @@ -5,20 +5,17 @@ * 2.0. */ -import React, { PropsWithChildren, useState } from 'react'; +import { Buffer } from 'buffer'; +import { flatten, zip } from 'lodash'; +import React, { useState } from 'react'; import { - EuiTitle, EuiText, EuiButton, - EuiFlexGroup, - EuiFlexItem, - EuiHorizontalRule, EuiSpacer, - EuiCard, - EuiIcon, - EuiIconProps, EuiButtonGroup, EuiCodeBlock, + EuiSteps, + EuiSkeletonRectangle, } from '@elastic/eui'; import { StepPanel, @@ -26,178 +23,221 @@ import { StepPanelFooter, } from '../../../shared/step_panel'; import { useWizard } from '.'; +import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; +type ElasticAgentPlatform = 'linux-tar' | 'macos' | 'windows'; export function InstallElasticAgent() { - const { goToStep, goBack, getState, setState } = useWizard(); + const { goToStep, goBack, getState, CurrentStep } = useWizard(); const wizardState = getState(); - const [elasticAgentPlatform, setElasticAgentPlatform] = useState( - wizardState.elasticAgentPlatform - ); - const [alternativeShippers, setAlternativeShippers] = useState( - wizardState.alternativeShippers - ); + const [elasticAgentPlatform, setElasticAgentPlatform] = + useState('linux-tar'); function onContinue() { - setState({ ...getState(), elasticAgentPlatform, alternativeShippers }); goToStep('collectLogs'); } - function createAlternativeShipperToggle( - type: NonNullable - ) { - return () => { - setAlternativeShippers({ - ...alternativeShippers, - [type]: !alternativeShippers[type], - }); - }; - } - function onBack() { goBack(); } - return ( - - Back - , - - Continue - , - ]} - /> + const { data: installShipperSetup, status: installShipperSetupStatus } = + useFetcher((callApi) => { + if (CurrentStep === InstallElasticAgent) { + return callApi( + 'POST /internal/observability_onboarding/custom_logs/install_shipper_setup', + { + params: { + body: { + name: wizardState.datasetName, + state: { + datasetName: wizardState.datasetName, + namespace: wizardState.namespace, + customConfigurations: wizardState.customConfigurations, + logFilePaths: wizardState.logFilePaths, + }, + }, + }, + } + ); + } + }, []); + + const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher( + (callApi) => { + if (CurrentStep === InstallElasticAgent && installShipperSetup) { + return callApi( + 'GET /api/observability_onboarding/elastic_agent/config', + { + headers: { + authorization: `ApiKey ${installShipperSetup.apiKeyEncoded}`, + }, + } + ); } - > + }, + [installShipperSetup?.apiKeyId, installShipperSetup?.apiKeyEncoded] + ); + + const apiKeyEncoded = installShipperSetup?.apiKeyEncoded; + + return ( +

- Select a platform and run the command to install, enroll, and start - the Elastic Agent. Do this for each host. For other platforms, see - our downloads page. Review host requirements and other installation - options. + Add Elastic Agent to your hosts to begin sending data to your + Elastic Cloud. Run standalone if you want to download and manage + each agent configuration file on your own, or enroll in Fleet, for + centralized management of all your agents through our Fleet managed + interface.

- + +

+ Select a platform and run the command to install in your + Terminal, enroll, and start the Elastic Agent. Do this for + each host. For other platforms, see our downloads page. + Review host requirements and other installation options. +

+ + + + setElasticAgentPlatform(id as typeof elasticAgentPlatform) + } + /> + + + + {getInstallShipperCommand({ + elasticAgentPlatform, + apiKeyEncoded, + apiEndpoint: installShipperSetup?.apiEndpoint, + scriptDownloadUrl: + installShipperSetup?.scriptDownloadUrl, + })} + + + + ), + }, + { + title: 'Configure the agent', + status: + yamlConfigStatus === FETCH_STATUS.LOADING + ? 'loading' + : 'incomplete', + children: ( + <> + +

+ Copy the config below to the elastic agent.yml on the host + where the Elastic Agent is installed. +

+ + + + + {yamlConfig} + + + + + Download config file + + + ), + }, ]} - type="single" - idSelected={elasticAgentPlatform} - onChange={(id: string) => - setElasticAgentPlatform(id as typeof elasticAgentPlatform) - } /> - - - {PLATFORM_COMMAND[elasticAgentPlatform]} - - - - - - - - - - - - - - - - - - - - - + + Back + , + + Continue + , + ]} + /> ); } -function LogsTypeSection({ - title, - description, - children, -}: PropsWithChildren<{ title: string; description: string }>) { - return ( - <> - -

{title}

- - - -

{description}

- - - {children} - - ); -} - -function OptionCard({ - title, - iconType, - onClick, - isSelected, +function getInstallShipperCommand({ + elasticAgentPlatform, + apiKeyEncoded = '$API_KEY', + apiEndpoint = '$API_ENDPOINT', + scriptDownloadUrl = '$SCRIPT_DOWNLOAD_URL', }: { - title: string; - iconType: EuiIconProps['type']; - onClick: () => void; - isSelected: boolean; + elasticAgentPlatform: ElasticAgentPlatform; + apiKeyEncoded: string | undefined; + apiEndpoint: string | undefined; + scriptDownloadUrl: string | undefined; }) { - return ( - } - title={title} - titleSize="xs" - paddingSize="m" - style={{ height: 56 }} - onClick={onClick} - hasBorder={true} - display={isSelected ? 'primary' : undefined} - /> - ); + const setupScriptFilename = 'standalone_agent_setup.sh'; + const PLATFORM_COMMAND: Record = { + 'linux-tar': oneLine` + curl ${scriptDownloadUrl} -o ${setupScriptFilename} && + sudo bash ${setupScriptFilename} ${apiKeyEncoded} ${apiEndpoint} + `, + macos: oneLine` + curl -O https://elastic.co/agent-setup.sh && + sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ== + `, + windows: oneLine` + curl -O https://elastic.co/agent-setup.sh && + sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ== + `, + }; + return PLATFORM_COMMAND[elasticAgentPlatform]; } -const PLATFORM_COMMAND = { - 'linux-tar': `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`, - macos: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`, - windows: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`, - deb: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`, - rpm: `curl -O https://elastic.co/agent-setup.sh && sudo bash agent-setup.sh -- service.name=my-service --url=https://elasticsearch:8220 --enrollment-token=SRSc2ozWUItWXNuWE5oZzdERFU6anJtY0FIzhSRGlzeTJYcUF5UklfUQ==`, -} as const; +function oneLine(parts: TemplateStringsArray, ...args: string[]) { + const str = flatten(zip(parts, args)).join(''); + return str.replace(/\s+/g, ' ').trim(); +} diff --git a/x-pack/plugins/observability_onboarding/public/context/create_wizard_context.tsx b/x-pack/plugins/observability_onboarding/public/context/create_wizard_context.tsx index 7e7937ebc7e14..140373e688eb0 100644 --- a/x-pack/plugins/observability_onboarding/public/context/create_wizard_context.tsx +++ b/x-pack/plugins/observability_onboarding/public/context/create_wizard_context.tsx @@ -14,7 +14,7 @@ import React, { useRef, } from 'react'; -interface WizardContext { +export interface WizardContext { CurrentStep: ComponentType; goToStep: (step: StepKey) => void; goBack: () => void; @@ -172,8 +172,9 @@ export function createWizardContext< } function useWizard() { - const { CurrentStep: _, ...rest } = useContext(context); - return rest; + // const { CurrentStep: _, ...rest } = useContext(context); + // return rest; + return useContext(context); } return { context, Provider, Step, useWizard }; diff --git a/x-pack/plugins/observability_onboarding/server/lib/get_authentication_api_key.ts b/x-pack/plugins/observability_onboarding/server/lib/get_authentication_api_key.ts new file mode 100644 index 0000000000000..6353487bf70df --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/lib/get_authentication_api_key.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { HTTPAuthorizationHeader } from '@kbn/security-plugin/server'; + +export const getAuthenticationAPIKey = (request: KibanaRequest) => { + const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request); + if (authorizationHeader && authorizationHeader.credentials) { + const apiKey = Buffer.from(authorizationHeader.credentials, 'base64') + .toString() + .split(':'); + return { + apiKeyId: apiKey[0], + apiKey: apiKey[1], + }; + } + throw new Error('Authorization header is missing'); +}; diff --git a/x-pack/plugins/observability_onboarding/server/plugin.ts b/x-pack/plugins/observability_onboarding/server/plugin.ts index 3045ad66c869e..c46e4d24d9fe9 100644 --- a/x-pack/plugins/observability_onboarding/server/plugin.ts +++ b/x-pack/plugins/observability_onboarding/server/plugin.ts @@ -23,6 +23,7 @@ import { ObservabilityOnboardingPluginStartDependencies, } from './types'; import { ObservabilityOnboardingConfig } from '.'; +import { observabilityOnboardingState } from './saved_objects/observability_onboarding_status'; export class ObservabilityOnboardingPlugin implements @@ -47,6 +48,8 @@ export class ObservabilityOnboardingPlugin ) { this.logger.debug('observability_onboarding: Setup'); + core.savedObjects.registerType(observabilityOnboardingState); + const resourcePlugins = mapValues(plugins, (value, key) => { return { setup: value, diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/create_shipper_api_key.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/create_shipper_api_key.ts new file mode 100644 index 0000000000000..0648d68be35c3 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/create_shipper_api_key.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; + +export function createShipperApiKey( + esClient: ElasticsearchClient, + name: string +) { + // Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent + return esClient.security.createApiKey({ + body: { + name: `standalone_agent_custom_logs_${name}`, + metadata: { application: 'logs' }, + role_descriptors: { + standalone_agent: { + cluster: ['monitor'], + indices: [ + { + names: ['logs-*-*', 'metrics-*-*'], + privileges: ['auto_configure', 'create_doc'], + }, + ], + }, + }, + }, + }); +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_latest_observability_onboarding_state.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_latest_observability_onboarding_state.ts new file mode 100644 index 0000000000000..fa379fc089c65 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_latest_observability_onboarding_state.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + ObservabilityOnboardingState, + SavedObservabilityOnboardingState, +} from '../../saved_objects/observability_onboarding_status'; + +export async function findLatestObservabilityOnboardingState({ + savedObjectsClient, +}: { + savedObjectsClient: SavedObjectsClientContract; +}): Promise { + const result = await savedObjectsClient.find({ + type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + page: 1, + perPage: 1, + sortField: `updated_at`, + sortOrder: 'desc', + }); + if (result.total === 0) { + return undefined; + } + const { id, updated_at: updatedAt, attributes } = result.saved_objects[0]; + return { + id, + updatedAt: updatedAt ? Date.parse(updatedAt) : 0, + ...attributes, + }; +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_es_hosts.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_es_hosts.ts new file mode 100644 index 0000000000000..f9d35da7d3534 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_es_hosts.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { CloudSetup } from '@kbn/cloud-plugin/server'; +import { decodeCloudId } from '@kbn/fleet-plugin/common'; + +const DEFAULT_ES_HOSTS = ['http://localhost:9200']; + +export function getESHosts({ + cloudSetup, + esClient, +}: { + cloudSetup: CloudSetup; + esClient: Client; +}): string[] { + if (cloudSetup.cloudId) { + const cloudUrl = decodeCloudId(cloudSetup.cloudId)?.elasticsearchUrl; + if (cloudUrl) { + return [cloudUrl]; + } + } + + const aliveConnections = esClient.connectionPool.connections.filter( + ({ status }) => status === 'alive' + ); + if (aliveConnections.length) { + return aliveConnections.map(({ url }) => { + const { protocol, host } = new URL(url); + return `${protocol}//${host}`; + }); + } + + return DEFAULT_ES_HOSTS; +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_kibana_url.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_kibana_url.ts new file mode 100644 index 0000000000000..95110f4707576 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_kibana_url.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart } from '@kbn/core/server'; + +export function getKibanaUrl({ http }: CoreStart, path = '') { + const basePath = http.basePath; + const { protocol, hostname, port } = http.getServerInfo(); + return `${protocol}://${hostname}:${port}${basePath.prepend(path)}`; +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_observability_onboarding_state.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_observability_onboarding_state.ts new file mode 100644 index 0000000000000..081431c17ce3c --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/get_observability_onboarding_state.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + ObservabilityOnboardingState, + SavedObservabilityOnboardingState, +} from '../../saved_objects/observability_onboarding_status'; + +export async function getObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, +}: { + savedObjectsClient: SavedObjectsClientContract; + apiKeyId: string; +}): Promise { + try { + const result = await savedObjectsClient.get( + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + apiKeyId + ); + const { id, updated_at: updatedAt, attributes } = result; + return { + id, + updatedAt: updatedAt ? Date.parse(updatedAt) : 0, + ...attributes, + }; + } catch (error) { + return undefined; + } +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts new file mode 100644 index 0000000000000..b99a670fea57e --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import Boom from '@hapi/boom'; +import type { Client } from '@elastic/elasticsearch'; +import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; +import { getESHosts } from './get_es_hosts'; +import { getKibanaUrl } from './get_kibana_url'; +import { createShipperApiKey } from './create_shipper_api_key'; +import { saveObservabilityOnboardingState } from './save_observability_onboarding_state'; +import { + ObservabilityOnboardingState, + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + SavedObservabilityOnboardingState, +} from '../../saved_objects/observability_onboarding_status'; +import { getObservabilityOnboardingState } from './get_observability_onboarding_state'; +import { findLatestObservabilityOnboardingState } from './find_latest_observability_onboarding_state'; +import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key'; + +const createApiKeyRoute = createObservabilityOnboardingServerRoute({ + endpoint: + 'POST /internal/observability_onboarding/custom_logs/install_shipper_setup', + options: { tags: [] }, + params: t.type({ + body: t.type({ + name: t.string, + state: t.record(t.string, t.unknown), + }), + }), + async handler(resources): Promise<{ + apiKeyId: string; + apiKeyEncoded: string; + apiEndpoint: string; + scriptDownloadUrl: string; + esHost: string; + }> { + const { + context, + params: { + body: { name, state }, + }, + core, + plugins, + request, + } = resources; + const coreStart = await core.start(); + const scriptDownloadUrl = getKibanaUrl( + coreStart, + '/plugins/observabilityOnboarding/assets/standalone_agent_setup.sh' + ); + const apiEndpoint = getKibanaUrl( + coreStart, + '/api/observability_onboarding/custom_logs' + ); + const { + elasticsearch: { client }, + } = await context.core; + const { id: apiKeyId, encoded: apiKeyEncoded } = await createShipperApiKey( + client.asCurrentUser, + name + ); + const [esHost] = getESHosts({ + cloudSetup: plugins.cloud.setup, + esClient: coreStart.elasticsearch.client.asInternalUser as Client, + }); + + const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); + + await saveObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, + observabilityOnboardingState: { state } as ObservabilityOnboardingState, + }); + + return { + apiKeyId, // key the status off this + apiKeyEncoded, + apiEndpoint, + scriptDownloadUrl, + esHost, + }; + }, +}); + +const stepProgressUpdateRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /api/observability_onboarding/custom_logs/step/{name}', + options: { tags: [] }, + params: t.type({ + path: t.type({ + name: t.string, + }), + query: t.type({ + status: t.string, + }), + }), + async handler(resources): Promise { + const { + params: { + path: { name }, + query: { status }, + }, + request, + core, + } = resources; + const { apiKeyId } = getAuthenticationAPIKey(request); + const coreStart = await core.start(); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); + + const savedObservabilityOnboardingState = + await getObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, + }); + + if (!savedObservabilityOnboardingState) { + return { + message: + 'Unable to report setup progress - onboarding session not found.', + }; + } + + const { id, updatedAt, ...observabilityOnboardingState } = + savedObservabilityOnboardingState; + await saveObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, + observabilityOnboardingState: { + ...observabilityOnboardingState, + progress: { + ...observabilityOnboardingState.progress, + [name]: status, + }, + }, + }); + return { name, status }; + }, +}); + +const getStateRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/custom_logs/state', + options: { tags: [] }, + params: t.type({ + query: t.type({ + apiKeyId: t.string, + }), + }), + async handler(resources): Promise<{ + savedObservabilityOnboardingState: SavedObservabilityOnboardingState | null; + }> { + const { + params: { + query: { apiKeyId }, + }, + core, + } = resources; + const coreStart = await core.start(); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); + const savedObservabilityOnboardingState = + (await getObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, + })) || null; + return { savedObservabilityOnboardingState }; + }, +}); + +const getLatestStateRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/custom_logs/state/latest', + options: { tags: [] }, + async handler(resources): Promise<{ + savedObservabilityOnboardingState: SavedObservabilityOnboardingState | null; + }> { + const { core } = resources; + const coreStart = await core.start(); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); + const savedObservabilityOnboardingState = + (await findLatestObservabilityOnboardingState({ savedObjectsClient })) || + null; + return { savedObservabilityOnboardingState }; + }, +}); + +const customLogsExistsRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/custom_logs/exists', + options: { tags: [] }, + params: t.type({ + query: t.type({ + dataset: t.string, + namespace: t.string, + }), + }), + async handler(resources): Promise<{ exists: boolean }> { + const { + core, + request, + params: { + query: { dataset, namespace }, + }, + } = resources; + const coreStart = await core.start(); + const esClient = + coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + try { + const { hits } = await esClient.search({ + index: `logs-${dataset}-${namespace}`, + terminate_after: 1, + }); + const total = hits.total as { value: number }; + return { exists: total.value > 0 }; + } catch (error) { + if (error.statusCode === 404) { + return { exists: false }; + } + throw Boom.boomify(error, { + statusCode: error.statusCode, + message: error.message, + data: error.body, + }); + } + }, +}); + +const deleteStatesRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'DELETE /internal/observability_onboarding/custom_logs/states', + options: { tags: [] }, + async handler(resources): Promise { + const { core } = resources; + const coreStart = await core.start(); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); + const findStatesResult = + await savedObjectsClient.find({ + type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + }); + const bulkDeleteResult = await savedObjectsClient.bulkDelete( + findStatesResult.saved_objects + ); + return { bulkDeleteResult }; + }, +}); + +export const customLogsRouteRepository = { + ...createApiKeyRoute, + ...stepProgressUpdateRoute, + ...getStateRoute, + ...getLatestStateRoute, + ...customLogsExistsRoute, + ...deleteStatesRoute, +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/save_observability_onboarding_state.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/save_observability_onboarding_state.ts new file mode 100644 index 0000000000000..1ae1ab8a43111 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/save_observability_onboarding_state.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + ObservabilityOnboardingState, + SavedObservabilityOnboardingState, +} from '../../saved_objects/observability_onboarding_status'; + +interface Options { + savedObjectsClient: SavedObjectsClientContract; + observabilityOnboardingState: ObservabilityOnboardingState; + apiKeyId: string; +} +export async function saveObservabilityOnboardingState({ + savedObjectsClient, + observabilityOnboardingState, + apiKeyId, +}: Options): Promise { + const { + id, + attributes, + updated_at: updatedAt, + } = await savedObjectsClient.update( + OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + apiKeyId, + observabilityOnboardingState, + { upsert: observabilityOnboardingState } + ); + return { + id, + ...(attributes as ObservabilityOnboardingState), + updatedAt: updatedAt ? Date.parse(updatedAt) : 0, + }; +} diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts new file mode 100644 index 0000000000000..fb27500639fc1 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dump, load } from 'js-yaml'; + +export const generateYml = ({ + datasetName = '', + namespace = '', + customConfigurations, + logFilePaths = [], + apiKey, + esHost, + logfileId, +}: { + datasetName?: string; + namespace?: string; + customConfigurations?: string; + logFilePaths?: string[]; + apiKey: string; + esHost: string[]; + logfileId: string; +}) => { + const customConfigYaml = load(customConfigurations ?? ''); + + return dump({ + ...{ + outputs: { + default: { + type: 'elasticsearch', + hosts: esHost, + api_key: apiKey, + }, + }, + inputs: [ + { + id: logfileId, + type: 'logfile', + data_stream: { + namespace, + }, + streams: [ + { + id: `logs-onboarding-${datasetName}`, + data_stream: { + dataset: datasetName, + }, + paths: logFilePaths, + }, + ], + }, + ], + }, + ...customConfigYaml, + }); +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts new file mode 100644 index 0000000000000..82acc3729e183 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key'; +import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; +import { findLatestObservabilityOnboardingState } from '../custom_logs/find_latest_observability_onboarding_state'; +import { getESHosts } from '../custom_logs/get_es_hosts'; +import { generateYml } from './generate_yml'; + +const generateConfig = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /api/observability_onboarding/elastic_agent/config', + options: { tags: [] }, + async handler(resources): Promise { + const { core, plugins, request } = resources; + const { apiKeyId, apiKey } = getAuthenticationAPIKey(request); + + const coreStart = await core.start(); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); + + const esHost = getESHosts({ + cloudSetup: plugins.cloud.setup, + esClient: coreStart.elasticsearch.client.asInternalUser as Client, + }); + + const savedState = await findLatestObservabilityOnboardingState({ + savedObjectsClient, + }); + + const yaml = generateYml({ + datasetName: savedState?.state.datasetName, + customConfigurations: savedState?.state.customConfigurations, + logFilePaths: savedState?.state.logFilePaths, + namespace: savedState?.state.namespace, + apiKey: `${apiKeyId}:${apiKey}`, + esHost, + logfileId: `custom-logs-${Date.now()}`, + }); + + return yaml; + }, +}); + +export const elasticAgentRouteRepository = { + ...generateConfig, +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/index.ts b/x-pack/plugins/observability_onboarding/server/routes/index.ts index 6a1067465787c..4833f44a2936c 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/index.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/index.ts @@ -9,10 +9,14 @@ import type { ServerRouteRepository, } from '@kbn/server-route-repository'; import { statusRouteRepository } from './status/route'; +import { customLogsRouteRepository } from './custom_logs/route'; +import { elasticAgentRouteRepository } from './elastic_agent/route'; function getTypedObservabilityOnboardingServerRouteRepository() { const repository = { ...statusRouteRepository, + ...customLogsRouteRepository, + ...elasticAgentRouteRepository, }; return repository; diff --git a/x-pack/plugins/observability_onboarding/server/routes/register_routes.ts b/x-pack/plugins/observability_onboarding/server/routes/register_routes.ts index 83000c1eaeec4..0220f64582396 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/register_routes.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/register_routes.ts @@ -66,6 +66,13 @@ export function registerRoutes({ logger, params: decodedParams, plugins, + core: { + setup: core, + start: async () => { + const [coreStart] = await core.getStartServices(); + return coreStart; + }, + }, })) as any; if (data === undefined) { diff --git a/x-pack/plugins/observability_onboarding/server/routes/types.ts b/x-pack/plugins/observability_onboarding/server/routes/types.ts index c8f1c0dc99560..6988561f84e44 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/types.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/types.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { KibanaRequest, Logger } from '@kbn/core/server'; +import { CoreSetup, CoreStart, KibanaRequest, Logger } from '@kbn/core/server'; import { ObservabilityOnboardingServerRouteRepository } from '.'; import { ObservabilityOnboardingPluginSetupDependencies, @@ -26,6 +26,10 @@ export interface ObservabilityOnboardingRouteHandlerResources { >; }; }; + core: { + setup: CoreSetup; + start: () => Promise; + }; } export interface ObservabilityOnboardingRouteCreateOptions { diff --git a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts new file mode 100644 index 0000000000000..13cc0e902e617 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsType } from '@kbn/core/server'; + +export const OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE = + 'observability-onboarding-state'; + +export interface ObservabilityOnboardingState { + state: { + datasetName: string; + customConfigurations: string; + logFilePaths: string[]; + namespace: string; + }; + progress: Record; +} + +export interface SavedObservabilityOnboardingState + extends ObservabilityOnboardingState { + id: string; + updatedAt: number; +} + +export const observabilityOnboardingState: SavedObjectsType = { + name: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, + hidden: false, + namespaceType: 'multiple', + mappings: { + properties: { + state: { type: 'object', dynamic: false }, + progress: { type: 'object', dynamic: false }, + }, + }, +}; diff --git a/x-pack/plugins/observability_onboarding/server/types.ts b/x-pack/plugins/observability_onboarding/server/types.ts index 99e7157178ff1..93113a5043904 100644 --- a/x-pack/plugins/observability_onboarding/server/types.ts +++ b/x-pack/plugins/observability_onboarding/server/types.ts @@ -5,21 +5,27 @@ * 2.0. */ +import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server'; import { CustomRequestHandlerContext } from '@kbn/core/server'; import { PluginSetup as DataPluginSetup, PluginStart as DataPluginStart, } from '@kbn/data-plugin/server'; import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; export interface ObservabilityOnboardingPluginSetupDependencies { data: DataPluginSetup; observability: ObservabilityPluginSetup; + cloud: CloudSetup; + usageCollection: UsageCollectionSetup; } export interface ObservabilityOnboardingPluginStartDependencies { data: DataPluginStart; observability: undefined; + cloud: CloudStart; + usageCollection: undefined; } // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/x-pack/plugins/observability_onboarding/tsconfig.json b/x-pack/plugins/observability_onboarding/tsconfig.json index a4ce3ecdc6f09..3452fd941d51f 100644 --- a/x-pack/plugins/observability_onboarding/tsconfig.json +++ b/x-pack/plugins/observability_onboarding/tsconfig.json @@ -23,7 +23,12 @@ "@kbn/config-schema", "@kbn/shared-ux-router", "@kbn/i18n-react", + "@kbn/cloud-plugin", + "@kbn/fleet-plugin", + "@kbn/usage-collection-plugin", "@kbn/observability-shared-plugin", + "@kbn/core-http-server", + "@kbn/security-plugin", ], "exclude": [ "target/**/*",