From 86e330406d896aa62ad2cf78b5505782d90fd321 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 5 Apr 2022 15:30:37 -0400 Subject: [PATCH] [Fleet] Encrypt ssl fields in logstash output (#129131) (cherry picked from commit 420359bacdfb23651da3b44580a6c2f2c6507b50) --- packages/kbn-doc-links/src/get_doc_links.ts | 1 + packages/kbn-doc-links/src/types.ts | 1 + .../fleet/common/types/models/output.ts | 1 + .../encryption_key_required_callout.tsx | 43 +++++++++++ .../edit_output_flyout/index.test.tsx | 20 +++++ .../components/edit_output_flyout/index.tsx | 10 +++ .../edit_output_flyout/use_output_form.tsx | 14 +++- x-pack/plugins/fleet/server/errors/index.ts | 2 +- x-pack/plugins/fleet/server/plugin.ts | 3 + .../fleet/server/routes/setup/handlers.ts | 10 ++- .../fleet/server/saved_objects/index.ts | 19 ++++- .../fleet/server/services/output.test.ts | 40 ++++++++++ .../plugins/fleet/server/services/output.ts | 73 ++++++++++++++----- 13 files changed, 216 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/encryption_key_required_callout.tsx diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index 7d209035ab65a..7d58f177e0764 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -229,6 +229,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { kibana: { guide: `${KIBANA_DOCS}index.html`, autocompleteSuggestions: `${KIBANA_DOCS}kibana-concepts-analysts.html#autocomplete-suggestions`, + secureSavedObject: `${KIBANA_DOCS}xpack-security-secure-saved-objects.html`, xpackSecurity: `${KIBANA_DOCS}xpack-security.html`, }, upgradeAssistant: { diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts index c2e485e1003e6..9a4ff1a58cf13 100644 --- a/packages/kbn-doc-links/src/types.ts +++ b/packages/kbn-doc-links/src/types.ts @@ -213,6 +213,7 @@ export interface DocLinks { readonly kibana: { readonly guide: string; readonly autocompleteSuggestions: string; + readonly secureSavedObject: string; readonly xpackSecurity: string; }; readonly upgradeAssistant: { diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts index 1e22c88f64cb2..dbdde9cf34e5b 100644 --- a/x-pack/plugins/fleet/common/types/models/output.ts +++ b/x-pack/plugins/fleet/common/types/models/output.ts @@ -29,6 +29,7 @@ export interface NewOutput { export type OutputSOAttributes = NewOutput & { output_id?: string; + ssl?: string; // encrypted ssl field }; export type Output = NewOutput & { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/encryption_key_required_callout.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/encryption_key_required_callout.tsx new file mode 100644 index 0000000000000..b4a85baa43b35 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/encryption_key_required_callout.tsx @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { EuiCallOut, EuiLink } from '@elastic/eui'; + +import { useStartServices } from '../../../../hooks'; + +export const EncryptionKeyRequiredCallout: React.FunctionComponent = () => { + const { docLinks } = useStartServices(); + return ( + + } + > + + + + ), + }} + /> + + ); +}; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx index 46bdc6d5b1785..c6d86a3fc4fbe 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx @@ -9,6 +9,7 @@ import React from 'react'; import type { Output } from '../../../../types'; import { createFleetTestRendererMock } from '../../../../../../mock'; +import { useFleetStatus } from '../../../../../../hooks/use_fleet_status'; import { EditOutputFlyout } from '.'; @@ -20,8 +21,11 @@ jest.mock('../../../../../../hooks/use_fleet_status', () => ({ FleetStatusProvider: (props: any) => { return props.children; }, + useFleetStatus: jest.fn().mockReturnValue({}), })); +const mockedUsedFleetStatus = useFleetStatus as jest.MockedFunction; + function renderFlyout(output?: Output) { const renderer = createFleetTestRendererMock(); @@ -66,4 +70,20 @@ describe('EditOutputFlyout', () => { expect(utils.queryByLabelText('Client SSL certificate')).not.toBeNull(); expect(utils.queryByLabelText('Server SSL certificate authorities')).not.toBeNull(); }); + + it('should show a callout in the flyout if the selected output is logstash and no encrypted key is set', async () => { + mockedUsedFleetStatus.mockReturnValue({ + missingRequirements: ['encrypted_saved_object_encryption_key_required'], + } as any); + const { utils } = renderFlyout({ + type: 'logstash', + name: 'logstash output', + id: 'output123', + is_default: false, + is_default_monitoring: false, + }); + + // Show logstash SSL inputs + expect(utils.getByText('Additional setup required')).not.toBeNull(); + }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index 2517d5c9e6f5c..72642e68d85c1 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -37,6 +37,7 @@ import { useBreadcrumbs, useStartServices } from '../../../../hooks'; import { YamlCodeEditorWithPlaceholder } from './yaml_code_editor_with_placeholder'; import { useOutputForm } from './use_output_form'; +import { EncryptionKeyRequiredCallout } from './encryption_key_required_callout'; export interface EditOutputFlyoutProps { output?: Output; @@ -60,6 +61,9 @@ export const EditOutputFlyout: React.FunctionComponent = const isLogstashOutput = inputs.typeInput.value === 'logstash'; const isESOutput = inputs.typeInput.value === 'elasticsearch'; + const showLogstashNeedEncryptedSavedObjectCallout = + isLogstashOutput && !form.hasEncryptedSavedObjectConfigured; + return ( @@ -160,6 +164,12 @@ export const EditOutputFlyout: React.FunctionComponent = )} /> + {showLogstashNeedEncryptedSavedObjectCallout && ( + <> + + + + )} {isLogstashOutput && ( <> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx index cd927923a4fe1..a27e9ad80ddfa 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx @@ -16,6 +16,7 @@ import { useSwitchInput, useStartServices, sendPutOutput, + useFleetStatus, } from '../../../../hooks'; import type { Output, PostOutputRequest } from '../../../../types'; import { useConfirmModal } from '../../hooks/use_confirm_modal'; @@ -32,6 +33,12 @@ import { import { confirmUpdate } from './confirm_update'; export function useOutputForm(onSucess: () => void, output?: Output) { + const fleetStatus = useFleetStatus(); + + const hasEncryptedSavedObjectConfigured = !fleetStatus.missingRequirements?.includes( + 'encrypted_saved_object_encryption_key_required' + ); + const [isLoading, setIsloading] = useState(false); const { notifications } = useStartServices(); const { confirm } = useConfirmModal(); @@ -234,6 +241,11 @@ export function useOutputForm(onSucess: () => void, output?: Output) { inputs, submit, isLoading, - isDisabled: isLoading || isPreconfigured || (output && !hasChanged), + hasEncryptedSavedObjectConfigured, + isDisabled: + isLoading || + isPreconfigured || + (output && !hasChanged) || + (isLogstash && !hasEncryptedSavedObjectConfigured), }; } diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index 41ebe9ef713f8..ca3209a58b2a5 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -51,7 +51,7 @@ export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError ); } } - +export class FleetEncryptedSavedObjectEncryptionKeyRequired extends IngestManagerError {} export class FleetSetupError extends IngestManagerError {} export class GenerateServiceTokenError extends IngestManagerError {} export class FleetUnauthorizedError extends IngestManagerError {} diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 272e92fca6eae..b94c80be75c4b 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -7,6 +7,7 @@ import type { Observable } from 'rxjs'; import { BehaviorSubject } from 'rxjs'; +import { take } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import type { @@ -416,6 +417,8 @@ export class FleetPlugin summary: 'Fleet is setting up', }); + await plugins.licensing.license$.pipe(take(1)).toPromise(); + await setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), core.elasticsearch.client.asInternalUser diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.ts index 60094c532b913..83c6a25f96b09 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.ts @@ -21,17 +21,25 @@ export const getFleetStatusHandler: FleetRequestHandler = async (context, reques context.core.elasticsearch.client.asInternalUser ); + let isReady = true; const missingRequirements: GetFleetStatusResponse['missing_requirements'] = []; + if (!isApiKeysEnabled) { + isReady = false; missingRequirements.push('api_keys'); } if (!isFleetServerSetup) { + isReady = false; missingRequirements.push('fleet_server'); } + if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { + missingRequirements.push('encrypted_saved_object_encryption_key_required'); + } + const body: GetFleetStatusResponse = { - isReady: missingRequirements.length === 0, + isReady, missing_requirements: missingRequirements, }; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 262795330deae..915d841dd287e 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -118,7 +118,7 @@ const getSavedObjectTypes = ( config: { type: 'flattened' }, config_yaml: { type: 'text' }, is_preconfigured: { type: 'boolean', index: false }, - ssl: { type: 'flattened', index: false }, + ssl: { type: 'binary' }, }, }, migrations: { @@ -310,5 +310,22 @@ export function registerSavedObjects( export function registerEncryptedSavedObjects( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ) { + encryptedSavedObjects.registerType({ + type: OUTPUT_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set([{ key: 'ssl', dangerouslyExposeValue: true }]), + attributesToExcludeFromAAD: new Set([ + 'output_id', + 'name', + 'type', + 'is_default', + 'is_default_monitoring', + 'hosts', + 'ca_sha256', + 'ca_trusted_fingerprint', + 'config', + 'config_yaml', + 'is_preconfigured', + ]), + }); // Encrypted saved objects } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 211abecbc4395..0a7d5393a63e1 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -161,6 +161,8 @@ function getMockedSoClient( }; }); + mockedAppContextService.getInternalUserSOClient.mockReturnValue(soClient); + return soClient; } @@ -169,6 +171,8 @@ describe('Output Service', () => { mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.removeOutputFromAll.mockReset(); + mockedAppContextService.getInternalUserSOClient.mockReset(); + mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); }); describe('create', () => { it('work with a predefined id', async () => { @@ -321,6 +325,42 @@ describe('Output Service', () => { { is_default: false } ); }); + + // With logstash output + it('should throw if encryptedSavedObject is not configured', async () => { + const soClient = getMockedSoClient({}); + + await expect( + outputService.create( + soClient, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'logstash', + }, + { id: 'output-test' } + ) + ).rejects.toThrow(`Logstash output needs encrypted saved object api key to be set`); + }); + + it('should work if encryptedSavedObject is configured', async () => { + const soClient = getMockedSoClient({}); + mockedAppContextService.getEncryptedSavedObjectsSetup.mockReturnValue({ + canEncrypt: true, + } as any); + await outputService.create( + soClient, + { + is_default: false, + is_default_monitoring: false, + name: 'Test', + type: 'logstash', + }, + { id: 'output-test' } + ); + expect(soClient.create).toBeCalled(); + }); }); describe('update', () => { diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 3d2f5de0339ca..4aecbd6e03349 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -5,8 +5,9 @@ * 2.0. */ -import type { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import type { KibanaRequest, SavedObject, SavedObjectsClientContract } from 'src/core/server'; import uuid from 'uuid/v5'; +import { omit } from 'lodash'; import type { NewOutput, Output, OutputSOAttributes } from '../types'; import { @@ -16,7 +17,11 @@ import { AGENT_POLICY_SAVED_OBJECT_TYPE, } from '../constants'; import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT, outputType } from '../../common'; -import { OutputUnauthorizedError, OutputInvalidError } from '../errors'; +import { + OutputUnauthorizedError, + OutputInvalidError, + FleetEncryptedSavedObjectEncryptionKeyRequired, +} from '../errors'; import { agentPolicyService } from './agent_policy'; import { appContextService } from './app_context'; @@ -27,6 +32,21 @@ const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE; const DEFAULT_ES_HOSTS = ['http://localhost:9200']; +const fakeRequest = { + headers: {}, + getBasePath: () => '', + path: '/', + route: { settings: {} }, + url: { + href: '/', + }, + raw: { + req: { + url: '/', + }, + }, +} as unknown as KibanaRequest; + // differentiate function isUUID(val: string) { return ( @@ -45,10 +65,12 @@ export function outputIdToUuid(id: string) { } function outputSavedObjectToOutput(so: SavedObject) { - const { output_id: outputId, ...atributes } = so.attributes; + const { output_id: outputId, ssl, ...atributes } = so.attributes; + return { id: outputId ?? so.id, ...atributes, + ...(ssl ? { ssl: JSON.parse(ssl as string) } : {}), }; } @@ -86,8 +108,12 @@ async function validateLogstashOutputNotUsedInAPMPolicy( } class OutputService { + private get encryptedSoClient() { + return appContextService.getInternalUserSOClient(fakeRequest); + } + private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { - return await soClient.find({ + return await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], search: 'true', @@ -95,7 +121,7 @@ class OutputService { } private async _getDefaultMonitoringOutputsSO(soClient: SavedObjectsClientContract) { - return await soClient.find({ + return await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default_monitoring'], search: 'true', @@ -164,10 +190,15 @@ class OutputService { output: NewOutput, options?: { id?: string; fromPreconfiguration?: boolean; overwrite?: boolean } ): Promise { - const data: OutputSOAttributes = { ...output }; + const data: OutputSOAttributes = { ...omit(output, 'ssl') }; if (output.type === outputType.Logstash) { await validateLogstashOutputNotUsedInAPMPolicy(soClient, undefined, data.is_default); + if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { + throw new FleetEncryptedSavedObjectEncryptionKeyRequired( + 'Logstash output needs encrypted saved object api key to be set' + ); + } } // ensure only default output exists @@ -202,15 +233,16 @@ class OutputService { data.output_id = options?.id; } - const newSo = await soClient.create(SAVED_OBJECT_TYPE, data, { + if (output.ssl) { + data.ssl = JSON.stringify(output.ssl); + } + + const newSo = await this.encryptedSoClient.create(SAVED_OBJECT_TYPE, data, { overwrite: options?.overwrite || options?.fromPreconfiguration, id: options?.id ? outputIdToUuid(options.id) : undefined, }); - return { - id: options?.id ?? newSo.id, - ...newSo.attributes, - }; + return outputSavedObjectToOutput(newSo); } public async bulkGet( @@ -218,7 +250,7 @@ class OutputService { ids: string[], { ignoreNotFound = false } = { ignoreNotFound: true } ) { - const res = await soClient.bulkGet( + const res = await this.encryptedSoClient.bulkGet( ids.map((id) => ({ id: outputIdToUuid(id), type: SAVED_OBJECT_TYPE })) ); @@ -237,7 +269,7 @@ class OutputService { } public async list(soClient: SavedObjectsClientContract) { - const outputs = await soClient.find({ + const outputs = await this.encryptedSoClient.find({ type: SAVED_OBJECT_TYPE, page: 1, perPage: SO_SEARCH_LIMIT, @@ -254,7 +286,10 @@ class OutputService { } public async get(soClient: SavedObjectsClientContract, id: string): Promise { - const outputSO = await soClient.get(SAVED_OBJECT_TYPE, outputIdToUuid(id)); + const outputSO = await this.encryptedSoClient.get( + SAVED_OBJECT_TYPE, + outputIdToUuid(id) + ); if (outputSO.error) { throw new Error(outputSO.error.message); @@ -292,7 +327,7 @@ class OutputService { id ); - return soClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); + return this.encryptedSoClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); } public async update( @@ -311,7 +346,7 @@ class OutputService { ); } - const updateData: Nullable> = { ...data }; + const updateData: Nullable> = { ...omit(data, 'ssl') }; const mergedType = data.type ?? originalOutput.type; const mergedIsDefault = data.is_default ?? originalOutput.is_default; @@ -331,6 +366,10 @@ class OutputService { } } + if (data.ssl) { + updateData.ssl = JSON.stringify(data.ssl); + } + // ensure only default output exists if (data.is_default) { const defaultDataOuputId = await this.getDefaultDataOutputId(soClient); @@ -359,7 +398,7 @@ class OutputService { if (mergedType === outputType.Elasticsearch && updateData.hosts) { updateData.hosts = updateData.hosts.map(normalizeHostsForAgents); } - const outputSO = await soClient.update>( + const outputSO = await this.encryptedSoClient.update>( SAVED_OBJECT_TYPE, outputIdToUuid(id), updateData