From bc91dc2caaed57e4c38f2ae4a49355b314d252f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 9 Mar 2021 11:52:31 +0100 Subject: [PATCH] [Fleet] Migrate ES client (#92805) (#94051) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- api_docs/fleet.json | 2 +- .../server/routes/agent_policy/handlers.ts | 3 +- .../server/routes/data_streams/handlers.ts | 78 +++---- .../fleet/server/routes/epm/handlers.ts | 18 +- .../routes/package_policy/handlers.test.ts | 5 +- .../server/routes/package_policy/handlers.ts | 13 +- .../fleet/server/routes/setup/handlers.ts | 8 +- .../server/services/api_keys/security.ts | 11 - .../elasticsearch/datastream_ilm/install.ts | 15 +- .../elasticsearch/datastream_ilm/remove.ts | 20 +- .../services/epm/elasticsearch/ilm/install.ts | 20 +- .../elasticsearch/ingest_pipeline/install.ts | 51 ++-- .../elasticsearch/ingest_pipeline/remove.ts | 11 +- .../elasticsearch/template/install.test.ts | 122 +++++----- .../epm/elasticsearch/template/install.ts | 151 +++++------- .../epm/elasticsearch/template/template.ts | 36 ++- .../epm/elasticsearch/transform/install.ts | 35 ++- .../epm/elasticsearch/transform/remove.ts | 62 +++-- .../elasticsearch/transform/transform.test.ts | 217 ++++++++---------- .../epm/packages/_install_package.test.ts | 10 +- .../services/epm/packages/_install_package.ts | 29 +-- .../epm/packages/bulk_install_packages.ts | 9 +- .../server/services/epm/packages/install.ts | 69 +++--- .../server/services/epm/packages/remove.ts | 41 ++-- .../fleet/server/services/package_policy.ts | 4 +- .../fleet/server/services/setup.test.ts | 4 +- x-pack/plugins/fleet/server/services/setup.ts | 49 ++-- x-pack/plugins/fleet/server/types/index.tsx | 4 - 28 files changed, 465 insertions(+), 632 deletions(-) diff --git a/api_docs/fleet.json b/api_docs/fleet.json index fc5c2211fbd85..14884efd970b2 100644 --- a/api_docs/fleet.json +++ b/api_docs/fleet.json @@ -2453,7 +2453,7 @@ "description": [], "source": { "path": "x-pack/plugins/fleet/server/services/package_policy.ts", - "lineNumber": 594 + "lineNumber": 592 }, "signature": [ "PackagePolicyService" diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts index 5a4b58bb41931..0496c8c5b0b8f 100644 --- a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts @@ -104,7 +104,6 @@ export const createAgentPolicyHandler: RequestHandler< > = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; const withSysMonitoring = request.query.sys_monitoring ?? false; try { @@ -130,7 +129,7 @@ export const createAgentPolicyHandler: RequestHandler< if (withSysMonitoring && newSysPackagePolicy !== undefined && agentPolicy !== undefined) { newSysPackagePolicy.policy_id = agentPolicy.id; newSysPackagePolicy.namespace = agentPolicy.namespace; - await packagePolicyService.create(soClient, esClient, callCluster, newSysPackagePolicy, { + await packagePolicyService.create(soClient, esClient, newSysPackagePolicy, { user, bumpRevision: false, }); diff --git a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts index 3d05cea261f70..a928c80f6dd81 100644 --- a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts @@ -16,40 +16,37 @@ import { defaultIngestErrorHandler } from '../../errors'; const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*'; -interface ESDataStreamInfoResponse { - data_streams: Array<{ +interface ESDataStreamInfo { + name: string; + timestamp_field: { name: string; - timestamp_field: { + }; + indices: Array<{ index_name: string; index_uuid: string }>; + generation: number; + _meta?: { + package?: { name: string; }; - indices: Array<{ index_name: string; index_uuid: string }>; - generation: number; - _meta?: { - package?: { - name: string; - }; - managed_by?: string; - managed?: boolean; - [key: string]: any; - }; - status: string; - template: string; - ilm_policy: string; - hidden: boolean; - }>; + managed_by?: string; + managed?: boolean; + [key: string]: any; + }; + status: string; + template: string; + ilm_policy: string; + hidden: boolean; } -interface ESDataStreamStatsResponse { - data_streams: Array<{ - data_stream: string; - backing_indices: number; - store_size_bytes: number; - maximum_timestamp: number; - }>; +interface ESDataStreamStats { + data_stream: string; + backing_indices: number; + store_size_bytes: number; + maximum_timestamp: number; } export const getListHandler: RequestHandler = async (context, request, response) => { - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; + const body: GetDataStreamsResponse = { data_streams: [], }; @@ -57,22 +54,21 @@ export const getListHandler: RequestHandler = async (context, request, response) try { // Get matching data streams, their stats, and package SOs const [ - { data_streams: dataStreamsInfo }, - { data_streams: dataStreamStats }, + { + body: { data_streams: dataStreamsInfo }, + }, + { + body: { data_streams: dataStreamStats }, + }, packageSavedObjects, ] = await Promise.all([ - callCluster('transport.request', { - method: 'GET', - path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}`, - }) as Promise, - callCluster('transport.request', { - method: 'GET', - path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}/_stats`, - }) as Promise, + esClient.indices.getDataStream({ name: DATA_STREAM_INDEX_PATTERN }), + esClient.indices.dataStreamsStats({ name: DATA_STREAM_INDEX_PATTERN }), getPackageSavedObjects(context.core.savedObjects.client), ]); - const dataStreamsInfoByName = keyBy(dataStreamsInfo, 'name'); - const dataStreamsStatsByName = keyBy(dataStreamStats, 'data_stream'); + + const dataStreamsInfoByName = keyBy(dataStreamsInfo, 'name'); + const dataStreamsStatsByName = keyBy(dataStreamStats, 'data_stream'); // Combine data stream info const dataStreams = merge(dataStreamsInfoByName, dataStreamsStatsByName); @@ -99,8 +95,10 @@ export const getListHandler: RequestHandler = async (context, request, response) // Query backing indices to extract data stream dataset, namespace, and type values const { - aggregations: { dataset, namespace, type }, - } = await callCluster('search', { + body: { + aggregations: { dataset, namespace, type }, + }, + } = await esClient.search({ index: dataStream.indices.map((index) => index.index_name), body: { size: 0, diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 9d2edb4f0f155..7237bdb296cd6 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -226,7 +226,7 @@ export const installPackageFromRegistryHandler: RequestHandler< TypeOf > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const { pkgkey } = request.params; const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); @@ -235,7 +235,7 @@ export const installPackageFromRegistryHandler: RequestHandler< installSource: 'registry', savedObjectsClient, pkgkey, - callCluster, + esClient, force: request.body?.force, }); const body: InstallPackageResponse = { @@ -250,7 +250,7 @@ export const installPackageFromRegistryHandler: RequestHandler< pkgName, pkgVersion, installedPkg, - callCluster, + esClient, }); return defaultResult; @@ -278,10 +278,10 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler< TypeOf > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const bulkInstalledResponses = await bulkInstallPackages({ savedObjectsClient, - callCluster, + esClient, packagesToUpgrade: request.body.packages, }); const payload = bulkInstalledResponses.map(bulkInstallServiceResponseToHttpEntry); @@ -303,14 +303,14 @@ export const installPackageByUploadHandler: RequestHandler< }); } const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later const archiveBuffer = Buffer.from(request.body); try { const res = await installPackage({ installSource: 'upload', savedObjectsClient, - callCluster, + esClient, archiveBuffer, contentType, }); @@ -329,8 +329,8 @@ export const deletePackageHandler: RequestHandler< try { const { pkgkey } = request.params; const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - const res = await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); + const esClient = context.core.elasticsearch.client.asCurrentUser; + const res = await removeInstallation({ savedObjectsClient, pkgkey, esClient }); const body: DeletePackageResponse = { response: res, }; diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index 865f07c0f9271..6359fa45fd0f3 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -27,7 +27,7 @@ jest.mock('../../services/package_policy', (): { compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), - create: jest.fn((soClient, esClient, callCluster, newData) => + create: jest.fn((soClient, esClient, newData) => Promise.resolve({ ...newData, inputs: newData.inputs.map((input) => ({ @@ -204,7 +204,8 @@ describe('When calling package policy', () => { ); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packagePolicyServiceMock.create.mock.calls[0][3]).toEqual({ + + expect(packagePolicyServiceMock.create.mock.calls[0][2]).toEqual({ policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index d456ef6b525b7..14d7e38752cf2 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -79,7 +79,6 @@ export const createPackagePolicyHandler: RequestHandler< > = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; try { const newData = await packagePolicyService.runExternalCallbacks( @@ -90,15 +89,9 @@ export const createPackagePolicyHandler: RequestHandler< ); // Create package policy - const packagePolicy = await packagePolicyService.create( - soClient, - esClient, - callCluster, - newData, - { - user, - } - ); + const packagePolicy = await packagePolicyService.create(soClient, esClient, newData, { + user, + }); const body: CreatePackagePolicyResponse = { item: packagePolicy }; return response.ok({ body, diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.ts index 0dc3a0109d682..b306b4d6802ad 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.ts @@ -63,9 +63,8 @@ export const createFleetSetupHandler: RequestHandler< try { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - await setupIngestManager(soClient, esClient, callCluster); - await setupFleet(soClient, esClient, callCluster, { + await setupIngestManager(soClient, esClient); + await setupFleet(soClient, esClient, { forceRecreate: request.body?.forceRecreate ?? false, }); @@ -80,11 +79,10 @@ export const createFleetSetupHandler: RequestHandler< export const FleetSetupHandler: RequestHandler = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; try { const body: PostIngestSetupResponse = { isInitialized: true }; - await setupIngestManager(soClient, esClient, callCluster); + await setupIngestManager(soClient, esClient); return response.ok({ body, }); diff --git a/x-pack/plugins/fleet/server/services/api_keys/security.ts b/x-pack/plugins/fleet/server/services/api_keys/security.ts index 836e7b2343f6c..599785cb5ff7b 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/security.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/security.ts @@ -10,7 +10,6 @@ import type { Request } from '@hapi/hapi'; import { KibanaRequest } from '../../../../../../src/core/server'; import type { SavedObjectsClientContract } from '../../../../../../src/core/server'; import { FleetAdminUserInvalidError, isESClientError } from '../../errors'; -import type { CallESAsCurrentUser } from '../../types'; import { appContextService } from '../app_context'; import { outputService } from '../output'; @@ -56,16 +55,6 @@ export async function createAPIKey( throw err; } } -export async function authenticate(callCluster: CallESAsCurrentUser) { - try { - await callCluster('transport.request', { - path: '/_security/_authenticate', - method: 'GET', - }); - } catch (e) { - throw new Error('ApiKey is not valid: impossible to authenticate user'); - } -} export async function invalidateAPIKeys(soClient: SavedObjectsClientContract, ids: string[]) { const adminUser = await outputService.getAdminUser(soClient); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts index 7d821e1416c9f..4f114e0df11cc 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'kibana/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; import type { @@ -13,7 +13,6 @@ import type { InstallablePackage, RegistryDataStream, } from '../../../../../common/types/models'; -import type { CallESAsCurrentUser } from '../../../../types'; import { getInstallation } from '../../packages'; import { saveInstalledEsRefs } from '../../packages/install'; import { getAsset } from '../transform/common'; @@ -33,7 +32,7 @@ interface IlmPathDataset { export const installIlmForDataStream = async ( registryPackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name }); @@ -46,7 +45,7 @@ export const installIlmForDataStream = async ( // delete all previous ilm await deleteIlms( - callCluster, + esClient, previousInstalledIlmEsAssets.map((asset) => asset.id) ); // install the latest dataset @@ -86,7 +85,7 @@ export const installIlmForDataStream = async ( ); const installationPromises = ilmInstallations.map(async (ilmInstallation) => { - return handleIlmInstall({ callCluster, ilmInstallation }); + return handleIlmInstall({ esClient, ilmInstallation }); }); installedIlms = await Promise.all(installationPromises).then((results) => results.flat()); @@ -111,13 +110,13 @@ export const installIlmForDataStream = async ( }; async function handleIlmInstall({ - callCluster, + esClient, ilmInstallation, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; ilmInstallation: IlmInstallation; }): Promise { - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: `/_ilm/policy/${ilmInstallation.installationName}`, body: ilmInstallation.content, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts index 651710c02fc38..e45e2f59efe26 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts @@ -5,20 +5,24 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'kibana/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType } from '../../../../types'; -import type { CallESAsCurrentUser, EsAssetReference } from '../../../../types'; +import type { EsAssetReference } from '../../../../types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants'; -export const deleteIlms = async (callCluster: CallESAsCurrentUser, ilmPolicyIds: string[]) => { +export const deleteIlms = async (esClient: ElasticsearchClient, ilmPolicyIds: string[]) => { await Promise.all( ilmPolicyIds.map(async (ilmPolicyId) => { - await callCluster('transport.request', { - method: 'DELETE', - path: `_ilm/policy/${ilmPolicyId}`, - ignore: [404, 400], - }); + await esClient.transport.request( + { + method: 'DELETE', + path: `_ilm/policy/${ilmPolicyId}`, + }, + { + ignore: [404, 400], + } + ); }) ); }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts index b6123d190c0eb..8d6f37345902e 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts @@ -5,11 +5,12 @@ * 2.0. */ +import type { ElasticsearchClient } from 'kibana/server'; + import { ElasticsearchAssetType } from '../../../../types'; -import type { CallESAsCurrentUser } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; -export async function installILMPolicy(paths: string[], callCluster: CallESAsCurrentUser) { +export async function installILMPolicy(paths: string[], esClient: ElasticsearchClient) { const ilmPaths = paths.filter((path) => isILMPolicy(path)); if (!ilmPaths.length) return; await Promise.all( @@ -18,7 +19,7 @@ export async function installILMPolicy(paths: string[], callCluster: CallESAsCur const { file } = getPathParts(path); const name = file.substr(0, file.lastIndexOf('.')); try { - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: '/_ilm/policy/' + name, body, @@ -29,19 +30,8 @@ export async function installILMPolicy(paths: string[], callCluster: CallESAsCur }) ); } + const isILMPolicy = (path: string) => { const pathParts = getPathParts(path); return pathParts.type === ElasticsearchAssetType.ilmPolicy; }; -export async function policyExists( - name: string, - callCluster: CallESAsCurrentUser -): Promise { - const response = await callCluster('transport.request', { - method: 'GET', - path: '/_ilm/policy/?filter_path=' + name, - }); - - // If the response contains a key, it means the policy exists - return Object.keys(response).length > 0; -} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 735242c1e6984..ac5aca7ab1c14 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -5,15 +5,11 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { TransportRequestOptions } from '@elastic/elasticsearch/lib/Transport'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { ElasticsearchAssetType } from '../../../../types'; -import type { - EsAssetReference, - RegistryDataStream, - InstallablePackage, - CallESAsCurrentUser, -} from '../../../../types'; +import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; import type { ArchiveEntry } from '../../archive'; import { saveInstalledEsRefs } from '../../packages/install'; @@ -30,7 +26,7 @@ interface RewriteSubstitution { export const installPipelines = async ( installablePackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { // unlike other ES assets, pipeline names are versioned so after a template is updated @@ -77,7 +73,7 @@ export const installPipelines = async ( acc.push( installPipelinesForDataStream({ dataStream, - callCluster, + esClient, paths: pipelinePaths, pkgVersion: installablePackage.version, }) @@ -110,12 +106,12 @@ export function rewriteIngestPipeline( } export async function installPipelinesForDataStream({ - callCluster, + esClient, pkgVersion, paths, dataStream, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; pkgVersion: string; paths: string[]; dataStream: RegistryDataStream; @@ -153,33 +149,30 @@ export async function installPipelinesForDataStream({ }); const installationPromises = pipelines.map(async (pipeline) => { - return installPipeline({ callCluster, pipeline }); + return installPipeline({ esClient, pipeline }); }); return Promise.all(installationPromises); } async function installPipeline({ - callCluster, + esClient, pipeline, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; pipeline: any; }): Promise { - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - body: any; - headers?: any; - } = { - method: 'PUT', - path: `/_ingest/pipeline/${pipeline.nameForInstallation}`, - ignore: [404], + const esClientParams = { + id: pipeline.nameForInstallation, body: pipeline.contentForInstallation, }; + + const esClientRequestOptions: TransportRequestOptions = { + ignore: [404], + }; + if (pipeline.extension === 'yml') { - callClusterParams.headers = { + esClientRequestOptions.headers = { // pipeline is YAML 'Content-Type': 'application/yaml', // but we want JSON responses (to extract error messages, status code, or other metadata) @@ -187,12 +180,8 @@ async function installPipeline({ }; } - // This uses the catch-all endpoint 'transport.request' because we have to explicitly - // set the Content-Type header above for sending yml data. Setting the headers is not - // exposed in the convenience endpoint 'ingest.putPipeline' of elasticsearch-js-legacy - // which we could otherwise use. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + await esClient.ingest.putPipeline(esClientParams, esClientRequestOptions); + return { id: pipeline.nameForInstallation, type: ElasticsearchAssetType.ingestPipeline }; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts index 46f9be53f8864..b7de8fd8ff5ed 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts @@ -5,18 +5,17 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { appContextService } from '../../../'; import { ElasticsearchAssetType } from '../../../../types'; -import type { CallESAsCurrentUser } from '../../../../types'; import { IngestManagerError } from '../../../../errors'; import { getInstallation } from '../../packages/get'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import type { EsAssetReference } from '../../../../../common'; export const deletePreviousPipelines = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, pkgName: string, previousPkgVersion: string @@ -30,7 +29,7 @@ export const deletePreviousPipelines = async ( type === ElasticsearchAssetType.ingestPipeline && id.includes(previousPkgVersion) ); const deletePipelinePromises = installedPipelines.map(({ type, id }) => { - return deletePipeline(callCluster, id); + return deletePipeline(esClient, id); }); try { await Promise.all(deletePipelinePromises); @@ -59,11 +58,11 @@ export const deletePipelineRefs = async ( installed_es: filteredAssets, }); }; -export async function deletePipeline(callCluster: CallESAsCurrentUser, id: string): Promise { +export async function deletePipeline(esClient: ElasticsearchClient, id: string): Promise { // '*' shouldn't ever appear here, but it still would delete all ingest pipelines if (id && id !== '*') { try { - await callCluster('ingest.deletePipeline', { id }); + await esClient.ingest.deletePipeline({ id }); } catch (err) { // Only throw if error is not a 404 error. Sometimes the pipeline is already deleted, but we have // duplicate references to them, see https://github.com/elastic/kibana/issues/91192 diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index d68d7715436a3..8c58238588b2f 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -20,17 +20,10 @@ describe('EPM install', () => { }); it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() - .callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } - }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixUnset = { @@ -49,31 +42,28 @@ describe('EPM install', () => { const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixUnset = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixUnset, packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() - .callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } - }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixFalse = { @@ -93,31 +83,28 @@ describe('EPM install', () => { const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixFalse = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixFalse, packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); }); it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() - .callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { index_templates: [] }; - } - }); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixTrue = { @@ -137,41 +124,37 @@ describe('EPM install', () => { const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; const templatePriorityDatasetIsPrefixTrue = 150; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixTrue, packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); }); it('tests installPackage remove the aliases property if the property existed', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient() - .callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return { - index_templates: [ - { - name: 'metrics-package.dataset', - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, - }, + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + index_templates: [ + { + name: 'metrics-package.dataset', + index_template: { + index_patterns: ['metrics-package.dataset-*'], + template: { aliases: {} }, }, - ], - }; - } - }); + }, + ], + }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixUnset = { @@ -190,18 +173,23 @@ describe('EPM install', () => { const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixUnset = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixUnset, packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const removeAliases = callCluster.mock.calls[1][1].body; + const removeAliases = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; expect(removeAliases.template.aliases).not.toBeDefined(); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[2][1].body; + + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[1][0]!.body as Record< + string, + any + >; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 2769b97fe48a1..98ba970fda39b 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -6,7 +6,7 @@ */ import Boom from '@hapi/boom'; -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { ElasticsearchAssetType } from '../../../../types'; import type { @@ -14,7 +14,6 @@ import type { TemplateRef, RegistryElasticsearch, InstallablePackage, - CallESAsCurrentUser, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -32,15 +31,15 @@ import { export const installTemplates = async ( installablePackage: InstallablePackage, - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, paths: string[], savedObjectsClient: SavedObjectsClientContract ): Promise => { // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates - await installPreBuiltComponentTemplates(paths, callCluster); - await installPreBuiltTemplates(paths, callCluster); + await installPreBuiltComponentTemplates(paths, esClient); + await installPreBuiltTemplates(paths, esClient); // remove package installation's references to index templates await removeAssetsFromInstalledEsByType( @@ -66,7 +65,7 @@ export const installTemplates = async ( acc.push( installTemplateForDataStream({ pkg: installablePackage, - callCluster, + esClient, dataStream, }) ); @@ -83,38 +82,23 @@ export const installTemplates = async ( return []; }; -const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCurrentUser) => { +const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => { const templatePaths = paths.filter((path) => isTemplate(path)); const templateInstallPromises = templatePaths.map(async (path) => { const { file } = getPathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - let templateAPIPath = '_template'; - // v2 index templates need to be installed through the new API endpoint. - // Checking for 'template' and 'composed_of' should catch them all. - // For the new v2 format, see https://github.com/elastic/elasticsearch/issues/53101 + const esClientParams = { name: templateName, body: content }; + const esClientRequestOptions = { ignore: [404] }; + if (content.hasOwnProperty('template') || content.hasOwnProperty('composed_of')) { - templateAPIPath = '_index_template'; + // Template is v2 + return esClient.indices.putIndexTemplate(esClientParams, esClientRequestOptions); + } else { + // template is V1 + return esClient.indices.putTemplate(esClientParams, esClientRequestOptions); } - - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - body: any; - } = { - method: 'PUT', - path: `/${templateAPIPath}/${templateName}`, - ignore: [404], - body: content, - }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return callCluster('transport.request', callClusterParams); }); try { return await Promise.all(templateInstallPromises); @@ -127,7 +111,7 @@ const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCu const installPreBuiltComponentTemplates = async ( paths: string[], - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) => { const templatePaths = paths.filter((path) => isComponentTemplate(path)); const templateInstallPromises = templatePaths.map(async (path) => { @@ -135,22 +119,14 @@ const installPreBuiltComponentTemplates = async ( const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - body: any; - } = { - method: 'PUT', - path: `/_component_template/${templateName}`, - ignore: [404], + const esClientParams = { + name: templateName, body: content, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint for component templates yet. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return callCluster('transport.request', callClusterParams); + + return esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }); }); + try { return await Promise.all(templateInstallPromises); } catch (e) { @@ -178,16 +154,16 @@ const isComponentTemplate = (path: string) => { export async function installTemplateForDataStream({ pkg, - callCluster, + esClient, dataStream, }: { pkg: InstallablePackage; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; dataStream: RegistryDataStream; }): Promise { const fields = await loadFieldsFromYaml(pkg, dataStream.path); return installTemplate({ - callCluster, + esClient, fields, dataStream, packageVersion: pkg.version, @@ -198,22 +174,18 @@ export async function installTemplateForDataStream({ function putComponentTemplate( body: object | undefined, name: string, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): { clusterPromise: Promise; name: string } | undefined { if (body) { - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - body: any; - } = { - method: 'PUT', - path: `/_component_template/${name}`, - ignore: [404], + const esClientParams = { + name, body, }; - return { clusterPromise: callCluster('transport.request', callClusterParams), name }; + return { + clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }), + name, + }; } } @@ -253,7 +225,7 @@ function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | async function installDataStreamComponentTemplates( templateName: string, registryElasticsearch: RegistryElasticsearch | undefined, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) { const templates: string[] = []; const componentPromises: Array> = []; @@ -263,13 +235,13 @@ async function installDataStreamComponentTemplates( const mappings = putComponentTemplate( compTemplates.mappingsTemplate, `${templateName}-mappings`, - callCluster + esClient ); const settings = putComponentTemplate( compTemplates.settingsTemplate, `${templateName}-settings`, - callCluster + esClient ); if (mappings) { @@ -288,13 +260,13 @@ async function installDataStreamComponentTemplates( } export async function installTemplate({ - callCluster, + esClient, fields, dataStream, packageVersion, packageName, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; fields: Field[]; dataStream: RegistryDataStream; packageVersion: string; @@ -316,11 +288,14 @@ export async function installTemplate({ } // Datastream now throw an error if the aliases field is present so ensure that we remove that field. - const getTemplateRes = await callCluster('transport.request', { - method: 'GET', - path: `/_index_template/${templateName}`, - ignore: [404], - }); + const { body: getTemplateRes } = await esClient.indices.getIndexTemplate( + { + name: templateName, + }, + { + ignore: [404], + } + ); const existingIndexTemplate = getTemplateRes?.index_templates?.[0]; if ( @@ -328,15 +303,8 @@ export async function installTemplate({ existingIndexTemplate.name === templateName && existingIndexTemplate?.index_template?.template?.aliases ) { - const updateIndexTemplateParams: { - method: string; - path: string; - ignore: number[]; - body: any; - } = { - method: 'PUT', - path: `/_index_template/${templateName}`, - ignore: [404], + const updateIndexTemplateParams = { + name: templateName, body: { ...existingIndexTemplate.index_template, template: { @@ -346,18 +314,14 @@ export async function installTemplate({ }, }, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', updateIndexTemplateParams); + + await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] }); } const composedOfTemplates = await installDataStreamComponentTemplates( templateName, dataStream.elasticsearch, - callCluster + esClient ); const template = getTemplate({ @@ -374,23 +338,12 @@ export async function installTemplate({ }); // TODO: Check return values for errors - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - body: any; - } = { - method: 'PUT', - path: `/_index_template/${templateName}`, - ignore: [404], + const esClientParams = { + name: templateName, body: template, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + + await esClient.indices.putIndexTemplate(esClientParams, { ignore: [404] }); return { templateName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 01b9a92045b29..456ed95a8c3e4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -5,10 +5,11 @@ * 2.0. */ +import type { ElasticsearchClient } from 'kibana/server'; + import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, - CallESAsCurrentUser, TemplateRef, IndexTemplate, IndexTemplateMappings, @@ -424,14 +425,14 @@ function getBaseTemplate( } export const updateCurrentWriteIndices = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, templates: TemplateRef[] ): Promise => { if (!templates.length) return; - const allIndices = await queryDataStreamsFromTemplates(callCluster, templates); + const allIndices = await queryDataStreamsFromTemplates(esClient, templates); if (!allIndices.length) return; - return updateAllDataStreams(allIndices, callCluster); + return updateAllDataStreams(allIndices, esClient); }; function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is CurrentDataStream[] { @@ -439,26 +440,23 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur } const queryDataStreamsFromTemplates = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, templates: TemplateRef[] ): Promise => { const dataStreamPromises = templates.map((template) => { - return getDataStreams(callCluster, template); + return getDataStreams(esClient, template); }); const dataStreamObjects = await Promise.all(dataStreamPromises); return dataStreamObjects.filter(isCurrentDataStream).flat(); }; const getDataStreams = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, template: TemplateRef ): Promise => { const { templateName, indexTemplate } = template; - const res = await callCluster('transport.request', { - method: 'GET', - path: `/_data_stream/${templateName}-*`, - }); - const dataStreams = res.data_streams; + const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` }); + const dataStreams = body.data_streams; if (!dataStreams.length) return; return dataStreams.map((dataStream: any) => ({ dataStreamName: dataStream.name, @@ -468,22 +466,22 @@ const getDataStreams = async ( const updateAllDataStreams = async ( indexNameWithTemplates: CurrentDataStream[], - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise => { const updatedataStreamPromises = indexNameWithTemplates.map( ({ dataStreamName, indexTemplate }) => { - return updateExistingDataStream({ dataStreamName, callCluster, indexTemplate }); + return updateExistingDataStream({ dataStreamName, esClient, indexTemplate }); } ); await Promise.all(updatedataStreamPromises); }; const updateExistingDataStream = async ({ dataStreamName, - callCluster, + esClient, indexTemplate, }: { dataStreamName: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; indexTemplate: IndexTemplate; }) => { const { settings, mappings } = indexTemplate.template; @@ -496,7 +494,7 @@ const updateExistingDataStream = async ({ // try to update the mappings first try { - await callCluster('indices.putMapping', { + await esClient.indices.putMapping({ index: dataStreamName, body: mappings, write_index_only: true, @@ -505,7 +503,7 @@ const updateExistingDataStream = async ({ } catch (err) { try { const path = `/${dataStreamName}/_rollover`; - await callCluster('transport.request', { + await esClient.transport.request({ method: 'POST', path, }); @@ -518,7 +516,7 @@ const updateExistingDataStream = async ({ // for now, only update the pipeline if (!settings.index.default_pipeline) return; try { - await callCluster('indices.putSettings', { + await esClient.indices.putSettings({ index: dataStreamName, body: { index: { default_pipeline: settings.index.default_pipeline } }, }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 16b543846baee..cbd09b8d1e7a8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -5,16 +5,15 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'kibana/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { saveInstalledEsRefs } from '../../packages/install'; import { getPathParts } from '../../archive'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; import type { EsAssetReference, InstallablePackage } from '../../../../../common/types/models'; -import type { CallESAsCurrentUser } from '../../../../types'; import { getInstallation } from '../../packages'; import { appContextService } from '../../../app_context'; -import { isLegacyESClientError } from '../../../../errors'; import { deleteTransforms, deleteTransformRefs } from './remove'; import { getAsset } from './common'; @@ -27,7 +26,7 @@ interface TransformInstallation { export const installTransform = async ( installablePackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { const logger = appContextService.getLogger(); @@ -51,7 +50,7 @@ export const installTransform = async ( // delete all previous transform await deleteTransforms( - callCluster, + esClient, previousInstalledTransformEsAssets.map((asset) => asset.id) ); @@ -83,7 +82,7 @@ export const installTransform = async ( }); const installationPromises = transforms.map(async (transform) => { - return handleTransformInstall({ callCluster, transform }); + return handleTransformInstall({ esClient, transform }); }); installedTransforms = await Promise.all(installationPromises).then((results) => results.flat()); @@ -113,34 +112,32 @@ const isTransform = (path: string) => { }; async function handleTransformInstall({ - callCluster, + esClient, transform, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; transform: TransformInstallation; }): Promise { try { // defer validation on put if the source index is not available - await callCluster('transport.request', { - method: 'PUT', - path: `/_transform/${transform.installationName}`, - query: 'defer_validation=true', + await esClient.transform.putTransform({ + transform_id: transform.installationName, + defer_validation: true, body: transform.content, }); } catch (err) { // swallow the error if the transform already exists. const isAlreadyExistError = - isLegacyESClientError(err) && err?.body?.error?.type === 'resource_already_exists_exception'; + err instanceof ResponseError && + err?.body?.error?.type === 'resource_already_exists_exception'; if (!isAlreadyExistError) { throw err; } } - await callCluster('transport.request', { - method: 'POST', - path: `/_transform/${transform.installationName}/_start`, - // Ignore error if the transform is already started - ignore: [409], - }); + await esClient.transform.startTransform( + { transform_id: transform.installationName }, + { ignore: [409] } + ); return { id: transform.installationName, type: ElasticsearchAssetType.transform }; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts index 2611978be20a2..248c03e43add9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts @@ -5,64 +5,62 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'kibana/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType } from '../../../../types'; -import type { CallESAsCurrentUser, EsAssetReference } from '../../../../types'; +import type { EsAssetReference } from '../../../../types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants'; import { appContextService } from '../../../app_context'; -export const stopTransforms = async (transformIds: string[], callCluster: CallESAsCurrentUser) => { +export const stopTransforms = async (transformIds: string[], esClient: ElasticsearchClient) => { for (const transformId of transformIds) { - await callCluster('transport.request', { - method: 'POST', - path: `/_transform/${transformId}/_stop`, - query: 'force=true', - ignore: [404], - }); + await esClient.transform.stopTransform( + { transform_id: transformId, force: true }, + { ignore: [404] } + ); } }; -export const deleteTransforms = async ( - callCluster: CallESAsCurrentUser, - transformIds: string[] -) => { +export const deleteTransforms = async (esClient: ElasticsearchClient, transformIds: string[]) => { const logger = appContextService.getLogger(); if (transformIds.length) { logger.info(`Deleting currently installed transform ids ${transformIds}`); } await Promise.all( transformIds.map(async (transformId) => { - // get the index the transform - const transformResponse: { + interface TransformResponse { count: number; transforms?: Array<{ dest: { index: string; }; }>; - } = await callCluster('transport.request', { - method: 'GET', - path: `/_transform/${transformId}`, - ignore: [404], - }); + } + + // get the index the transform + const { body: transformResponse } = await esClient.transform.getTransform( + { transform_id: transformId }, + { ignore: [404] } + ); - await stopTransforms([transformId], callCluster); - await callCluster('transport.request', { - method: 'DELETE', - query: 'force=true', - path: `/_transform/${transformId}`, - ignore: [404], - }); + await stopTransforms([transformId], esClient); + await esClient.transform.deleteTransform( + { force: true, transform_id: transformId }, + { ignore: [404] } + ); logger.info(`Deleted: ${transformId}`); if (transformResponse?.transforms) { // expect this to be 1 for (const transform of transformResponse.transforms) { - await callCluster('transport.request', { - method: 'DELETE', - path: `/${transform?.dest?.index}`, - ignore: [404], - }); + await esClient.transport.request( + { + method: 'DELETE', + path: `/${transform?.dest?.index}`, + }, + { + ignore: [404], + } + ); } } else { logger.warn(`cannot find transform for ${transformId}`); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 254196b4af053..32e062289f994 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -18,33 +18,28 @@ jest.mock('./common', () => { }; }); -import { errors as LegacyESErrors } from 'elasticsearch'; - -import type { - ILegacyScopedClusterClient, - SavedObject, - SavedObjectsClientContract, -} from 'kibana/server'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { DeeplyMockedKeys } from 'packages/kbn-utility-types/target/jest'; +import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType } from '../../../../types'; import type { Installation, RegistryPackage } from '../../../../types'; import { getInstallation, getInstallationObject } from '../../packages'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from '../../../../../../../../src/core/server/elasticsearch/client/mocks'; import { appContextService } from '../../../app_context'; import { getAsset } from './common'; import { installTransform } from './install'; describe('test transform install', () => { - let legacyScopedClusterClient: jest.Mocked; + let esClient: DeeplyMockedKeys; let savedObjectsClient: jest.Mocked; beforeEach(() => { appContextService.start(createAppContextStartContractMock()); - legacyScopedClusterClient = { - callAsInternalUser: jest.fn(), - callAsCurrentUser: jest.fn(), - }; + esClient = elasticsearchClientMock.createClusterClient().asInternalUser; (getInstallation as jest.MockedFunction).mockReset(); (getInstallationObject as jest.MockedFunction).mockReset(); savedObjectsClient = savedObjectsClientMock.create(); @@ -106,8 +101,8 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce( - Promise.resolve({ + esClient.transform.getTransform.mockReturnValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ { @@ -116,15 +111,9 @@ describe('test transform install', () => { }, }, ], - } as { - count: number; - transforms: Array<{ - dest: { - index: string; - }; - }>; }) ); + await installTransform( ({ name: 'endpoint', @@ -165,78 +154,75 @@ describe('test transform install', () => { 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', ], - legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transform.getTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'GET', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', - ignore: [404], + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', }, + { ignore: [404] }, ], + ]); + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0/_stop', - query: 'force=true', - ignore: [404], + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + force: true, }, + { ignore: [404] }, ], + ]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'DELETE', - query: 'force=true', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', - ignore: [404], + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + force: true, }, + { ignore: [404] }, ], + ]); + + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'DELETE', path: '/index', - ignore: [404], }, + { ignore: [404] }, ], + ]); + + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'PUT', - path: '/_transform/endpoint.metadata-default-0.16.0-dev.0', - query: 'defer_validation=true', + transform_id: 'endpoint.metadata-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], [ - 'transport.request', { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], + ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'POST', - path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start', - ignore: [409], + transform_id: 'endpoint.metadata-default-0.16.0-dev.0', }, + { ignore: [409] }, ], [ - 'transport.request', { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', }, + { ignore: [409] }, ], ]); @@ -315,7 +301,7 @@ describe('test transform install', () => { attributes: { installed_es: [] }, } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser = jest.fn(); + await installTransform( ({ name: 'endpoint', @@ -338,29 +324,28 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], - legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], + ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', }, + { ignore: [409] }, ], ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', @@ -400,8 +385,8 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce( - Promise.resolve({ + esClient.transform.getTransform.mockReturnValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ { @@ -410,15 +395,9 @@ describe('test transform install', () => { }, }, ], - } as { - count: number; - transforms: Array<{ - dest: { - index: string; - }; - }>; }) ); + await installTransform( ({ name: 'endpoint', @@ -455,46 +434,49 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, [], - legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transform.getTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'GET', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', - ignore: [404], + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', }, + { ignore: [404] }, ], + ]); + + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'POST', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0/_stop', - query: 'force=true', - ignore: [404], + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + force: true, }, + { ignore: [404] }, ], + ]); + + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'DELETE', - query: 'force=true', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', - ignore: [404], + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + force: true, }, + { ignore: [404] }, ], + ]); + + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'DELETE', path: '/index', - ignore: [404], }, + { ignore: [404] }, ], ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', @@ -533,23 +515,18 @@ describe('test transform install', () => { attributes: { installed_es: [] }, } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser = jest.fn(); - - legacyScopedClusterClient.callAsCurrentUser.mockImplementation( - async (endpoint, clientParams, options) => { - if ( - endpoint === 'transport.request' && - clientParams?.method === 'PUT' && - clientParams?.path === '/_transform/endpoint.metadata_current-default-0.16.0-dev.0' - ) { - const err: LegacyESErrors._Abstract & { body?: any } = new LegacyESErrors.BadRequest(); - err.body = { - error: { type: 'resource_already_exists_exception' }, - }; - throw err; - } - } + + esClient.transport.request.mockImplementationOnce(() => + elasticsearchClientMock.createErrorTransportRequestPromise( + new ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { error: { type: 'resource_already_exists_exception' } }, + }) + ) + ) ); + await installTransform( ({ name: 'endpoint', @@ -572,29 +549,23 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], - legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ - 'transport.request', { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], - [ - 'transport.request', - { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], - }, - ], ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [{ transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0' }, { ignore: [409] }], + ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 97c29ebff145a..7996cbfb79ef8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,8 +5,7 @@ * 2.0. */ -import { LegacyScopedClusterClient } from 'src/core/server'; -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from 'src/core/server/mocks'; import { appContextService } from '../../app_context'; @@ -40,10 +39,11 @@ function sleep(millis: number) { describe('_installPackage', () => { let soClient: jest.Mocked; - let callCluster: jest.Mocked; + let esClient: jest.Mocked; + beforeEach(async () => { soClient = savedObjectsClientMock.create(); - callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; appContextService.start(createAppContextStartContractMock()); }); afterEach(async () => { @@ -65,7 +65,7 @@ describe('_installPackage', () => { const installationPromise = _installPackage({ savedObjectsClient: soClient, - callCluster, + esClient, paths: [], packageInfo: { title: 'title', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 2774149f252e3..ca9d490609636 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -5,18 +5,13 @@ * 2.0. */ -import type { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { ElasticsearchAssetType } from '../../../types'; -import type { - AssetReference, - Installation, - CallESAsCurrentUser, - InstallType, -} from '../../../types'; +import type { AssetReference, Installation, InstallType } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import { installTemplates } from '../elasticsearch/template/install'; import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/'; @@ -37,7 +32,7 @@ import { deleteKibanaSavedObjectsAssets } from './remove'; export async function _installPackage({ savedObjectsClient, - callCluster, + esClient, installedPkg, paths, packageInfo, @@ -45,7 +40,7 @@ export async function _installPackage({ installSource, }: { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; installedPkg?: SavedObject; paths: string[]; packageInfo: InstallablePackage; @@ -131,12 +126,12 @@ export async function _installPackage({ // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified // per data stream and we should then save them - await installILMPolicy(paths, callCluster); + await installILMPolicy(paths, esClient); const installedDataStreamIlm = await installIlmForDataStream( packageInfo, paths, - callCluster, + esClient, savedObjectsClient ); @@ -144,31 +139,31 @@ export async function _installPackage({ const installedPipelines = await installPipelines( packageInfo, paths, - callCluster, + esClient, savedObjectsClient ); // install or update the templates referencing the newly installed pipelines const installedTemplates = await installTemplates( packageInfo, - callCluster, + esClient, paths, savedObjectsClient ); // update current backing indices of each data stream - await updateCurrentWriteIndices(callCluster, installedTemplates); + await updateCurrentWriteIndices(esClient, installedTemplates); const installedTransforms = await installTransform( packageInfo, paths, - callCluster, + esClient, savedObjectsClient ); // if this is an update or retrying an update, delete the previous version's pipelines if ((installType === 'update' || installType === 'reupdate') && installedPkg) { await deletePreviousPipelines( - callCluster, + esClient, savedObjectsClient, pkgName, installedPkg.attributes.version @@ -177,7 +172,7 @@ export async function _installPackage({ // pipelines from a different version may have installed during a failed update if (installType === 'rollback' && installedPkg) { await deletePreviousPipelines( - callCluster, + esClient, savedObjectsClient, pkgName, installedPkg.attributes.install_version diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index e6df3d5e7a83d..b726c60fc1e5e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -5,9 +5,8 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import type { CallESAsCurrentUser } from '../../../types'; import * as Registry from '../registry'; import { getInstallationObject } from './index'; @@ -17,13 +16,13 @@ import type { BulkInstallResponse, IBulkInstallPackageError } from './install'; interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; packagesToUpgrade: string[]; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; } export async function bulkInstallPackages({ savedObjectsClient, packagesToUpgrade, - callCluster, + esClient, }: BulkInstallPackagesParams): Promise { const installedAndLatestPromises = packagesToUpgrade.map((pkgToUpgrade) => Promise.all([ @@ -38,7 +37,7 @@ export async function bulkInstallPackages({ const [installedPkg, latestPkg] = result.value; return upgradePackage({ savedObjectsClient, - callCluster, + esClient, installedPkg, latestPkg, pkgToUpgrade, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 5350d67b313c1..4d2a5528270e4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -9,7 +9,7 @@ import semverGt from 'semver/functions/gt'; import semverLt from 'semver/functions/lt'; import Boom from '@hapi/boom'; import type { UnwrapPromise } from '@kbn/utility-types'; -import type { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; import { generateESIndexPatterns } from '../elasticsearch/template/template'; @@ -25,7 +25,6 @@ import { KibanaAssetType } from '../../../types'; import type { AssetReference, Installation, - CallESAsCurrentUser, AssetType, EsAssetReference, InstallType, @@ -50,16 +49,21 @@ import { _installPackage } from './_install_package'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster } = options; + const { savedObjectsClient, pkgName, esClient } = options; try { const latestPackage = await Registry.fetchFindLatestPackage(pkgName); const pkgkey = Registry.pkgToPkgKey({ name: latestPackage.name, version: latestPackage.version, }); - return installPackage({ installSource: 'registry', savedObjectsClient, pkgkey, callCluster }); + return installPackage({ + installSource: 'registry', + savedObjectsClient, + pkgkey, + esClient, + }); } catch (err) { throw err; } @@ -67,13 +71,13 @@ export async function installLatestPackage(options: { export async function ensureInstalledDefaultPackages( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise { const installations = []; const bulkResponse = await bulkInstallPackages({ savedObjectsClient, packagesToUpgrade: Object.values(defaultPackages), - callCluster, + esClient, }); for (const resp of bulkResponse) { @@ -96,9 +100,9 @@ export async function ensureInstalledDefaultPackages( export async function ensureInstalledPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster } = options; + const { savedObjectsClient, pkgName, esClient } = options; const installedPackage = await getInstallation({ savedObjectsClient, pkgName }); if (installedPackage) { return installedPackage; @@ -107,7 +111,7 @@ export async function ensureInstalledPackage(options: { await installLatestPackage({ savedObjectsClient, pkgName, - callCluster, + esClient, }); const installation = await getInstallation({ savedObjectsClient, pkgName }); if (!installation) throw new Error(`could not get installation ${pkgName}`); @@ -120,14 +124,14 @@ export async function handleInstallPackageFailure({ pkgName, pkgVersion, installedPkg, - callCluster, + esClient, }: { savedObjectsClient: SavedObjectsClientContract; error: IngestManagerError | Boom.Boom | Error; pkgName: string; pkgVersion: string; installedPkg: SavedObject | undefined; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }) { if (error instanceof IngestManagerError) { return; @@ -143,7 +147,7 @@ export async function handleInstallPackageFailure({ const installType = getInstallType({ pkgVersion, installedPkg }); if (installType === 'install' || installType === 'reinstall') { logger.error(`uninstalling ${pkgkey} after error installing`); - await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); + await removeInstallation({ savedObjectsClient, pkgkey, esClient }); } if (installType === 'update') { @@ -159,7 +163,7 @@ export async function handleInstallPackageFailure({ installSource: 'registry', savedObjectsClient, pkgkey: prevVersion, - callCluster, + esClient, }); } } catch (e) { @@ -175,14 +179,14 @@ export type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageEr interface UpgradePackageParams { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; installedPkg: UnwrapPromise>; latestPkg: UnwrapPromise>; pkgToUpgrade: string; } export async function upgradePackage({ savedObjectsClient, - callCluster, + esClient, installedPkg, latestPkg, pkgToUpgrade, @@ -198,7 +202,7 @@ export async function upgradePackage({ installSource: 'registry', savedObjectsClient, pkgkey, - callCluster, + esClient, }); return { name: pkgToUpgrade, @@ -213,7 +217,7 @@ export async function upgradePackage({ pkgName: latestPkg.name, pkgVersion: latestPkg.version, installedPkg, - callCluster, + esClient, }); return { name: pkgToUpgrade, error: installFailed }; } @@ -234,14 +238,14 @@ export async function upgradePackage({ interface InstallRegistryPackageParams { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; force?: boolean; } async function installPackageFromRegistry({ savedObjectsClient, pkgkey, - callCluster, + esClient, force = false, }: InstallRegistryPackageParams): Promise { // TODO: change epm API to /packageName/version so we don't need to do this @@ -262,7 +266,7 @@ async function installPackageFromRegistry({ return _installPackage({ savedObjectsClient, - callCluster, + esClient, installedPkg, paths, packageInfo, @@ -273,7 +277,7 @@ async function installPackageFromRegistry({ interface InstallUploadedArchiveParams { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; archiveBuffer: Buffer; contentType: string; } @@ -284,7 +288,7 @@ export type InstallPackageParams = async function installPackageByUpload({ savedObjectsClient, - callCluster, + esClient, archiveBuffer, contentType, }: InstallUploadedArchiveParams): Promise { @@ -319,7 +323,7 @@ async function installPackageByUpload({ return _installPackage({ savedObjectsClient, - callCluster, + esClient, installedPkg, paths, packageInfo, @@ -334,20 +338,20 @@ export async function installPackage(args: InstallPackageParams) { } if (args.installSource === 'registry') { - const { savedObjectsClient, pkgkey, callCluster, force } = args; + const { savedObjectsClient, pkgkey, esClient, force } = args; return installPackageFromRegistry({ savedObjectsClient, pkgkey, - callCluster, + esClient, force, }); } else if (args.installSource === 'upload') { - const { savedObjectsClient, callCluster, archiveBuffer, contentType } = args; + const { savedObjectsClient, esClient, archiveBuffer, contentType } = args; return installPackageByUpload({ savedObjectsClient, - callCluster, + esClient, archiveBuffer, contentType, }); @@ -441,7 +445,7 @@ export const removeAssetsFromInstalledEsByType = async ( export async function ensurePackagesCompletedInstall( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) { const installingPackages = await getPackageSavedObjects(savedObjectsClient, { searchFields: ['install_status'], @@ -457,7 +461,12 @@ export async function ensurePackagesCompletedInstall( // reinstall package if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) { acc.push( - installPackage({ installSource: 'registry', savedObjectsClient, pkgkey, callCluster }) + installPackage({ + installSource: 'registry', + savedObjectsClient, + pkgkey, + esClient, + }) ); } return acc; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 0648404312ad7..da407c1d4cfa0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { SavedObjectsClientContract } from 'src/core/server'; +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import Boom from '@hapi/boom'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; @@ -13,7 +13,6 @@ import { ElasticsearchAssetType } from '../../../types'; import type { AssetReference, AssetType, - CallESAsCurrentUser, EsAssetReference, KibanaAssetReference, Installation, @@ -32,9 +31,9 @@ import { getInstallation, savedObjectTypes } from './index'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgkey, callCluster } = options; + const { savedObjectsClient, pkgkey, esClient } = options; // TODO: the epm api should change to /name/version so we don't need to do this const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installation = await getInstallation({ savedObjectsClient, pkgName }); @@ -55,7 +54,7 @@ export async function removeInstallation(options: { // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; - await deleteAssets(installation, savedObjectsClient, callCluster); + await deleteAssets(installation, savedObjectsClient, esClient); // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -88,17 +87,17 @@ function deleteKibanaAssets( }); } -function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallESAsCurrentUser) { +function deleteESAssets(installedObjects: EsAssetReference[], esClient: ElasticsearchClient) { return installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; if (assetType === ElasticsearchAssetType.ingestPipeline) { - return deletePipeline(callCluster, id); + return deletePipeline(esClient, id); } else if (assetType === ElasticsearchAssetType.indexTemplate) { - return deleteTemplate(callCluster, id); + return deleteTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { - return deleteTransforms(callCluster, [id]); + return deleteTransforms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { - return deleteIlms(callCluster, [id]); + return deleteIlms(esClient, [id]); } }); } @@ -106,12 +105,12 @@ function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallE async function deleteAssets( { installed_es: installedEs, installed_kibana: installedKibana }: Installation, savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) { const logger = appContextService.getLogger(); const deletePromises: Array> = [ - ...deleteESAssets(installedEs, callCluster), + ...deleteESAssets(installedEs, esClient), ...deleteKibanaAssets(installedKibana, savedObjectsClient), ]; @@ -125,25 +124,11 @@ async function deleteAssets( } } -async function deleteTemplate(callCluster: CallESAsCurrentUser, name: string): Promise { +async function deleteTemplate(esClient: ElasticsearchClient, name: string): Promise { // '*' shouldn't ever appear here, but it still would delete all templates if (name && name !== '*') { try { - const callClusterParams: { - method: string; - path: string; - ignore: number[]; - } = { - method: 'DELETE', - path: `/_index_template/${name}`, - ignore: [404], - }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + await esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }); } catch { throw new Error(`error deleting template ${name}`); } diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 54772096fa88f..877d332bc5680 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -37,7 +37,6 @@ import type { PackagePolicy, PackagePolicySOAttributes, RegistryPackage, - CallESAsCurrentUser, } from '../types'; import { ExternalCallback } from '..'; @@ -60,7 +59,6 @@ class PackagePolicyService { public async create( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, packagePolicy: NewPackagePolicy, options?: { id?: string; user?: AuthenticatedUser; bumpRevision?: boolean } ): Promise { @@ -94,7 +92,7 @@ class PackagePolicyService { ensureInstalledPackage({ savedObjectsClient: soClient, pkgName: packagePolicy.package.name, - callCluster, + esClient, }), getPackageInfo({ savedObjectsClient: soClient, diff --git a/x-pack/plugins/fleet/server/services/setup.test.ts b/x-pack/plugins/fleet/server/services/setup.test.ts index 8120e41ade606..212b0fabd26fb 100644 --- a/x-pack/plugins/fleet/server/services/setup.test.ts +++ b/x-pack/plugins/fleet/server/services/setup.test.ts @@ -44,7 +44,7 @@ describe('setupIngestManager', () => { soClient.update = mockedMethodThrowsError(); const esClient = context.core.elasticsearch.client.asCurrentUser; - const setupPromise = setupIngestManager(soClient, esClient, jest.fn()); + const setupPromise = setupIngestManager(soClient, esClient); await expect(setupPromise).rejects.toThrow('SO method mocked to throw'); await expect(setupPromise).rejects.toThrow(Error); }); @@ -57,7 +57,7 @@ describe('setupIngestManager', () => { soClient.update = mockedMethodThrowsCustom(); const esClient = context.core.elasticsearch.client.asCurrentUser; - const setupPromise = setupIngestManager(soClient, esClient, jest.fn()); + const setupPromise = setupIngestManager(soClient, esClient); await expect(setupPromise).rejects.toThrow('method mocked to throw'); await expect(setupPromise).rejects.toThrow(CustomTestError); }); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index d1f927ad076b0..d6bb04f5e572a 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -8,8 +8,6 @@ import uuid from 'uuid'; import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import type { CallESAsCurrentUser } from '../types'; - import { packageToPackagePolicy, DEFAULT_AGENT_POLICIES_PACKAGES, @@ -45,16 +43,14 @@ export interface SetupStatus { export async function setupIngestManager( soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise { - return awaitIfPending(async () => createSetupSideEffects(soClient, esClient, callCluster)); + return awaitIfPending(async () => createSetupSideEffects(soClient, esClient)); } async function createSetupSideEffects( soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise { const [ installedPackages, @@ -63,11 +59,11 @@ async function createSetupSideEffects( { created: defaultFleetServerPolicyCreated, policy: defaultFleetServerPolicy }, ] = await Promise.all([ // packages installed by default - ensureInstalledDefaultPackages(soClient, callCluster), + ensureInstalledDefaultPackages(soClient, esClient), outputService.ensureDefaultOutput(soClient), agentPolicyService.ensureDefaultAgentPolicy(soClient, esClient), agentPolicyService.ensureDefaultFleetServerAgentPolicy(soClient, esClient), - updateFleetRoleIfExists(callCluster), + updateFleetRoleIfExists(esClient), settingsService.getSettings(soClient).catch((e: any) => { if (e.isBoom && e.output.statusCode === 404) { const defaultSettings = createDefaultSettings(); @@ -84,21 +80,20 @@ async function createSetupSideEffects( // will occur between upgrading the package and reinstalling the previously failed package. // By moving this outside of the Promise.all, the upgrade will occur first, and then we'll attempt to reinstall any // packages that are stuck in the installing state. - await ensurePackagesCompletedInstall(soClient, callCluster); + await ensurePackagesCompletedInstall(soClient, esClient); await awaitIfFleetServerSetupPending(); const fleetServerPackage = await ensureInstalledPackage({ savedObjectsClient: soClient, pkgName: FLEET_SERVER_PACKAGE, - callCluster, + esClient, }); if (defaultFleetServerPolicyCreated) { await addPackageToAgentPolicy( soClient, esClient, - callCluster, fleetServerPackage, defaultFleetServerPolicy, defaultOutput @@ -142,7 +137,6 @@ async function createSetupSideEffects( await addPackageToAgentPolicy( soClient, esClient, - callCluster, installedPackage, agentPolicyWithPackagePolicies, defaultOutput @@ -156,27 +150,23 @@ async function createSetupSideEffects( return { isIntialized: true }; } -async function updateFleetRoleIfExists(callCluster: CallESAsCurrentUser) { +async function updateFleetRoleIfExists(esClient: ElasticsearchClient) { try { - await callCluster('transport.request', { - method: 'GET', - path: `/_security/role/${FLEET_ENROLL_ROLE}`, - }); + await esClient.security.getRole({ name: FLEET_ENROLL_ROLE }); } catch (e) { - if (e.status === 404) { + if (e.statusCode === 404) { return; } throw e; } - return putFleetRole(callCluster); + return putFleetRole(esClient); } -async function putFleetRole(callCluster: CallESAsCurrentUser) { - return callCluster('transport.request', { - method: 'PUT', - path: `/_security/role/${FLEET_ENROLL_ROLE}`, +async function putFleetRole(esClient: ElasticsearchClient) { + return await esClient.security.putRole({ + name: FLEET_ENROLL_ROLE, body: { cluster: ['monitor', 'manage_api_key'], indices: [ @@ -192,12 +182,11 @@ async function putFleetRole(callCluster: CallESAsCurrentUser) { export async function setupFleet( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, options?: { forceRecreate?: boolean } ) { // Create fleet_enroll role // This should be done directly in ES at some point - const res = await putFleetRole(callCluster); + const { body: res } = await putFleetRole(esClient); // If the role is already created skip the rest unless you have forceRecreate set to true if (options?.forceRecreate !== true && res.role.created === false) { @@ -205,9 +194,8 @@ export async function setupFleet( } const password = generateRandomPassword(); // Create fleet enroll user - await callCluster('transport.request', { - method: 'PUT', - path: `/_security/user/${FLEET_ENROLL_USERNAME}`, + await esClient.security.putUser({ + username: FLEET_ENROLL_USERNAME, body: { password, roles: [FLEET_ENROLL_ROLE], @@ -257,7 +245,6 @@ function generateRandomPassword() { async function addPackageToAgentPolicy( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, packageToInstall: Installation, agentPolicy: AgentPolicy, defaultOutput: Output @@ -275,7 +262,7 @@ async function addPackageToAgentPolicy( agentPolicy.namespace ); - await packagePolicyService.create(soClient, esClient, callCluster, newPackagePolicy, { + await packagePolicyService.create(soClient, esClient, newPackagePolicy, { bumpRevision: false, }); } diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index fda1568c56e0e..885809d767323 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -5,8 +5,6 @@ * 2.0. */ -import { LegacyScopedClusterClient } from 'src/core/server'; - export { // Object types Agent, @@ -86,8 +84,6 @@ export { FleetServerPolicy, } from '../../common'; -export type CallESAsCurrentUser = LegacyScopedClusterClient['callAsCurrentUser']; - export type AgentPolicyUpdateHandler = ( action: 'created' | 'updated' | 'deleted', agentPolicyId: string