From 2d004711d5908b0ad6252588d781ee6d01e7d0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 14:12:45 +0100 Subject: [PATCH 01/36] Migrate `deletePipeline` to new ESclient --- .../fleet/server/routes/epm/handlers.ts | 10 +++- .../elasticsearch/ingest_pipeline/remove.ts | 12 ++--- .../epm/packages/_install_package.test.ts | 10 +++- .../services/epm/packages/_install_package.ts | 8 +-- .../epm/packages/bulk_install_packages.ts | 5 +- .../server/services/epm/packages/install.ts | 53 +++++++++++++++---- .../server/services/epm/packages/remove.ts | 20 ++++--- .../fleet/server/services/package_policy.ts | 1 + x-pack/plugins/fleet/server/services/setup.ts | 5 +- 9 files changed, 93 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 47e53b27a600f..14f08ded49b44 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -225,6 +225,7 @@ export const installPackageFromRegistryHandler: RequestHandler< > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const { pkgkey } = request.params; const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); @@ -234,6 +235,7 @@ export const installPackageFromRegistryHandler: RequestHandler< savedObjectsClient, pkgkey, callCluster, + esClient, force: request.body?.force, }); const body: InstallPackageResponse = { @@ -249,6 +251,7 @@ export const installPackageFromRegistryHandler: RequestHandler< pkgVersion, installedPkg, callCluster, + esClient, }); return defaultResult; @@ -277,9 +280,11 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler< > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const bulkInstalledResponses = await bulkInstallPackages({ savedObjectsClient, callCluster, + esClient, packagesToUpgrade: request.body.packages, }); const payload = bulkInstalledResponses.map(bulkInstallServiceResponseToHttpEntry); @@ -302,6 +307,7 @@ export const installPackageByUploadHandler: RequestHandler< } const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later const archiveBuffer = Buffer.from(request.body); try { @@ -309,6 +315,7 @@ export const installPackageByUploadHandler: RequestHandler< installSource: 'upload', savedObjectsClient, callCluster, + esClient, archiveBuffer, contentType, }); @@ -328,7 +335,8 @@ export const deletePackageHandler: RequestHandler< const { pkgkey } = request.params; const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - const res = await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); + const esClient = context.core.elasticsearch.client.asCurrentUser; + const res = await removeInstallation({ savedObjectsClient, pkgkey, callCluster, esClient }); const body: DeletePackageResponse = { response: res, }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts index 4acc4767de525..1fe0937aae203 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts @@ -5,15 +5,15 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { appContextService } from '../../../'; -import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types'; +import { ElasticsearchAssetType } from '../../../../types'; import { IngestManagerError } from '../../../../errors'; import { getInstallation } from '../../packages/get'; import { PACKAGES_SAVED_OBJECT_TYPE, EsAssetReference } from '../../../../../common'; export const deletePreviousPipelines = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, pkgName: string, previousPkgVersion: string @@ -27,7 +27,7 @@ export const deletePreviousPipelines = async ( type === ElasticsearchAssetType.ingestPipeline && id.includes(previousPkgVersion) ); const deletePipelinePromises = installedPipelines.map(({ type, id }) => { - return deletePipeline(callCluster, id); + return deletePipeline(esClient, id); }); try { await Promise.all(deletePipelinePromises); @@ -56,11 +56,11 @@ export const deletePipelineRefs = async ( installed_es: filteredAssets, }); }; -export async function deletePipeline(callCluster: CallESAsCurrentUser, id: string): Promise { +export async function deletePipeline(esClient: ElasticsearchClient, id: string): Promise { // '*' shouldn't ever appear here, but it still would delete all ingest pipelines if (id && id !== '*') { try { - await callCluster('ingest.deletePipeline', { id }); + await esClient.ingest.deletePipeline({ id }); } catch (err) { // Only throw if error is not a 404 error. Sometimes the pipeline is already deleted, but we have // duplicate references to them, see https://github.com/elastic/kibana/issues/91192 diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index a55b763d7b2bb..6125f3c908fc4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { SavedObjectsClientContract, LegacyScopedClusterClient } from 'src/core/server'; +import { + SavedObjectsClientContract, + LegacyScopedClusterClient, + ElasticsearchClient, +} from 'src/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from 'src/core/server/mocks'; import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; @@ -38,9 +42,12 @@ function sleep(millis: number) { describe('_installPackage', () => { let soClient: jest.Mocked; let callCluster: jest.Mocked; + let esClient: jest.Mocked; + beforeEach(async () => { soClient = savedObjectsClientMock.create(); callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; appContextService.start(createAppContextStartContractMock()); }); afterEach(async () => { @@ -63,6 +70,7 @@ describe('_installPackage', () => { const installationPromise = _installPackage({ savedObjectsClient: soClient, callCluster, + esClient, paths: [], packageInfo: { title: 'title', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 2a6a4c93cb8e0..adf1014b5bcde 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; import { InstallablePackage, InstallSource, @@ -41,6 +41,7 @@ import { ConcurrentInstallOperationError } from '../../../errors'; export async function _installPackage({ savedObjectsClient, callCluster, + esClient, installedPkg, paths, packageInfo, @@ -49,6 +50,7 @@ export async function _installPackage({ }: { savedObjectsClient: SavedObjectsClientContract; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; installedPkg?: SavedObject; paths: string[]; packageInfo: InstallablePackage; @@ -171,7 +173,7 @@ export async function _installPackage({ // if this is an update or retrying an update, delete the previous version's pipelines if ((installType === 'update' || installType === 'reupdate') && installedPkg) { await deletePreviousPipelines( - callCluster, + esClient, savedObjectsClient, pkgName, installedPkg.attributes.version @@ -180,7 +182,7 @@ export async function _installPackage({ // pipelines from a different version may have installed during a failed update if (installType === 'rollback' && installedPkg) { await deletePreviousPipelines( - callCluster, + esClient, savedObjectsClient, pkgName, installedPkg.attributes.install_version diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index ac41f550c1c72..9d0941e1de1f2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { CallESAsCurrentUser } from '../../../types'; import * as Registry from '../registry'; import { getInstallationObject } from './index'; @@ -15,12 +15,14 @@ interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; packagesToUpgrade: string[]; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; } export async function bulkInstallPackages({ savedObjectsClient, packagesToUpgrade, callCluster, + esClient, }: BulkInstallPackagesParams): Promise { const installedAndLatestPromises = packagesToUpgrade.map((pkgToUpgrade) => Promise.all([ @@ -36,6 +38,7 @@ export async function bulkInstallPackages({ return upgradePackage({ savedObjectsClient, callCluster, + esClient, installedPkg, latestPkg, pkgToUpgrade, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index ac3c0f3952127..5eb665d20e671 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -9,7 +9,7 @@ import semverGt from 'semver/functions/gt'; import semverLt from 'semver/functions/lt'; import Boom from '@hapi/boom'; import { UnwrapPromise } from '@kbn/utility-types'; -import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; import { generateESIndexPatterns } from '../elasticsearch/template/template'; import { isRequiredPackage } from './index'; import { @@ -51,15 +51,22 @@ export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster } = options; + const { savedObjectsClient, pkgName, callCluster, esClient } = options; try { const latestPackage = await Registry.fetchFindLatestPackage(pkgName); const pkgkey = Registry.pkgToPkgKey({ name: latestPackage.name, version: latestPackage.version, }); - return installPackage({ installSource: 'registry', savedObjectsClient, pkgkey, callCluster }); + return installPackage({ + installSource: 'registry', + savedObjectsClient, + pkgkey, + callCluster, + esClient, + }); } catch (err) { throw err; } @@ -67,13 +74,15 @@ export async function installLatestPackage(options: { export async function ensureInstalledDefaultPackages( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient ): Promise { const installations = []; const bulkResponse = await bulkInstallPackages({ savedObjectsClient, packagesToUpgrade: Object.values(defaultPackages), callCluster, + esClient, }); for (const resp of bulkResponse) { @@ -97,8 +106,9 @@ export async function ensureInstalledPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster } = options; + const { savedObjectsClient, pkgName, callCluster, esClient } = options; const installedPackage = await getInstallation({ savedObjectsClient, pkgName }); if (installedPackage) { return installedPackage; @@ -108,6 +118,7 @@ export async function ensureInstalledPackage(options: { savedObjectsClient, pkgName, callCluster, + esClient, }); const installation = await getInstallation({ savedObjectsClient, pkgName }); if (!installation) throw new Error(`could not get installation ${pkgName}`); @@ -121,6 +132,7 @@ export async function handleInstallPackageFailure({ pkgVersion, installedPkg, callCluster, + esClient, }: { savedObjectsClient: SavedObjectsClientContract; error: IngestManagerError | Boom.Boom | Error; @@ -128,6 +140,7 @@ export async function handleInstallPackageFailure({ pkgVersion: string; installedPkg: SavedObject | undefined; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }) { if (error instanceof IngestManagerError) { return; @@ -143,7 +156,7 @@ export async function handleInstallPackageFailure({ const installType = getInstallType({ pkgVersion, installedPkg }); if (installType === 'install' || installType === 'reinstall') { logger.error(`uninstalling ${pkgkey} after error installing`); - await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); + await removeInstallation({ savedObjectsClient, pkgkey, callCluster, esClient }); } if (installType === 'update') { @@ -160,6 +173,7 @@ export async function handleInstallPackageFailure({ savedObjectsClient, pkgkey: prevVersion, callCluster, + esClient, }); } } catch (e) { @@ -176,6 +190,7 @@ export type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageEr interface UpgradePackageParams { savedObjectsClient: SavedObjectsClientContract; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; installedPkg: UnwrapPromise>; latestPkg: UnwrapPromise>; pkgToUpgrade: string; @@ -183,6 +198,7 @@ interface UpgradePackageParams { export async function upgradePackage({ savedObjectsClient, callCluster, + esClient, installedPkg, latestPkg, pkgToUpgrade, @@ -199,6 +215,7 @@ export async function upgradePackage({ savedObjectsClient, pkgkey, callCluster, + esClient, }); return { name: pkgToUpgrade, @@ -214,6 +231,7 @@ export async function upgradePackage({ pkgVersion: latestPkg.version, installedPkg, callCluster, + esClient, }); return { name: pkgToUpgrade, error: installFailed }; } @@ -235,6 +253,7 @@ interface InstallRegistryPackageParams { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; force?: boolean; } @@ -242,6 +261,7 @@ async function installPackageFromRegistry({ savedObjectsClient, pkgkey, callCluster, + esClient, force = false, }: InstallRegistryPackageParams): Promise { // TODO: change epm API to /packageName/version so we don't need to do this @@ -263,6 +283,7 @@ async function installPackageFromRegistry({ return _installPackage({ savedObjectsClient, callCluster, + esClient, installedPkg, paths, packageInfo, @@ -274,6 +295,7 @@ async function installPackageFromRegistry({ interface InstallUploadedArchiveParams { savedObjectsClient: SavedObjectsClientContract; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; archiveBuffer: Buffer; contentType: string; } @@ -285,6 +307,7 @@ export type InstallPackageParams = async function installPackageByUpload({ savedObjectsClient, callCluster, + esClient, archiveBuffer, contentType, }: InstallUploadedArchiveParams): Promise { @@ -320,6 +343,7 @@ async function installPackageByUpload({ return _installPackage({ savedObjectsClient, callCluster, + esClient, installedPkg, paths, packageInfo, @@ -334,20 +358,22 @@ export async function installPackage(args: InstallPackageParams) { } if (args.installSource === 'registry') { - const { savedObjectsClient, pkgkey, callCluster, force } = args; + const { savedObjectsClient, pkgkey, esClient, callCluster, force } = args; return installPackageFromRegistry({ savedObjectsClient, pkgkey, + esClient, callCluster, force, }); } else if (args.installSource === 'upload') { - const { savedObjectsClient, callCluster, archiveBuffer, contentType } = args; + const { savedObjectsClient, esClient, callCluster, archiveBuffer, contentType } = args; return installPackageByUpload({ savedObjectsClient, callCluster, + esClient, archiveBuffer, contentType, }); @@ -441,7 +467,8 @@ export const removeAssetsFromInstalledEsByType = async ( export async function ensurePackagesCompletedInstall( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient ) { const installingPackages = await getPackageSavedObjects(savedObjectsClient, { searchFields: ['install_status'], @@ -457,7 +484,13 @@ export async function ensurePackagesCompletedInstall( // reinstall package if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) { acc.push( - installPackage({ installSource: 'registry', savedObjectsClient, pkgkey, callCluster }) + installPackage({ + installSource: 'registry', + savedObjectsClient, + pkgkey, + esClient, + callCluster, + }) ); } return acc; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 78471f70adc3c..b91f7da87d273 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import Boom from '@hapi/boom'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { @@ -31,8 +31,9 @@ export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgkey, callCluster } = options; + const { savedObjectsClient, pkgkey, callCluster, esClient } = options; // TODO: the epm api should change to /name/version so we don't need to do this const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installation = await getInstallation({ savedObjectsClient, pkgName }); @@ -53,7 +54,7 @@ export async function removeInstallation(options: { // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; - await deleteAssets(installation, savedObjectsClient, callCluster); + await deleteAssets(installation, savedObjectsClient, callCluster, esClient); // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -86,11 +87,15 @@ function deleteKibanaAssets( }); } -function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallESAsCurrentUser) { +function deleteESAssets( + installedObjects: EsAssetReference[], + callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient +) { return installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; if (assetType === ElasticsearchAssetType.ingestPipeline) { - return deletePipeline(callCluster, id); + return deletePipeline(esClient, id); } else if (assetType === ElasticsearchAssetType.indexTemplate) { return deleteTemplate(callCluster, id); } else if (assetType === ElasticsearchAssetType.transform) { @@ -104,12 +109,13 @@ function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallE async function deleteAssets( { installed_es: installedEs, installed_kibana: installedKibana }: Installation, savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser + callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient ) { const logger = appContextService.getLogger(); const deletePromises: Array> = [ - ...deleteESAssets(installedEs, callCluster), + ...deleteESAssets(installedEs, callCluster, esClient), ...deleteKibanaAssets(installedKibana, savedObjectsClient), ]; diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 335cd7c956faf..20e697ea8f918 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -92,6 +92,7 @@ class PackagePolicyService { savedObjectsClient: soClient, pkgName: packagePolicy.package.name, callCluster, + esClient, }), getPackageInfo({ savedObjectsClient: soClient, diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 2a3166e9dc729..c78521648c476 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -63,7 +63,7 @@ async function createSetupSideEffects( { created: defaultFleetServerPolicyCreated, policy: defaultFleetServerPolicy }, ] = await Promise.all([ // packages installed by default - ensureInstalledDefaultPackages(soClient, callCluster), + ensureInstalledDefaultPackages(soClient, callCluster, esClient), outputService.ensureDefaultOutput(soClient), agentPolicyService.ensureDefaultAgentPolicy(soClient, esClient), isFleetServerEnabled @@ -86,7 +86,7 @@ async function createSetupSideEffects( // will occur between upgrading the package and reinstalling the previously failed package. // By moving this outside of the Promise.all, the upgrade will occur first, and then we'll attempt to reinstall any // packages that are stuck in the installing state. - await ensurePackagesCompletedInstall(soClient, callCluster); + await ensurePackagesCompletedInstall(soClient, callCluster, esClient); if (isFleetServerEnabled) { await awaitIfFleetServerSetupPending(); @@ -95,6 +95,7 @@ async function createSetupSideEffects( savedObjectsClient: soClient, pkgName: FLEET_SERVER_PACKAGE, callCluster, + esClient, }); if (defaultFleetServerPolicyCreated) { From 0b52acca1c8013a315ac5b3da82934fdca04b373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 14:28:08 +0100 Subject: [PATCH 02/36] Migrate `installILMPolicy` to new client --- .../fleet/server/services/epm/elasticsearch/ilm/install.ts | 5 +++-- .../fleet/server/services/epm/packages/_install_package.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts index 644baa3a77e7d..496a5788b60bf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts @@ -5,10 +5,11 @@ * 2.0. */ +import { ElasticsearchClient } from 'kibana/server'; import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; -export async function installILMPolicy(paths: string[], callCluster: CallESAsCurrentUser) { +export async function installILMPolicy(paths: string[], esClient: ElasticsearchClient) { const ilmPaths = paths.filter((path) => isILMPolicy(path)); if (!ilmPaths.length) return; await Promise.all( @@ -17,7 +18,7 @@ export async function installILMPolicy(paths: string[], callCluster: CallESAsCur const { file } = getPathParts(path); const name = file.substr(0, file.lastIndexOf('.')); try { - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: '/_ilm/policy/' + name, body, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index adf1014b5bcde..c1530c38156f8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -136,7 +136,7 @@ export async function _installPackage({ // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified // per data stream and we should then save them - await installILMPolicy(paths, callCluster); + await installILMPolicy(paths, esClient); const installedDataStreamIlm = await installIlmForDataStream( packageInfo, From b6af091aed3e70beb6cefce86b0bc36575ec9975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 14:31:39 +0100 Subject: [PATCH 03/36] Remove dead function --- .../services/epm/elasticsearch/ilm/install.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts index 496a5788b60bf..d3e7a638dc46f 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts @@ -6,7 +6,7 @@ */ import { ElasticsearchClient } from 'kibana/server'; -import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types'; +import { ElasticsearchAssetType } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; export async function installILMPolicy(paths: string[], esClient: ElasticsearchClient) { @@ -29,19 +29,8 @@ export async function installILMPolicy(paths: string[], esClient: ElasticsearchC }) ); } + const isILMPolicy = (path: string) => { const pathParts = getPathParts(path); return pathParts.type === ElasticsearchAssetType.ilmPolicy; }; -export async function policyExists( - name: string, - callCluster: CallESAsCurrentUser -): Promise { - const response = await callCluster('transport.request', { - method: 'GET', - path: '/_ilm/policy/?filter_path=' + name, - }); - - // If the response contains a key, it means the policy exists - return Object.keys(response).length > 0; -} From c3add82ba3363ccbfeadef656c487deabfcabf95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 14:38:39 +0100 Subject: [PATCH 04/36] Migrate `deleteIlms` to new client --- .../elasticsearch/datastream_ilm/install.ts | 5 +++-- .../elasticsearch/datastream_ilm/remove.ts | 20 +++++++++++-------- .../services/epm/packages/_install_package.ts | 1 + .../server/services/epm/packages/remove.ts | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts index a18c2710ce776..1dd8338f94d6e 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType, EsAssetReference, @@ -32,6 +32,7 @@ export const installIlmForDataStream = async ( registryPackage: InstallablePackage, paths: string[], callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name }); @@ -44,7 +45,7 @@ export const installIlmForDataStream = async ( // delete all previous ilm await deleteIlms( - callCluster, + esClient, previousInstalledIlmEsAssets.map((asset) => asset.id) ); // install the latest dataset diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts index 6f0cd6ecc935f..f2f630d389bf1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts @@ -5,18 +5,22 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; -import { CallESAsCurrentUser, ElasticsearchAssetType, EsAssetReference } from '../../../../types'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchAssetType, EsAssetReference } from '../../../../types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants'; -export const deleteIlms = async (callCluster: CallESAsCurrentUser, ilmPolicyIds: string[]) => { +export const deleteIlms = async (esClient: ElasticsearchClient, ilmPolicyIds: string[]) => { await Promise.all( ilmPolicyIds.map(async (ilmPolicyId) => { - await callCluster('transport.request', { - method: 'DELETE', - path: `_ilm/policy/${ilmPolicyId}`, - ignore: [404, 400], - }); + await esClient.transport.request( + { + method: 'DELETE', + path: `_ilm/policy/${ilmPolicyId}`, + }, + { + ignore: [404, 400], + } + ); }) ); }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index c1530c38156f8..ef2b99096a5c2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -142,6 +142,7 @@ export async function _installPackage({ packageInfo, paths, callCluster, + esClient, savedObjectsClient ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index b91f7da87d273..342f52e12d2b7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -101,7 +101,7 @@ function deleteESAssets( } else if (assetType === ElasticsearchAssetType.transform) { return deleteTransforms(callCluster, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { - return deleteIlms(callCluster, [id]); + return deleteIlms(esClient, [id]); } }); } From 75be7c290daed962efaf18e80d9313b86aac6eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 14:41:54 +0100 Subject: [PATCH 05/36] Migrate `deleteTemplate` to new client --- .../fleet/server/services/epm/packages/remove.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 342f52e12d2b7..e64c5391844c6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -97,7 +97,7 @@ function deleteESAssets( if (assetType === ElasticsearchAssetType.ingestPipeline) { return deletePipeline(esClient, id); } else if (assetType === ElasticsearchAssetType.indexTemplate) { - return deleteTemplate(callCluster, id); + return deleteTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { return deleteTransforms(callCluster, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { @@ -129,25 +129,23 @@ async function deleteAssets( } } -async function deleteTemplate(callCluster: CallESAsCurrentUser, name: string): Promise { +async function deleteTemplate(esClient: ElasticsearchClient, name: string): Promise { // '*' shouldn't ever appear here, but it still would delete all templates if (name && name !== '*') { try { - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; } = { method: 'DELETE', path: `/_index_template/${name}`, - ignore: [404], }; // This uses the catch-all endpoint 'transport.request' because there is no // convenience endpoint using the new _index_template API yet. // The existing convenience endpoint `indices.putTemplate` only sends to _template, // which does not support v2 templates. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + await esClient.transport.request(esClientParams, { ignore: [404] }); } catch { throw new Error(`error deleting template ${name}`); } From f8ca0f356e36a84199af0867f7dbfb9769a006b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 15:02:29 +0100 Subject: [PATCH 06/36] Migrate `deleteTransports` to new client --- .../epm/elasticsearch/transform/install.ts | 5 +- .../epm/elasticsearch/transform/remove.ts | 78 +++++++++++-------- .../elasticsearch/transform/transform.test.ts | 15 +++- .../services/epm/packages/_install_package.ts | 1 + .../server/services/epm/packages/remove.ts | 2 +- 5 files changed, 66 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 948a9c56746f3..f55f1cddf4653 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; import { saveInstalledEsRefs } from '../../packages/install'; import { getPathParts } from '../../archive'; @@ -30,6 +30,7 @@ export const installTransform = async ( installablePackage: InstallablePackage, paths: string[], callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { const logger = appContextService.getLogger(); @@ -53,7 +54,7 @@ export const installTransform = async ( // delete all previous transform await deleteTransforms( - callCluster, + esClient, previousInstalledTransformEsAssets.map((asset) => asset.id) ); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts index 0e947e0f0b90b..3d1b7c6ed59ed 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts @@ -5,26 +5,27 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; -import { CallESAsCurrentUser, ElasticsearchAssetType, EsAssetReference } from '../../../../types'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ElasticsearchAssetType, EsAssetReference } from '../../../../types'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants'; import { appContextService } from '../../../app_context'; -export const stopTransforms = async (transformIds: string[], callCluster: CallESAsCurrentUser) => { +export const stopTransforms = async (transformIds: string[], esClient: ElasticsearchClient) => { for (const transformId of transformIds) { - await callCluster('transport.request', { - method: 'POST', - path: `/_transform/${transformId}/_stop`, - query: 'force=true', - ignore: [404], - }); + await esClient.transport.request( + { + method: 'POST', + path: `/_transform/${transformId}/_stop`, + querystring: 'force=true', + }, + { + ignore: [404], + } + ); } }; -export const deleteTransforms = async ( - callCluster: CallESAsCurrentUser, - transformIds: string[] -) => { +export const deleteTransforms = async (esClient: ElasticsearchClient, transformIds: string[]) => { const logger = appContextService.getLogger(); if (transformIds.length) { logger.info(`Deleting currently installed transform ids ${transformIds}`); @@ -32,35 +33,50 @@ export const deleteTransforms = async ( await Promise.all( transformIds.map(async (transformId) => { // get the index the transform - const transformResponse: { + const { body } = await esClient.transport.request( + { + method: 'GET', + path: `/_transform/${transformId}`, + }, + { + ignore: [404], + } + ); + + // `transport.request` doesn't accept generics, so we cast the type here. + const transformResponse = body as { count: number; transforms?: Array<{ dest: { index: string; }; }>; - } = await callCluster('transport.request', { - method: 'GET', - path: `/_transform/${transformId}`, - ignore: [404], - }); + }; - await stopTransforms([transformId], callCluster); - await callCluster('transport.request', { - method: 'DELETE', - query: 'force=true', - path: `/_transform/${transformId}`, - ignore: [404], - }); + await stopTransforms([transformId], esClient); + await esClient.transport.request( + { + method: 'DELETE', + querystring: 'force=true', + path: `/_transform/${transformId}`, + }, + { + ignore: [404], + } + ); logger.info(`Deleted: ${transformId}`); if (transformResponse?.transforms) { // expect this to be 1 for (const transform of transformResponse.transforms) { - await callCluster('transport.request', { - method: 'DELETE', - path: `/${transform?.dest?.index}`, - ignore: [404], - }); + await esClient.transport.request( + { + method: 'DELETE', + path: `/${transform?.dest?.index}`, + }, + { + ignore: [404], + } + ); } } else { logger.warn(`cannot find transform for ${transformId}`); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index bd944391b5f23..977a52ed6d5af 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -19,16 +19,24 @@ jest.mock('./common', () => { import { errors as LegacyESErrors } from 'elasticsearch'; import { installTransform } from './install'; -import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server'; +import { + ElasticsearchClient, + ILegacyScopedClusterClient, + SavedObject, + SavedObjectsClientContract, +} from 'kibana/server'; import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types'; import { getInstallation, getInstallationObject } from '../../packages'; import { getAsset } from './common'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from '../../../../../../../../src/core/server/elasticsearch/client/mocks'; import { appContextService } from '../../../app_context'; describe('test transform install', () => { let legacyScopedClusterClient: jest.Mocked; + let esClient: jest.Mocked; let savedObjectsClient: jest.Mocked; beforeEach(() => { appContextService.start(createAppContextStartContractMock()); @@ -36,6 +44,7 @@ describe('test transform install', () => { callAsInternalUser: jest.fn(), callAsCurrentUser: jest.fn(), }; + esClient = elasticsearchClientMock.createClusterClient().asInternalUser; (getInstallation as jest.MockedFunction).mockReset(); (getInstallationObject as jest.MockedFunction).mockReset(); savedObjectsClient = savedObjectsClientMock.create(); @@ -157,6 +166,7 @@ describe('test transform install', () => { 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', ], legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); @@ -330,6 +340,7 @@ describe('test transform install', () => { } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); @@ -447,6 +458,7 @@ describe('test transform install', () => { } as unknown) as RegistryPackage, [], legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); @@ -564,6 +576,7 @@ describe('test transform install', () => { } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], legacyScopedClusterClient.callAsCurrentUser, + esClient, savedObjectsClient ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index ef2b99096a5c2..cfbba8ed65e26 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -168,6 +168,7 @@ export async function _installPackage({ packageInfo, paths, callCluster, + esClient, savedObjectsClient ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index e64c5391844c6..0e1046ff82260 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -99,7 +99,7 @@ function deleteESAssets( } else if (assetType === ElasticsearchAssetType.indexTemplate) { return deleteTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { - return deleteTransforms(callCluster, [id]); + return deleteTransforms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { return deleteIlms(esClient, [id]); } From 5e553a0cd6154740bb7b38ff63cd10814ecf1725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 15:06:12 +0100 Subject: [PATCH 07/36] Remove old client from `deleteAssets` --- .../fleet/server/services/epm/packages/remove.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 0e1046ff82260..042350541c9e8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -54,7 +54,7 @@ export async function removeInstallation(options: { // Delete the installed assets. Don't include installation.package_assets. Those are irrelevant to users const installedAssets = [...installation.installed_kibana, ...installation.installed_es]; - await deleteAssets(installation, savedObjectsClient, callCluster, esClient); + await deleteAssets(installation, savedObjectsClient, esClient); // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -87,11 +87,7 @@ function deleteKibanaAssets( }); } -function deleteESAssets( - installedObjects: EsAssetReference[], - callCluster: CallESAsCurrentUser, - esClient: ElasticsearchClient -) { +function deleteESAssets(installedObjects: EsAssetReference[], esClient: ElasticsearchClient) { return installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; if (assetType === ElasticsearchAssetType.ingestPipeline) { @@ -109,13 +105,12 @@ function deleteESAssets( async function deleteAssets( { installed_es: installedEs, installed_kibana: installedKibana }: Installation, savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser, esClient: ElasticsearchClient ) { const logger = appContextService.getLogger(); const deletePromises: Array> = [ - ...deleteESAssets(installedEs, callCluster, esClient), + ...deleteESAssets(installedEs, esClient), ...deleteKibanaAssets(installedKibana, savedObjectsClient), ]; From 2c17bf825a3d3de6232cc6d139a4c4a6f7200995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 15:12:35 +0100 Subject: [PATCH 08/36] Remove old client from `removeInstallation` --- x-pack/plugins/fleet/server/routes/epm/handlers.ts | 3 +-- x-pack/plugins/fleet/server/services/epm/packages/install.ts | 2 +- x-pack/plugins/fleet/server/services/epm/packages/remove.ts | 4 +--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 14f08ded49b44..68ed803f0e18d 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -334,9 +334,8 @@ export const deletePackageHandler: RequestHandler< try { const { pkgkey } = request.params; const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const esClient = context.core.elasticsearch.client.asCurrentUser; - const res = await removeInstallation({ savedObjectsClient, pkgkey, callCluster, esClient }); + const res = await removeInstallation({ savedObjectsClient, pkgkey, esClient }); const body: DeletePackageResponse = { response: res, }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 5eb665d20e671..e512b019a8c0f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -156,7 +156,7 @@ export async function handleInstallPackageFailure({ const installType = getInstallType({ pkgVersion, installedPkg }); if (installType === 'install' || installType === 'reinstall') { logger.error(`uninstalling ${pkgkey} after error installing`); - await removeInstallation({ savedObjectsClient, pkgkey, callCluster, esClient }); + await removeInstallation({ savedObjectsClient, pkgkey, esClient }); } if (installType === 'update') { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 042350541c9e8..d0652ffcd837a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -11,7 +11,6 @@ import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE } from '.. import { AssetReference, AssetType, - CallESAsCurrentUser, ElasticsearchAssetType, EsAssetReference, KibanaAssetReference, @@ -30,10 +29,9 @@ import { removeArchiveEntries } from '../archive/storage'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgkey, callCluster, esClient } = options; + const { savedObjectsClient, pkgkey, esClient } = options; // TODO: the epm api should change to /name/version so we don't need to do this const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installation = await getInstallation({ savedObjectsClient, pkgName }); From 2e02613ac81aa488dd7a53b1960c872a252a1dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 15:28:49 +0100 Subject: [PATCH 09/36] Migrate `updateCurrentWriteIndices` to new client --- .../epm/elasticsearch/template/template.ts | 32 +++++++++---------- .../services/epm/packages/_install_package.ts | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index b86c989f8c24c..c04f263660cb2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -5,10 +5,10 @@ * 2.0. */ +import { ElasticsearchClient } from 'kibana/server'; import { Field, Fields } from '../../fields/field'; import { RegistryDataStream, - CallESAsCurrentUser, TemplateRef, IndexTemplate, IndexTemplateMappings, @@ -376,14 +376,14 @@ function getBaseTemplate( } export const updateCurrentWriteIndices = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, templates: TemplateRef[] ): Promise => { if (!templates.length) return; - const allIndices = await queryDataStreamsFromTemplates(callCluster, templates); + const allIndices = await queryDataStreamsFromTemplates(esClient, templates); if (!allIndices.length) return; - return updateAllDataStreams(allIndices, callCluster); + return updateAllDataStreams(allIndices, esClient); }; function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is CurrentDataStream[] { @@ -391,26 +391,26 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur } const queryDataStreamsFromTemplates = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, templates: TemplateRef[] ): Promise => { const dataStreamPromises = templates.map((template) => { - return getDataStreams(callCluster, template); + return getDataStreams(esClient, template); }); const dataStreamObjects = await Promise.all(dataStreamPromises); return dataStreamObjects.filter(isCurrentDataStream).flat(); }; const getDataStreams = async ( - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, template: TemplateRef ): Promise => { const { templateName, indexTemplate } = template; - const res = await callCluster('transport.request', { + const { body } = await esClient.transport.request({ method: 'GET', path: `/_data_stream/${templateName}-*`, }); - const dataStreams = res.data_streams; + const dataStreams = body.data_streams; if (!dataStreams.length) return; return dataStreams.map((dataStream: any) => ({ dataStreamName: dataStream.name, @@ -420,22 +420,22 @@ const getDataStreams = async ( const updateAllDataStreams = async ( indexNameWithTemplates: CurrentDataStream[], - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise => { const updatedataStreamPromises = indexNameWithTemplates.map( ({ dataStreamName, indexTemplate }) => { - return updateExistingDataStream({ dataStreamName, callCluster, indexTemplate }); + return updateExistingDataStream({ dataStreamName, esClient, indexTemplate }); } ); await Promise.all(updatedataStreamPromises); }; const updateExistingDataStream = async ({ dataStreamName, - callCluster, + esClient, indexTemplate, }: { dataStreamName: string; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; indexTemplate: IndexTemplate; }) => { const { settings, mappings } = indexTemplate.template; @@ -448,7 +448,7 @@ const updateExistingDataStream = async ({ // try to update the mappings first try { - await callCluster('indices.putMapping', { + await esClient.indices.putMapping({ index: dataStreamName, body: mappings, write_index_only: true, @@ -457,7 +457,7 @@ const updateExistingDataStream = async ({ } catch (err) { try { const path = `/${dataStreamName}/_rollover`; - await callCluster('transport.request', { + await esClient.transport.request({ method: 'POST', path, }); @@ -470,7 +470,7 @@ const updateExistingDataStream = async ({ // for now, only update the pipeline if (!settings.index.default_pipeline) return; try { - await callCluster('indices.putSettings', { + await esClient.indices.putSettings({ index: dataStreamName, body: { index: { default_pipeline: settings.index.default_pipeline } }, }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index cfbba8ed65e26..4dbb49c6c5033 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -162,7 +162,7 @@ export async function _installPackage({ ); // update current backing indices of each data stream - await updateCurrentWriteIndices(callCluster, installedTemplates); + await updateCurrentWriteIndices(esClient, installedTemplates); const installedTransforms = await installTransform( packageInfo, From 1c1a9d587b619e4ee522272a886ac06d53bfe8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 15:35:17 +0100 Subject: [PATCH 10/36] Remove old client from `installTransform` --- .../epm/elasticsearch/transform/install.ts | 28 ++++++++++--------- .../elasticsearch/transform/transform.test.ts | 4 --- .../services/epm/packages/_install_package.ts | 1 - 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index f55f1cddf4653..c0236f6c4cf05 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -14,7 +14,6 @@ import { EsAssetReference, InstallablePackage, } from '../../../../../common/types/models'; -import { CallESAsCurrentUser } from '../../../../types'; import { getInstallation } from '../../packages'; import { deleteTransforms, deleteTransformRefs } from './remove'; import { getAsset } from './common'; @@ -29,7 +28,6 @@ interface TransformInstallation { export const installTransform = async ( installablePackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { @@ -86,7 +84,7 @@ export const installTransform = async ( }); const installationPromises = transforms.map(async (transform) => { - return handleTransformInstall({ callCluster, transform }); + return handleTransformInstall({ esClient, transform }); }); installedTransforms = await Promise.all(installationPromises).then((results) => results.flat()); @@ -116,18 +114,18 @@ const isTransform = (path: string) => { }; async function handleTransformInstall({ - callCluster, + esClient, transform, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; transform: TransformInstallation; }): Promise { try { // defer validation on put if the source index is not available - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: `/_transform/${transform.installationName}`, - query: 'defer_validation=true', + querystring: 'defer_validation=true', body: transform.content, }); } catch (err) { @@ -138,12 +136,16 @@ async function handleTransformInstall({ throw err; } } - await callCluster('transport.request', { - method: 'POST', - path: `/_transform/${transform.installationName}/_start`, - // Ignore error if the transform is already started - ignore: [409], - }); + await esClient.transport.request( + { + method: 'POST', + path: `/_transform/${transform.installationName}/_start`, + // Ignore error if the transform is already started + }, + { + ignore: [409], + } + ); return { id: transform.installationName, type: ElasticsearchAssetType.transform }; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 977a52ed6d5af..416e154e5358c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -165,7 +165,6 @@ describe('test transform install', () => { 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', ], - legacyScopedClusterClient.callAsCurrentUser, esClient, savedObjectsClient ); @@ -339,7 +338,6 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], - legacyScopedClusterClient.callAsCurrentUser, esClient, savedObjectsClient ); @@ -457,7 +455,6 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, [], - legacyScopedClusterClient.callAsCurrentUser, esClient, savedObjectsClient ); @@ -575,7 +572,6 @@ describe('test transform install', () => { ], } as unknown) as RegistryPackage, ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], - legacyScopedClusterClient.callAsCurrentUser, esClient, savedObjectsClient ); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 4dbb49c6c5033..3f102e0d0c647 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -167,7 +167,6 @@ export async function _installPackage({ const installedTransforms = await installTransform( packageInfo, paths, - callCluster, esClient, savedObjectsClient ); From cf61b1f0a0615c881eeefd70b5af55f3fb7e7ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 16:12:56 +0100 Subject: [PATCH 11/36] Migrate `installTemplates` to new client --- .../elasticsearch/template/install.test.ts | 38 +++++---- .../epm/elasticsearch/template/install.ts | 81 +++++++++---------- .../services/epm/packages/_install_package.ts | 2 +- 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index cdcd3972fd189..8a44779b68427 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -12,15 +12,16 @@ import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { installTemplate } from './install'; test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.transport.request.mockImplementation((params) => { if ( params && params.method === 'GET' && params.path === '/_index_template/metrics-package.dataset' ) { - return { index_templates: [] }; + return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); } + return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); }); const fields: Field[] = []; @@ -40,7 +41,7 @@ test('tests installPackage to use correct priority and index_patterns for data s const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixUnset = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixUnset, packageVersion: pkg.version, @@ -54,15 +55,16 @@ test('tests installPackage to use correct priority and index_patterns for data s }); test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.transport.request.mockImplementation((params) => { if ( params && params.method === 'GET' && params.path === '/_index_template/metrics-package.dataset' ) { - return { index_templates: [] }; + return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); } + return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); }); const fields: Field[] = []; @@ -83,7 +85,7 @@ test('tests installPackage to use correct priority and index_patterns for data s const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixFalse = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixFalse, packageVersion: pkg.version, @@ -97,15 +99,16 @@ test('tests installPackage to use correct priority and index_patterns for data s }); test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.transport.request.mockImplementation((params) => { if ( params && params.method === 'GET' && params.path === '/_index_template/metrics-package.dataset' ) { - return { index_templates: [] }; + return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); } + return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); }); const fields: Field[] = []; @@ -126,7 +129,7 @@ test('tests installPackage to use correct priority and index_patterns for data s const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*'; const templatePriorityDatasetIsPrefixTrue = 150; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixTrue, packageVersion: pkg.version, @@ -140,14 +143,14 @@ test('tests installPackage to use correct priority and index_patterns for data s }); test('tests installPackage remove the aliases property if the property existed', async () => { - const callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; - callCluster.mockImplementation(async (_, params) => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.transport.request.mockImplementation((params) => { if ( params && params.method === 'GET' && params.path === '/_index_template/metrics-package.dataset' ) { - return { + return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [ { name: 'metrics-package.dataset', @@ -157,8 +160,9 @@ test('tests installPackage remove the aliases property if the property existed', }, }, ], - }; + }); } + return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); }); const fields: Field[] = []; @@ -178,7 +182,7 @@ test('tests installPackage remove the aliases property if the property existed', const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*'; const templatePriorityDatasetIsPrefixUnset = 200; await installTemplate({ - callCluster, + esClient, fields, dataStream: dataStreamDatasetIsPrefixUnset, packageVersion: pkg.version, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 70afa78e723bc..6c6d4cacd9195 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -6,7 +6,7 @@ */ import Boom from '@hapi/boom'; -import { SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { RegistryDataStream, ElasticsearchAssetType, @@ -14,7 +14,6 @@ import { RegistryElasticsearch, InstallablePackage, } from '../../../../types'; -import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { @@ -29,15 +28,15 @@ import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../pa export const installTemplates = async ( installablePackage: InstallablePackage, - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, paths: string[], savedObjectsClient: SavedObjectsClientContract ): Promise => { // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates - await installPreBuiltComponentTemplates(paths, callCluster); - await installPreBuiltTemplates(paths, callCluster); + await installPreBuiltComponentTemplates(paths, esClient); + await installPreBuiltTemplates(paths, esClient); // remove package installation's references to index templates await removeAssetsFromInstalledEsByType( @@ -63,7 +62,7 @@ export const installTemplates = async ( acc.push( installTemplateForDataStream({ pkg: installablePackage, - callCluster, + esClient, dataStream, }) ); @@ -80,7 +79,7 @@ export const installTemplates = async ( return []; }; -const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCurrentUser) => { +const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => { const templatePaths = paths.filter((path) => isTemplate(path)); const templateInstallPromises = templatePaths.map(async (path) => { const { file } = getPathParts(path); @@ -95,15 +94,13 @@ const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCu templateAPIPath = '_index_template'; } - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; body: any; } = { method: 'PUT', path: `/${templateAPIPath}/${templateName}`, - ignore: [404], body: content, }; // This uses the catch-all endpoint 'transport.request' because there is no @@ -111,7 +108,7 @@ const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCu // The existing convenience endpoint `indices.putTemplate` only sends to _template, // which does not support v2 templates. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return callCluster('transport.request', callClusterParams); + return esClient.transport.request(esClientParams, { ignore: [404] }); }); try { return await Promise.all(templateInstallPromises); @@ -124,7 +121,7 @@ const installPreBuiltTemplates = async (paths: string[], callCluster: CallESAsCu const installPreBuiltComponentTemplates = async ( paths: string[], - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) => { const templatePaths = paths.filter((path) => isComponentTemplate(path)); const templateInstallPromises = templatePaths.map(async (path) => { @@ -132,22 +129,21 @@ const installPreBuiltComponentTemplates = async ( const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; body: any; } = { method: 'PUT', path: `/_component_template/${templateName}`, - ignore: [404], body: content, }; // This uses the catch-all endpoint 'transport.request' because there is no // convenience endpoint for component templates yet. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return callCluster('transport.request', callClusterParams); + return esClient.transport.request(esClientParams, { ignore: [404] }); }); + try { return await Promise.all(templateInstallPromises); } catch (e) { @@ -175,16 +171,16 @@ const isComponentTemplate = (path: string) => { export async function installTemplateForDataStream({ pkg, - callCluster, + esClient, dataStream, }: { pkg: InstallablePackage; - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; dataStream: RegistryDataStream; }): Promise { const fields = await loadFieldsFromYaml(pkg, dataStream.path); return installTemplate({ - callCluster, + esClient, fields, dataStream, packageVersion: pkg.version, @@ -195,22 +191,23 @@ export async function installTemplateForDataStream({ function putComponentTemplate( body: object | undefined, name: string, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): { clusterPromise: Promise; name: string } | undefined { if (body) { - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; body: any; } = { method: 'PUT', path: `/_component_template/${name}`, - ignore: [404], body, }; - return { clusterPromise: callCluster('transport.request', callClusterParams), name }; + return { + clusterPromise: esClient.transport.request(esClientParams, { ignore: [404] }), + name, + }; } } @@ -250,7 +247,7 @@ function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | async function installDataStreamComponentTemplates( templateName: string, registryElasticsearch: RegistryElasticsearch | undefined, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ) { const templates: string[] = []; const componentPromises: Array> = []; @@ -260,13 +257,13 @@ async function installDataStreamComponentTemplates( const mappings = putComponentTemplate( compTemplates.mappingsTemplate, `${templateName}-mappings`, - callCluster + esClient ); const settings = putComponentTemplate( compTemplates.settingsTemplate, `${templateName}-settings`, - callCluster + esClient ); if (mappings) { @@ -285,13 +282,13 @@ async function installDataStreamComponentTemplates( } export async function installTemplate({ - callCluster, + esClient, fields, dataStream, packageVersion, packageName, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; fields: Field[]; dataStream: RegistryDataStream; packageVersion: string; @@ -312,11 +309,15 @@ export async function installTemplate({ } // Datastream now throw an error if the aliases field is present so ensure that we remove that field. - const getTemplateRes = await callCluster('transport.request', { - method: 'GET', - path: `/_index_template/${templateName}`, - ignore: [404], - }); + const { body: getTemplateRes } = await esClient.transport.request( + { + method: 'GET', + path: `/_index_template/${templateName}`, + }, + { + ignore: [404], + } + ); const existingIndexTemplate = getTemplateRes?.index_templates?.[0]; if ( @@ -327,12 +328,10 @@ export async function installTemplate({ const updateIndexTemplateParams: { method: string; path: string; - ignore: number[]; body: any; } = { method: 'PUT', path: `/_index_template/${templateName}`, - ignore: [404], body: { ...existingIndexTemplate.index_template, template: { @@ -347,13 +346,13 @@ export async function installTemplate({ // The existing convenience endpoint `indices.putTemplate` only sends to _template, // which does not support v2 templates. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', updateIndexTemplateParams); + await esClient.transport.request(updateIndexTemplateParams, { ignore: [404] }); } const composedOfTemplates = await installDataStreamComponentTemplates( templateName, dataStream.elasticsearch, - callCluster + esClient ); const template = getTemplate({ @@ -369,15 +368,13 @@ export async function installTemplate({ }); // TODO: Check return values for errors - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; body: any; } = { method: 'PUT', path: `/_index_template/${templateName}`, - ignore: [404], body: template, }; // This uses the catch-all endpoint 'transport.request' because there is no @@ -385,7 +382,7 @@ export async function installTemplate({ // The existing convenience endpoint `indices.putTemplate` only sends to _template, // which does not support v2 templates. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + await esClient.transport.request(esClientParams, { ignore: [404] }); return { templateName, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 3f102e0d0c647..c94d9be79c61f 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -156,7 +156,7 @@ export async function _installPackage({ // install or update the templates referencing the newly installed pipelines const installedTemplates = await installTemplates( packageInfo, - callCluster, + esClient, paths, savedObjectsClient ); From 63a963449b65447ecfaab66a07809dfe51cc3a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 16:46:27 +0100 Subject: [PATCH 12/36] Migrate `installPipelines` to new client --- .../elasticsearch/ingest_pipeline/install.ts | 32 ++++++++++--------- .../services/epm/packages/_install_package.ts | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 983e10e15bfff..34797364a9324 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'src/core/server'; +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { EsAssetReference, RegistryDataStream, @@ -13,7 +13,6 @@ import { InstallablePackage, } from '../../../../types'; import { ArchiveEntry, getAsset, getPathParts } from '../../archive'; -import { CallESAsCurrentUser } from '../../../../types'; import { saveInstalledEsRefs } from '../../packages/install'; import { getInstallationObject } from '../../packages'; import { deletePipelineRefs } from './remove'; @@ -27,7 +26,7 @@ interface RewriteSubstitution { export const installPipelines = async ( installablePackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, + esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { // unlike other ES assets, pipeline names are versioned so after a template is updated @@ -74,7 +73,7 @@ export const installPipelines = async ( acc.push( installPipelinesForDataStream({ dataStream, - callCluster, + esClient, paths: pipelinePaths, pkgVersion: installablePackage.version, }) @@ -107,12 +106,12 @@ export function rewriteIngestPipeline( } export async function installPipelinesForDataStream({ - callCluster, + esClient, pkgVersion, paths, dataStream, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; pkgVersion: string; paths: string[]; dataStream: RegistryDataStream; @@ -150,33 +149,35 @@ export async function installPipelinesForDataStream({ }); const installationPromises = pipelines.map(async (pipeline) => { - return installPipeline({ callCluster, pipeline }); + return installPipeline({ esClient, pipeline }); }); return Promise.all(installationPromises); } async function installPipeline({ - callCluster, + esClient, pipeline, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; pipeline: any; }): Promise { - const callClusterParams: { + const esClientParams: { method: string; path: string; - ignore: number[]; body: any; - headers?: any; } = { method: 'PUT', path: `/_ingest/pipeline/${pipeline.nameForInstallation}`, - ignore: [404], body: pipeline.contentForInstallation, }; + + const esClientRequestParams: { ignore: number[]; headers?: any } = { + ignore: [404], + }; + if (pipeline.extension === 'yml') { - callClusterParams.headers = { + esClientRequestParams.headers = { // pipeline is YAML 'Content-Type': 'application/yaml', // but we want JSON responses (to extract error messages, status code, or other metadata) @@ -189,7 +190,8 @@ async function installPipeline({ // exposed in the convenience endpoint 'ingest.putPipeline' of elasticsearch-js-legacy // which we could otherwise use. // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await callCluster('transport.request', callClusterParams); + await esClient.transport.request(esClientParams, esClientRequestParams); + return { id: pipeline.nameForInstallation, type: ElasticsearchAssetType.ingestPipeline }; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index c94d9be79c61f..fd5c0bbb8ca09 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -150,7 +150,7 @@ export async function _installPackage({ const installedPipelines = await installPipelines( packageInfo, paths, - callCluster, + esClient, savedObjectsClient ); // install or update the templates referencing the newly installed pipelines From dbf08af20cb5e072af2eb7cc7309a51e1e3de490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 16:58:09 +0100 Subject: [PATCH 13/36] Migrate `installIlmForDataStream` to new client --- .../epm/elasticsearch/datastream_ilm/install.ts | 10 ++++------ .../server/services/epm/packages/_install_package.ts | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts index 1dd8338f94d6e..6b61f02689a87 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts @@ -12,7 +12,6 @@ import { InstallablePackage, RegistryDataStream, } from '../../../../../common/types/models'; -import { CallESAsCurrentUser } from '../../../../types'; import { getInstallation } from '../../packages'; import { deleteIlmRefs, deleteIlms } from './remove'; import { saveInstalledEsRefs } from '../../packages/install'; @@ -31,7 +30,6 @@ interface IlmPathDataset { export const installIlmForDataStream = async ( registryPackage: InstallablePackage, paths: string[], - callCluster: CallESAsCurrentUser, esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract ) => { @@ -85,7 +83,7 @@ export const installIlmForDataStream = async ( ); const installationPromises = ilmInstallations.map(async (ilmInstallation) => { - return handleIlmInstall({ callCluster, ilmInstallation }); + return handleIlmInstall({ esClient, ilmInstallation }); }); installedIlms = await Promise.all(installationPromises).then((results) => results.flat()); @@ -110,13 +108,13 @@ export const installIlmForDataStream = async ( }; async function handleIlmInstall({ - callCluster, + esClient, ilmInstallation, }: { - callCluster: CallESAsCurrentUser; + esClient: ElasticsearchClient; ilmInstallation: IlmInstallation; }): Promise { - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: `/_ilm/policy/${ilmInstallation.installationName}`, body: ilmInstallation.content, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index fd5c0bbb8ca09..0d23910baeca4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -141,7 +141,6 @@ export async function _installPackage({ const installedDataStreamIlm = await installIlmForDataStream( packageInfo, paths, - callCluster, esClient, savedObjectsClient ); From 82b595596535feb13ed9a3887e7d44e2f3584406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 17:07:59 +0100 Subject: [PATCH 14/36] Remove old client from `installPackage` --- .../fleet/server/routes/epm/handlers.ts | 3 --- .../epm/packages/_install_package.test.ts | 9 +------- .../services/epm/packages/_install_package.ts | 10 +------- .../server/services/epm/packages/install.ts | 23 ++++--------------- .../fleet/server/services/package_policy.ts | 1 - x-pack/plugins/fleet/server/services/setup.ts | 1 - 6 files changed, 6 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 68ed803f0e18d..dfb55ed8b2408 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -234,7 +234,6 @@ export const installPackageFromRegistryHandler: RequestHandler< installSource: 'registry', savedObjectsClient, pkgkey, - callCluster, esClient, force: request.body?.force, }); @@ -306,7 +305,6 @@ export const installPackageByUploadHandler: RequestHandler< }); } const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const esClient = context.core.elasticsearch.client.asCurrentUser; const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later const archiveBuffer = Buffer.from(request.body); @@ -314,7 +312,6 @@ export const installPackageByUploadHandler: RequestHandler< const res = await installPackage({ installSource: 'upload', savedObjectsClient, - callCluster, esClient, archiveBuffer, contentType, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 6125f3c908fc4..1413bfc7224f7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,11 +5,7 @@ * 2.0. */ -import { - SavedObjectsClientContract, - LegacyScopedClusterClient, - ElasticsearchClient, -} from 'src/core/server'; +import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from 'src/core/server/mocks'; import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; @@ -41,12 +37,10 @@ function sleep(millis: number) { describe('_installPackage', () => { let soClient: jest.Mocked; - let callCluster: jest.Mocked; let esClient: jest.Mocked; beforeEach(async () => { soClient = savedObjectsClientMock.create(); - callCluster = elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser; esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; appContextService.start(createAppContextStartContractMock()); }); @@ -69,7 +63,6 @@ describe('_installPackage', () => { const installationPromise = _installPackage({ savedObjectsClient: soClient, - callCluster, esClient, paths: [], packageInfo: { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 0d23910baeca4..b6a3a6cde03a1 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -14,13 +14,7 @@ import { ASSETS_SAVED_OBJECT_TYPE, } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; -import { - AssetReference, - Installation, - CallESAsCurrentUser, - ElasticsearchAssetType, - InstallType, -} from '../../../types'; +import { AssetReference, Installation, ElasticsearchAssetType, InstallType } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import { installTemplates } from '../elasticsearch/template/install'; import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/'; @@ -40,7 +34,6 @@ import { ConcurrentInstallOperationError } from '../../../errors'; export async function _installPackage({ savedObjectsClient, - callCluster, esClient, installedPkg, paths, @@ -49,7 +42,6 @@ export async function _installPackage({ installSource, }: { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; installedPkg?: SavedObject; paths: string[]; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index e512b019a8c0f..ec87b148f87e9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -50,10 +50,9 @@ import { _installPackage } from './_install_package'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster, esClient } = options; + const { savedObjectsClient, pkgName, esClient } = options; try { const latestPackage = await Registry.fetchFindLatestPackage(pkgName); const pkgkey = Registry.pkgToPkgKey({ @@ -64,7 +63,6 @@ export async function installLatestPackage(options: { installSource: 'registry', savedObjectsClient, pkgkey, - callCluster, esClient, }); } catch (err) { @@ -105,10 +103,9 @@ export async function ensureInstalledDefaultPackages( export async function ensureInstalledPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; }): Promise { - const { savedObjectsClient, pkgName, callCluster, esClient } = options; + const { savedObjectsClient, pkgName, esClient } = options; const installedPackage = await getInstallation({ savedObjectsClient, pkgName }); if (installedPackage) { return installedPackage; @@ -117,7 +114,6 @@ export async function ensureInstalledPackage(options: { await installLatestPackage({ savedObjectsClient, pkgName, - callCluster, esClient, }); const installation = await getInstallation({ savedObjectsClient, pkgName }); @@ -172,7 +168,6 @@ export async function handleInstallPackageFailure({ installSource: 'registry', savedObjectsClient, pkgkey: prevVersion, - callCluster, esClient, }); } @@ -214,7 +209,6 @@ export async function upgradePackage({ installSource: 'registry', savedObjectsClient, pkgkey, - callCluster, esClient, }); return { @@ -252,7 +246,6 @@ export async function upgradePackage({ interface InstallRegistryPackageParams { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; force?: boolean; } @@ -260,7 +253,6 @@ interface InstallRegistryPackageParams { async function installPackageFromRegistry({ savedObjectsClient, pkgkey, - callCluster, esClient, force = false, }: InstallRegistryPackageParams): Promise { @@ -282,7 +274,6 @@ async function installPackageFromRegistry({ return _installPackage({ savedObjectsClient, - callCluster, esClient, installedPkg, paths, @@ -294,7 +285,6 @@ async function installPackageFromRegistry({ interface InstallUploadedArchiveParams { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; archiveBuffer: Buffer; contentType: string; @@ -306,7 +296,6 @@ export type InstallPackageParams = async function installPackageByUpload({ savedObjectsClient, - callCluster, esClient, archiveBuffer, contentType, @@ -342,7 +331,6 @@ async function installPackageByUpload({ return _installPackage({ savedObjectsClient, - callCluster, esClient, installedPkg, paths, @@ -358,21 +346,19 @@ export async function installPackage(args: InstallPackageParams) { } if (args.installSource === 'registry') { - const { savedObjectsClient, pkgkey, esClient, callCluster, force } = args; + const { savedObjectsClient, pkgkey, esClient, force } = args; return installPackageFromRegistry({ savedObjectsClient, pkgkey, esClient, - callCluster, force, }); } else if (args.installSource === 'upload') { - const { savedObjectsClient, esClient, callCluster, archiveBuffer, contentType } = args; + const { savedObjectsClient, esClient, archiveBuffer, contentType } = args; return installPackageByUpload({ savedObjectsClient, - callCluster, esClient, archiveBuffer, contentType, @@ -489,7 +475,6 @@ export async function ensurePackagesCompletedInstall( savedObjectsClient, pkgkey, esClient, - callCluster, }) ); } diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 20e697ea8f918..8ae4497eb5b84 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -91,7 +91,6 @@ class PackagePolicyService { ensureInstalledPackage({ savedObjectsClient: soClient, pkgName: packagePolicy.package.name, - callCluster, esClient, }), getPackageInfo({ diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index c78521648c476..e56232023ef5d 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -94,7 +94,6 @@ async function createSetupSideEffects( const fleetServerPackage = await ensureInstalledPackage({ savedObjectsClient: soClient, pkgName: FLEET_SERVER_PACKAGE, - callCluster, esClient, }); From 56b4ad1e8c9d99d7ccdce1bbd2c80fb368250f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 17:12:45 +0100 Subject: [PATCH 15/36] Remove old client from `bulkInstallPackages` --- x-pack/plugins/fleet/server/routes/epm/handlers.ts | 4 ---- .../server/services/epm/packages/bulk_install_packages.ts | 4 ---- .../plugins/fleet/server/services/epm/packages/install.ts | 7 ------- x-pack/plugins/fleet/server/services/setup.ts | 2 +- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index dfb55ed8b2408..9f27789bd282a 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -224,7 +224,6 @@ export const installPackageFromRegistryHandler: RequestHandler< TypeOf > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const esClient = context.core.elasticsearch.client.asCurrentUser; const { pkgkey } = request.params; const { pkgName, pkgVersion } = splitPkgKey(pkgkey); @@ -249,7 +248,6 @@ export const installPackageFromRegistryHandler: RequestHandler< pkgName, pkgVersion, installedPkg, - callCluster, esClient, }); @@ -278,11 +276,9 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler< TypeOf > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const esClient = context.core.elasticsearch.client.asCurrentUser; const bulkInstalledResponses = await bulkInstallPackages({ savedObjectsClient, - callCluster, esClient, packagesToUpgrade: request.body.packages, }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 9d0941e1de1f2..2149903f48a33 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -6,7 +6,6 @@ */ import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import { CallESAsCurrentUser } from '../../../types'; import * as Registry from '../registry'; import { getInstallationObject } from './index'; import { BulkInstallResponse, IBulkInstallPackageError, upgradePackage } from './install'; @@ -14,14 +13,12 @@ import { BulkInstallResponse, IBulkInstallPackageError, upgradePackage } from '. interface BulkInstallPackagesParams { savedObjectsClient: SavedObjectsClientContract; packagesToUpgrade: string[]; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; } export async function bulkInstallPackages({ savedObjectsClient, packagesToUpgrade, - callCluster, esClient, }: BulkInstallPackagesParams): Promise { const installedAndLatestPromises = packagesToUpgrade.map((pkgToUpgrade) => @@ -37,7 +34,6 @@ export async function bulkInstallPackages({ const [installedPkg, latestPkg] = result.value; return upgradePackage({ savedObjectsClient, - callCluster, esClient, installedPkg, latestPkg, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index ec87b148f87e9..4190b86656b76 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -72,14 +72,12 @@ export async function installLatestPackage(options: { export async function ensureInstalledDefaultPackages( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser, esClient: ElasticsearchClient ): Promise { const installations = []; const bulkResponse = await bulkInstallPackages({ savedObjectsClient, packagesToUpgrade: Object.values(defaultPackages), - callCluster, esClient, }); @@ -127,7 +125,6 @@ export async function handleInstallPackageFailure({ pkgName, pkgVersion, installedPkg, - callCluster, esClient, }: { savedObjectsClient: SavedObjectsClientContract; @@ -135,7 +132,6 @@ export async function handleInstallPackageFailure({ pkgName: string; pkgVersion: string; installedPkg: SavedObject | undefined; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; }) { if (error instanceof IngestManagerError) { @@ -184,7 +180,6 @@ export type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageEr interface UpgradePackageParams { savedObjectsClient: SavedObjectsClientContract; - callCluster: CallESAsCurrentUser; esClient: ElasticsearchClient; installedPkg: UnwrapPromise>; latestPkg: UnwrapPromise>; @@ -192,7 +187,6 @@ interface UpgradePackageParams { } export async function upgradePackage({ savedObjectsClient, - callCluster, esClient, installedPkg, latestPkg, @@ -224,7 +218,6 @@ export async function upgradePackage({ pkgName: latestPkg.name, pkgVersion: latestPkg.version, installedPkg, - callCluster, esClient, }); return { name: pkgToUpgrade, error: installFailed }; diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index e56232023ef5d..21cc87d27b3ce 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -63,7 +63,7 @@ async function createSetupSideEffects( { created: defaultFleetServerPolicyCreated, policy: defaultFleetServerPolicy }, ] = await Promise.all([ // packages installed by default - ensureInstalledDefaultPackages(soClient, callCluster, esClient), + ensureInstalledDefaultPackages(soClient, esClient), outputService.ensureDefaultOutput(soClient), agentPolicyService.ensureDefaultAgentPolicy(soClient, esClient), isFleetServerEnabled From d8424938dd540a356827afdad5a55ad6c5748a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 17:14:19 +0100 Subject: [PATCH 16/36] Remove legacy client from `ensurePackagesCompletedInstall` --- x-pack/plugins/fleet/server/services/epm/packages/install.ts | 2 -- x-pack/plugins/fleet/server/services/setup.ts | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 4190b86656b76..76fc0306993f9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -22,7 +22,6 @@ import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../ import { AssetReference, Installation, - CallESAsCurrentUser, AssetType, EsAssetReference, InstallType, @@ -446,7 +445,6 @@ export const removeAssetsFromInstalledEsByType = async ( export async function ensurePackagesCompletedInstall( savedObjectsClient: SavedObjectsClientContract, - callCluster: CallESAsCurrentUser, esClient: ElasticsearchClient ) { const installingPackages = await getPackageSavedObjects(savedObjectsClient, { diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 21cc87d27b3ce..a228684ca14ce 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -86,7 +86,7 @@ async function createSetupSideEffects( // will occur between upgrading the package and reinstalling the previously failed package. // By moving this outside of the Promise.all, the upgrade will occur first, and then we'll attempt to reinstall any // packages that are stuck in the installing state. - await ensurePackagesCompletedInstall(soClient, callCluster, esClient); + await ensurePackagesCompletedInstall(soClient, esClient); if (isFleetServerEnabled) { await awaitIfFleetServerSetupPending(); From 95db78cdaf56564b3e4af191ccc831cc9dac4ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 17:15:25 +0100 Subject: [PATCH 17/36] Remove legacy client from `addPackageToAgentPolicy` --- .../fleet/server/routes/agent_policy/handlers.ts | 3 +-- .../server/routes/package_policy/handlers.test.ts | 2 +- .../fleet/server/routes/package_policy/handlers.ts | 13 +++---------- .../plugins/fleet/server/services/package_policy.ts | 2 -- x-pack/plugins/fleet/server/services/setup.ts | 5 +---- 5 files changed, 6 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts index 411f7a74a6ab2..a9e0700e359b1 100644 --- a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts @@ -104,7 +104,6 @@ export const createAgentPolicyHandler: RequestHandler< > = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; const withSysMonitoring = request.query.sys_monitoring ?? false; try { @@ -130,7 +129,7 @@ export const createAgentPolicyHandler: RequestHandler< if (withSysMonitoring && newSysPackagePolicy !== undefined && agentPolicy !== undefined) { newSysPackagePolicy.policy_id = agentPolicy.id; newSysPackagePolicy.namespace = agentPolicy.namespace; - await packagePolicyService.create(soClient, esClient, callCluster, newSysPackagePolicy, { + await packagePolicyService.create(soClient, esClient, newSysPackagePolicy, { user, bumpRevision: false, }); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index 813279f2a800f..7583e2bb38108 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -25,7 +25,7 @@ jest.mock('../../services/package_policy', (): { compilePackagePolicyInputs: jest.fn((packageInfo, dataInputs) => Promise.resolve(dataInputs)), buildPackagePolicyFromPackage: jest.fn(), bulkCreate: jest.fn(), - create: jest.fn((soClient, esClient, callCluster, newData) => + create: jest.fn((soClient, esClient, newData) => Promise.resolve({ ...newData, inputs: newData.inputs.map((input) => ({ diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 6b35f74b3febc..7b0e46ae8428d 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -77,7 +77,6 @@ export const createPackagePolicyHandler: RequestHandler< > = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; try { const newData = await packagePolicyService.runExternalCallbacks( @@ -88,15 +87,9 @@ export const createPackagePolicyHandler: RequestHandler< ); // Create package policy - const packagePolicy = await packagePolicyService.create( - soClient, - esClient, - callCluster, - newData, - { - user, - } - ); + const packagePolicy = await packagePolicyService.create(soClient, esClient, newData, { + user, + }); const body: CreatePackagePolicyResponse = { item: packagePolicy }; return response.ok({ body, diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 8ae4497eb5b84..da38e845ccc47 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -33,7 +33,6 @@ import { PackagePolicy, PackagePolicySOAttributes, RegistryPackage, - CallESAsCurrentUser, NewPackagePolicySchema, UpdatePackagePolicySchema, } from '../types'; @@ -57,7 +56,6 @@ class PackagePolicyService { public async create( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, packagePolicy: NewPackagePolicy, options?: { id?: string; user?: AuthenticatedUser; bumpRevision?: boolean } ): Promise { diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index a228684ca14ce..51335156d65f7 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -101,7 +101,6 @@ async function createSetupSideEffects( await addPackageToAgentPolicy( soClient, esClient, - callCluster, fleetServerPackage, defaultFleetServerPolicy, defaultOutput @@ -146,7 +145,6 @@ async function createSetupSideEffects( await addPackageToAgentPolicy( soClient, esClient, - callCluster, installedPackage, agentPolicyWithPackagePolicies, defaultOutput @@ -261,7 +259,6 @@ function generateRandomPassword() { async function addPackageToAgentPolicy( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, packageToInstall: Installation, agentPolicy: AgentPolicy, defaultOutput: Output @@ -279,7 +276,7 @@ async function addPackageToAgentPolicy( agentPolicy.namespace ); - await packagePolicyService.create(soClient, esClient, callCluster, newPackagePolicy, { + await packagePolicyService.create(soClient, esClient, newPackagePolicy, { bumpRevision: false, }); } From a31a6d740d12131e5805809032fd4d7d01315ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 18:11:08 +0100 Subject: [PATCH 18/36] Migrate `setupIngestManager` to new client --- .../fleet/server/routes/setup/handlers.ts | 8 ++---- .../fleet/server/services/setup.test.ts | 4 +-- x-pack/plugins/fleet/server/services/setup.ts | 28 ++++++++----------- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.ts index 0c6ba6d14b1be..0f83104620bbe 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.ts @@ -62,9 +62,8 @@ export const createFleetSetupHandler: RequestHandler< try { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - await setupIngestManager(soClient, esClient, callCluster); - await setupFleet(soClient, esClient, callCluster, { + await setupIngestManager(soClient, esClient); + await setupFleet(soClient, esClient, { forceRecreate: request.body?.forceRecreate ?? false, }); @@ -79,11 +78,10 @@ export const createFleetSetupHandler: RequestHandler< export const FleetSetupHandler: RequestHandler = async (context, request, response) => { const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; try { const body: PostIngestSetupResponse = { isInitialized: true }; - await setupIngestManager(soClient, esClient, callCluster); + await setupIngestManager(soClient, esClient); return response.ok({ body, }); diff --git a/x-pack/plugins/fleet/server/services/setup.test.ts b/x-pack/plugins/fleet/server/services/setup.test.ts index 479f28fa0a1ed..497d0371871f7 100644 --- a/x-pack/plugins/fleet/server/services/setup.test.ts +++ b/x-pack/plugins/fleet/server/services/setup.test.ts @@ -43,7 +43,7 @@ describe('setupIngestManager', () => { soClient.update = mockedMethodThrowsError(); const esClient = context.core.elasticsearch.client.asCurrentUser; - const setupPromise = setupIngestManager(soClient, esClient, jest.fn()); + const setupPromise = setupIngestManager(soClient, esClient); await expect(setupPromise).rejects.toThrow('SO method mocked to throw'); await expect(setupPromise).rejects.toThrow(Error); }); @@ -56,7 +56,7 @@ describe('setupIngestManager', () => { soClient.update = mockedMethodThrowsCustom(); const esClient = context.core.elasticsearch.client.asCurrentUser; - const setupPromise = setupIngestManager(soClient, esClient, jest.fn()); + const setupPromise = setupIngestManager(soClient, esClient); await expect(setupPromise).rejects.toThrow('method mocked to throw'); await expect(setupPromise).rejects.toThrow(CustomTestError); }); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 51335156d65f7..f0edc58756aac 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -7,7 +7,6 @@ import uuid from 'uuid'; import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import { CallESAsCurrentUser } from '../types'; import { agentPolicyService } from './agent_policy'; import { outputService } from './output'; import { @@ -44,16 +43,14 @@ export interface SetupStatus { export async function setupIngestManager( soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise { - return awaitIfPending(async () => createSetupSideEffects(soClient, esClient, callCluster)); + return awaitIfPending(async () => createSetupSideEffects(soClient, esClient)); } async function createSetupSideEffects( soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser + esClient: ElasticsearchClient ): Promise { const isFleetServerEnabled = appContextService.getConfig()?.agents.fleetServerEnabled; const [ @@ -69,7 +66,7 @@ async function createSetupSideEffects( isFleetServerEnabled ? agentPolicyService.ensureDefaultFleetServerAgentPolicy(soClient, esClient) : {}, - updateFleetRoleIfExists(callCluster), + updateFleetRoleIfExists(esClient), settingsService.getSettings(soClient).catch((e: any) => { if (e.isBoom && e.output.statusCode === 404) { const defaultSettings = createDefaultSettings(); @@ -158,25 +155,25 @@ async function createSetupSideEffects( return { isIntialized: true }; } -async function updateFleetRoleIfExists(callCluster: CallESAsCurrentUser) { +async function updateFleetRoleIfExists(esClient: ElasticsearchClient) { try { - await callCluster('transport.request', { + await esClient.transport.request({ method: 'GET', path: `/_security/role/${FLEET_ENROLL_ROLE}`, }); } catch (e) { - if (e.status === 404) { + if (e.statusCode === 404) { return; } throw e; } - return putFleetRole(callCluster); + return putFleetRole(esClient); } -async function putFleetRole(callCluster: CallESAsCurrentUser) { - return callCluster('transport.request', { +async function putFleetRole(esClient: ElasticsearchClient) { + return await esClient.transport.request({ method: 'PUT', path: `/_security/role/${FLEET_ENROLL_ROLE}`, body: { @@ -194,12 +191,11 @@ async function putFleetRole(callCluster: CallESAsCurrentUser) { export async function setupFleet( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, - callCluster: CallESAsCurrentUser, options?: { forceRecreate?: boolean } ) { // Create fleet_enroll role // This should be done directly in ES at some point - const res = await putFleetRole(callCluster); + const { body: res } = await putFleetRole(esClient); // If the role is already created skip the rest unless you have forceRecreate set to true if (options?.forceRecreate !== true && res.role.created === false) { @@ -207,7 +203,7 @@ export async function setupFleet( } const password = generateRandomPassword(); // Create fleet enroll user - await callCluster('transport.request', { + await esClient.transport.request({ method: 'PUT', path: `/_security/user/${FLEET_ENROLL_USERNAME}`, body: { From 59536b5146371f3ff043e3c36c58ba9dd4000ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 25 Feb 2021 18:14:46 +0100 Subject: [PATCH 19/36] Fix types inside `createSetupSideEffects` --- x-pack/plugins/fleet/server/services/setup.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index f0edc58756aac..8c05e537cdfc2 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -57,7 +57,7 @@ async function createSetupSideEffects( installedPackages, defaultOutput, { created: defaultAgentPolicyCreated, defaultAgentPolicy }, - { created: defaultFleetServerPolicyCreated, policy: defaultFleetServerPolicy }, + defaultFleetServerPolicy, ] = await Promise.all([ // packages installed by default ensureInstalledDefaultPackages(soClient, esClient), @@ -65,7 +65,7 @@ async function createSetupSideEffects( agentPolicyService.ensureDefaultAgentPolicy(soClient, esClient), isFleetServerEnabled ? agentPolicyService.ensureDefaultFleetServerAgentPolicy(soClient, esClient) - : {}, + : undefined, updateFleetRoleIfExists(esClient), settingsService.getSettings(soClient).catch((e: any) => { if (e.isBoom && e.output.statusCode === 404) { @@ -94,12 +94,12 @@ async function createSetupSideEffects( esClient, }); - if (defaultFleetServerPolicyCreated) { + if (defaultFleetServerPolicy && defaultFleetServerPolicy.created) { await addPackageToAgentPolicy( soClient, esClient, fleetServerPackage, - defaultFleetServerPolicy, + defaultFleetServerPolicy.policy, defaultOutput ); } From 82270f7ac2551e00d22fd782118c84a1e770d8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 1 Mar 2021 13:48:28 +0100 Subject: [PATCH 20/36] Fix failing jest tests --- .../routes/package_policy/handlers.test.ts | 3 +- .../elasticsearch/template/install.test.ts | 20 ++-- .../elasticsearch/transform/transform.test.ts | 92 +++++++------------ 3 files changed, 45 insertions(+), 70 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index 7583e2bb38108..1395705083282 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -202,7 +202,8 @@ describe('When calling package policy', () => { ); await routeHandler(context, request, response); expect(response.ok).toHaveBeenCalled(); - expect(packagePolicyServiceMock.create.mock.calls[0][3]).toEqual({ + + expect(packagePolicyServiceMock.create.mock.calls[0][2]).toEqual({ policy_id: 'a5ca00c0-b30c-11ea-9732-1bb05811278c', description: '', enabled: true, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 8a44779b68427..fc8a246cff8ff 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -47,8 +47,8 @@ test('tests installPackage to use correct priority and index_patterns for data s packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + + const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); @@ -91,8 +91,9 @@ test('tests installPackage to use correct priority and index_patterns for data s packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + + const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]); @@ -135,8 +136,8 @@ test('tests installPackage to use correct priority and index_patterns for data s packageVersion: pkg.version, packageName: pkg.name, }); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[1][1].body; + const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]); @@ -189,11 +190,10 @@ test('tests installPackage remove the aliases property if the property existed', packageName: pkg.name, }); - // @ts-ignore - const removeAliases = callCluster.mock.calls[1][1].body; + const removeAliases = esClient.transport.request.mock.calls[1][0].body as Record; expect(removeAliases.template.aliases).not.toBeDefined(); - // @ts-ignore - const sentTemplate = callCluster.mock.calls[2][1].body; + + const sentTemplate = esClient.transport.request.mock.calls[2][0].body as Record; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 416e154e5358c..7367cf62f31bf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -18,6 +18,7 @@ jest.mock('./common', () => { }); import { errors as LegacyESErrors } from 'elasticsearch'; +import { DeeplyMockedKeys } from 'packages/kbn-utility-types/target/jest'; import { installTransform } from './install'; import { ElasticsearchClient, @@ -36,7 +37,7 @@ import { appContextService } from '../../../app_context'; describe('test transform install', () => { let legacyScopedClusterClient: jest.Mocked; - let esClient: jest.Mocked; + let esClient: DeeplyMockedKeys; let savedObjectsClient: jest.Mocked; beforeEach(() => { appContextService.start(createAppContextStartContractMock()); @@ -106,8 +107,8 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce( - Promise.resolve({ + esClient.transport.request.mockReturnValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ { @@ -116,15 +117,9 @@ describe('test transform install', () => { }, }, ], - } as { - count: number; - transforms: Array<{ - dest: { - index: string; - }; - }>; }) ); + await installTransform( ({ name: 'endpoint', @@ -169,74 +164,66 @@ describe('test transform install', () => { savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'GET', path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', - ignore: [404], }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0/_stop', - query: 'force=true', - ignore: [404], + querystring: 'force=true', }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'DELETE', - query: 'force=true', + querystring: 'force=true', path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', - ignore: [404], }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'DELETE', path: '/index', - ignore: [404], }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'PUT', path: '/_transform/endpoint.metadata-default-0.16.0-dev.0', - query: 'defer_validation=true', + querystring: 'defer_validation=true', body: '{"content": "data"}', }, ], [ - 'transport.request', { method: 'PUT', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + querystring: 'defer_validation=true', body: '{"content": "data"}', }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start', - ignore: [409], }, + { ignore: [409] }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], }, + { ignore: [409] }, ], ]); @@ -342,23 +329,21 @@ describe('test transform install', () => { savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'PUT', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + querystring: 'defer_validation=true', body: '{"content": "data"}', }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], }, + { ignore: [409] }, ], ]); expect(savedObjectsClient.update.mock.calls).toEqual([ @@ -400,8 +385,8 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce( - Promise.resolve({ + esClient.transport.request.mockReturnValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ { @@ -410,15 +395,9 @@ describe('test transform install', () => { }, }, ], - } as { - count: number; - transforms: Array<{ - dest: { - index: string; - }; - }>; }) ); + await installTransform( ({ name: 'endpoint', @@ -459,40 +438,36 @@ describe('test transform install', () => { savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'GET', path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', - ignore: [404], }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0/_stop', - query: 'force=true', - ignore: [404], + querystring: 'force=true', }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'DELETE', - query: 'force=true', + querystring: 'force=true', path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', - ignore: [404], }, + { ignore: [404] }, ], [ - 'transport.request', { method: 'DELETE', path: '/index', - ignore: [404], }, + { ignore: [404] }, ], ]); expect(savedObjectsClient.update.mock.calls).toEqual([ @@ -576,25 +551,24 @@ describe('test transform install', () => { savedObjectsClient ); - expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([ + expect(esClient.transport.request.mock.calls).toEqual([ [ - 'transport.request', { method: 'PUT', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - query: 'defer_validation=true', + querystring: 'defer_validation=true', body: '{"content": "data"}', }, ], [ - 'transport.request', { method: 'POST', path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - ignore: [409], }, + { ignore: [409] }, ], ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', From a7972377f30315a25a4a37b62a130835a543f0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 1 Mar 2021 15:26:05 +0100 Subject: [PATCH 21/36] Properly handle `resource_already_exists_exception` errors --- .../epm/elasticsearch/transform/install.ts | 5 ++-- .../elasticsearch/transform/transform.test.ts | 27 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index c0236f6c4cf05..3dca064dc964e 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -6,6 +6,7 @@ */ import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { saveInstalledEsRefs } from '../../packages/install'; import { getPathParts } from '../../archive'; @@ -18,7 +19,6 @@ import { getInstallation } from '../../packages'; import { deleteTransforms, deleteTransformRefs } from './remove'; import { getAsset } from './common'; import { appContextService } from '../../../app_context'; -import { isLegacyESClientError } from '../../../../errors'; interface TransformInstallation { installationName: string; @@ -131,7 +131,8 @@ async function handleTransformInstall({ } catch (err) { // swallow the error if the transform already exists. const isAlreadyExistError = - isLegacyESClientError(err) && err?.body?.error?.type === 'resource_already_exists_exception'; + err instanceof ResponseError && + err?.body?.error?.type === 'resource_already_exists_exception'; if (!isAlreadyExistError) { throw err; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 7367cf62f31bf..41da851f19608 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -17,7 +17,7 @@ jest.mock('./common', () => { }; }); -import { errors as LegacyESErrors } from 'elasticsearch'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { DeeplyMockedKeys } from 'packages/kbn-utility-types/target/jest'; import { installTransform } from './install'; import { @@ -508,23 +508,18 @@ describe('test transform install', () => { attributes: { installed_es: [] }, } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser = jest.fn(); - legacyScopedClusterClient.callAsCurrentUser.mockImplementation( - async (endpoint, clientParams, options) => { - if ( - endpoint === 'transport.request' && - clientParams?.method === 'PUT' && - clientParams?.path === '/_transform/endpoint.metadata_current-default-0.16.0-dev.0' - ) { - const err: LegacyESErrors._Abstract & { body?: any } = new LegacyESErrors.BadRequest(); - err.body = { - error: { type: 'resource_already_exists_exception' }, - }; - throw err; - } - } + esClient.transport.request.mockImplementationOnce(() => + elasticsearchClientMock.createErrorTransportRequestPromise( + new ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { error: { type: 'resource_already_exists_exception' } }, + }) + ) + ) ); + await installTransform( ({ name: 'endpoint', From 8401da64ac14488688155b8fc02c6bee0abcd862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 1 Mar 2021 15:26:39 +0100 Subject: [PATCH 22/36] Cleanup test file --- .../epm/elasticsearch/transform/transform.test.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 41da851f19608..63ec268ad4362 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -20,12 +20,7 @@ jest.mock('./common', () => { import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { DeeplyMockedKeys } from 'packages/kbn-utility-types/target/jest'; import { installTransform } from './install'; -import { - ElasticsearchClient, - ILegacyScopedClusterClient, - SavedObject, - SavedObjectsClientContract, -} from 'kibana/server'; +import { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'kibana/server'; import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types'; import { getInstallation, getInstallationObject } from '../../packages'; import { getAsset } from './common'; @@ -36,15 +31,10 @@ import { elasticsearchClientMock } from '../../../../../../../../src/core/server import { appContextService } from '../../../app_context'; describe('test transform install', () => { - let legacyScopedClusterClient: jest.Mocked; let esClient: DeeplyMockedKeys; let savedObjectsClient: jest.Mocked; beforeEach(() => { appContextService.start(createAppContextStartContractMock()); - legacyScopedClusterClient = { - callAsInternalUser: jest.fn(), - callAsCurrentUser: jest.fn(), - }; esClient = elasticsearchClientMock.createClusterClient().asInternalUser; (getInstallation as jest.MockedFunction).mockReset(); (getInstallationObject as jest.MockedFunction).mockReset(); @@ -302,7 +292,7 @@ describe('test transform install', () => { attributes: { installed_es: [] }, } as unknown) as SavedObject) ); - legacyScopedClusterClient.callAsCurrentUser = jest.fn(); + await installTransform( ({ name: 'endpoint', From 3ecbde6fe055680446e13e159eabbc991bacb88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 10:54:45 +0100 Subject: [PATCH 23/36] Remove dead function --- .../fleet/server/services/api_keys/security.ts | 11 ----------- x-pack/plugins/fleet/server/types/index.tsx | 2 -- 2 files changed, 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/api_keys/security.ts b/x-pack/plugins/fleet/server/services/api_keys/security.ts index efd46c1a547dd..81ab47fa7a023 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/security.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/security.ts @@ -8,7 +8,6 @@ import type { Request } from '@hapi/hapi'; import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../src/core/server'; import { FleetAdminUserInvalidError, isESClientError } from '../../errors'; -import { CallESAsCurrentUser } from '../../types'; import { appContextService } from '../app_context'; import { outputService } from '../output'; @@ -54,16 +53,6 @@ export async function createAPIKey( throw err; } } -export async function authenticate(callCluster: CallESAsCurrentUser) { - try { - await callCluster('transport.request', { - path: '/_security/_authenticate', - method: 'GET', - }); - } catch (e) { - throw new Error('ApiKey is not valid: impossible to authenticate user'); - } -} export async function invalidateAPIKeys(soClient: SavedObjectsClientContract, ids: string[]) { const adminUser = await outputService.getAdminUser(soClient); diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index fda1568c56e0e..7081ab2447e76 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -86,8 +86,6 @@ export { FleetServerPolicy, } from '../../common'; -export type CallESAsCurrentUser = LegacyScopedClusterClient['callAsCurrentUser']; - export type AgentPolicyUpdateHandler = ( action: 'created' | 'updated' | 'deleted', agentPolicyId: string From 14f1965b71e91a247a46a87feb4f006626c1feb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 11:12:00 +0100 Subject: [PATCH 24/36] Migrate `getListHandler` to new client --- .../server/routes/data_streams/handlers.ts | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts index 4424fc478c682..61d496c506492 100644 --- a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts @@ -14,40 +14,37 @@ import { defaultIngestErrorHandler } from '../../errors'; const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*'; -interface ESDataStreamInfoResponse { - data_streams: Array<{ +interface ESDataStreamInfo { + name: string; + timestamp_field: { name: string; - timestamp_field: { + }; + indices: Array<{ index_name: string; index_uuid: string }>; + generation: number; + _meta?: { + package?: { name: string; }; - indices: Array<{ index_name: string; index_uuid: string }>; - generation: number; - _meta?: { - package?: { - name: string; - }; - managed_by?: string; - managed?: boolean; - [key: string]: any; - }; - status: string; - template: string; - ilm_policy: string; - hidden: boolean; - }>; + managed_by?: string; + managed?: boolean; + [key: string]: any; + }; + status: string; + template: string; + ilm_policy: string; + hidden: boolean; } -interface ESDataStreamStatsResponse { - data_streams: Array<{ - data_stream: string; - backing_indices: number; - store_size_bytes: number; - maximum_timestamp: number; - }>; +interface ESDataStreamStats { + data_stream: string; + backing_indices: number; + store_size_bytes: number; + maximum_timestamp: number; } export const getListHandler: RequestHandler = async (context, request, response) => { - const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const esClient = context.core.elasticsearch.client.asCurrentUser; + const body: GetDataStreamsResponse = { data_streams: [], }; @@ -55,22 +52,27 @@ export const getListHandler: RequestHandler = async (context, request, response) try { // Get matching data streams, their stats, and package SOs const [ - { data_streams: dataStreamsInfo }, - { data_streams: dataStreamStats }, + { + body: { data_streams: dataStreamsInfo }, + }, + { + body: { data_streams: dataStreamStats }, + }, packageSavedObjects, ] = await Promise.all([ - callCluster('transport.request', { + esClient.transport.request({ method: 'GET', path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}`, - }) as Promise, - callCluster('transport.request', { + }), + esClient.transport.request({ method: 'GET', path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}/_stats`, - }) as Promise, + }), getPackageSavedObjects(context.core.savedObjects.client), ]); - const dataStreamsInfoByName = keyBy(dataStreamsInfo, 'name'); - const dataStreamsStatsByName = keyBy(dataStreamStats, 'data_stream'); + + const dataStreamsInfoByName = keyBy(dataStreamsInfo, 'name'); + const dataStreamsStatsByName = keyBy(dataStreamStats, 'data_stream'); // Combine data stream info const dataStreams = merge(dataStreamsInfoByName, dataStreamsStatsByName); @@ -97,8 +99,10 @@ export const getListHandler: RequestHandler = async (context, request, response) // Query backing indices to extract data stream dataset, namespace, and type values const { - aggregations: { dataset, namespace, type }, - } = await callCluster('search', { + body: { + aggregations: { dataset, namespace, type }, + }, + } = await esClient.search({ index: dataStream.indices.map((index) => index.index_name), body: { size: 0, From 8ddaf7508215a19fa1a7a6008f72eabce0c19279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 11:46:13 +0100 Subject: [PATCH 25/36] Use `putIndexTemplate` method --- .../epm/elasticsearch/template/install.ts | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 6c6d4cacd9195..fa41ab36692dd 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -85,30 +85,11 @@ const installPreBuiltTemplates = async (paths: string[], esClient: Elasticsearch const { file } = getPathParts(path); const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - let templateAPIPath = '_template'; - // v2 index templates need to be installed through the new API endpoint. - // Checking for 'template' and 'composed_of' should catch them all. - // For the new v2 format, see https://github.com/elastic/elasticsearch/issues/53101 - if (content.hasOwnProperty('template') || content.hasOwnProperty('composed_of')) { - templateAPIPath = '_index_template'; - } - - const esClientParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/${templateAPIPath}/${templateName}`, - body: content, - }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return esClient.transport.request(esClientParams, { ignore: [404] }); + return esClient.indices.putIndexTemplate( + { name: templateName, body: content }, + { ignore: [404] } + ); }); try { return await Promise.all(templateInstallPromises); From b30449374141179d3b8e77bd1356f26e239c0ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 11:54:01 +0100 Subject: [PATCH 26/36] Use `ingest.putPipeline` method --- .../elasticsearch/ingest_pipeline/install.ts | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 34797364a9324..89009c9462c7c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { TransportRequestOptions } from '@elastic/elasticsearch/lib/Transport'; import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import { EsAssetReference, @@ -162,22 +163,17 @@ async function installPipeline({ esClient: ElasticsearchClient; pipeline: any; }): Promise { - const esClientParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/_ingest/pipeline/${pipeline.nameForInstallation}`, + const esClientParams = { + id: pipeline.nameForInstallation, body: pipeline.contentForInstallation, }; - const esClientRequestParams: { ignore: number[]; headers?: any } = { + const esClientRequestOptions: TransportRequestOptions = { ignore: [404], }; if (pipeline.extension === 'yml') { - esClientRequestParams.headers = { + esClientRequestOptions.headers = { // pipeline is YAML 'Content-Type': 'application/yaml', // but we want JSON responses (to extract error messages, status code, or other metadata) @@ -185,12 +181,7 @@ async function installPipeline({ }; } - // This uses the catch-all endpoint 'transport.request' because we have to explicitly - // set the Content-Type header above for sending yml data. Setting the headers is not - // exposed in the convenience endpoint 'ingest.putPipeline' of elasticsearch-js-legacy - // which we could otherwise use. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await esClient.transport.request(esClientParams, esClientRequestParams); + await esClient.ingest.putPipeline(esClientParams, esClientRequestOptions); return { id: pipeline.nameForInstallation, type: ElasticsearchAssetType.ingestPipeline }; } From c075e857156d4aafd08578e1c1f4cf062536a9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 12:02:51 +0100 Subject: [PATCH 27/36] Use `cluster.putComponentTemplate` --- .../epm/elasticsearch/template/install.ts | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index fa41ab36692dd..7e701baec17bb 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -110,19 +110,12 @@ const installPreBuiltComponentTemplates = async ( const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - const esClientParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/_component_template/${templateName}`, + const esClientParams = { + name: templateName, body: content, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint for component templates yet. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - return esClient.transport.request(esClientParams, { ignore: [404] }); + + return esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }); }); try { @@ -175,18 +168,13 @@ function putComponentTemplate( esClient: ElasticsearchClient ): { clusterPromise: Promise; name: string } | undefined { if (body) { - const esClientParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/_component_template/${name}`, + const esClientParams = { + name, body, }; return { - clusterPromise: esClient.transport.request(esClientParams, { ignore: [404] }), + clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }), name, }; } From 8cbb0524ce51f95e23054291270ff6d44a797a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 12:03:29 +0100 Subject: [PATCH 28/36] Cleanup references --- x-pack/plugins/fleet/server/types/index.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 7081ab2447e76..885809d767323 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -5,8 +5,6 @@ * 2.0. */ -import { LegacyScopedClusterClient } from 'src/core/server'; - export { // Object types Agent, From 09d5836a7b53bcb20e40064882147add51903a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 14:34:50 +0100 Subject: [PATCH 29/36] Handle v1 templates --- .../services/epm/elasticsearch/template/install.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 7e701baec17bb..f5f132228d5d8 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -86,10 +86,16 @@ const installPreBuiltTemplates = async (paths: string[], esClient: Elasticsearch const templateName = file.substr(0, file.lastIndexOf('.')); const content = JSON.parse(getAsset(path).toString('utf8')); - return esClient.indices.putIndexTemplate( - { name: templateName, body: content }, - { ignore: [404] } - ); + const esClientParams = { name: templateName, body: content }; + const esClientRequestOptions = { ignore: [404] }; + + if (content.hasOwnProperty('template') || content.hasOwnProperty('composed_of')) { + // Template is v2 + return esClient.indices.putIndexTemplate(esClientParams, esClientRequestOptions); + } else { + // template is V1 + return esClient.indices.putTemplate(esClientParams, esClientRequestOptions); + } }); try { return await Promise.all(templateInstallPromises); From 32f4ca8792314b347966ab542ac4a6eda9300d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 14:57:33 +0100 Subject: [PATCH 30/36] Use specific index template API methods --- .../epm/elasticsearch/template/install.ts | 39 +++++-------------- .../server/services/epm/packages/remove.ts | 14 +------ 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index f5f132228d5d8..901071f4fee3d 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -284,10 +284,9 @@ export async function installTemplate({ } // Datastream now throw an error if the aliases field is present so ensure that we remove that field. - const { body: getTemplateRes } = await esClient.transport.request( + const { body: getTemplateRes } = await esClient.indices.getIndexTemplate( { - method: 'GET', - path: `/_index_template/${templateName}`, + name: templateName, }, { ignore: [404], @@ -300,13 +299,8 @@ export async function installTemplate({ existingIndexTemplate.name === templateName && existingIndexTemplate?.index_template?.template?.aliases ) { - const updateIndexTemplateParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/_index_template/${templateName}`, + const updateIndexTemplateParams = { + name: templateName, body: { ...existingIndexTemplate.index_template, template: { @@ -316,12 +310,8 @@ export async function installTemplate({ }, }, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await esClient.transport.request(updateIndexTemplateParams, { ignore: [404] }); + + await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] }); } const composedOfTemplates = await installDataStreamComponentTemplates( @@ -343,21 +333,12 @@ export async function installTemplate({ }); // TODO: Check return values for errors - const esClientParams: { - method: string; - path: string; - body: any; - } = { - method: 'PUT', - path: `/_index_template/${templateName}`, + const esClientParams = { + name: templateName, body: template, }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await esClient.transport.request(esClientParams, { ignore: [404] }); + + await esClient.indices.putIndexTemplate(esClientParams, { ignore: [404] }); return { templateName, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index d0652ffcd837a..94599bdff85e8 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -126,19 +126,7 @@ async function deleteTemplate(esClient: ElasticsearchClient, name: string): Prom // '*' shouldn't ever appear here, but it still would delete all templates if (name && name !== '*') { try { - const esClientParams: { - method: string; - path: string; - } = { - method: 'DELETE', - path: `/_index_template/${name}`, - }; - // This uses the catch-all endpoint 'transport.request' because there is no - // convenience endpoint using the new _index_template API yet. - // The existing convenience endpoint `indices.putTemplate` only sends to _template, - // which does not support v2 templates. - // See src/core/server/elasticsearch/api_types.ts for available endpoints. - await esClient.transport.request(esClientParams, { ignore: [404] }); + await esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }); } catch { throw new Error(`error deleting template ${name}`); } From 93a7a28567408ad8b9cb5ea5ead66488e27630f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 15:04:22 +0100 Subject: [PATCH 31/36] Use specific data stream API methods --- .../fleet/server/routes/data_streams/handlers.ts | 10 ++-------- .../services/epm/elasticsearch/template/template.ts | 5 +---- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts index 61d496c506492..829feb9e4d4fa 100644 --- a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts @@ -60,14 +60,8 @@ export const getListHandler: RequestHandler = async (context, request, response) }, packageSavedObjects, ] = await Promise.all([ - esClient.transport.request({ - method: 'GET', - path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}`, - }), - esClient.transport.request({ - method: 'GET', - path: `/_data_stream/${DATA_STREAM_INDEX_PATTERN}/_stats`, - }), + esClient.indices.getDataStream({ name: DATA_STREAM_INDEX_PATTERN }), + esClient.indices.dataStreamsStats({ name: DATA_STREAM_INDEX_PATTERN }), getPackageSavedObjects(context.core.savedObjects.client), ]); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index c04f263660cb2..8517ecf184487 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -406,10 +406,7 @@ const getDataStreams = async ( template: TemplateRef ): Promise => { const { templateName, indexTemplate } = template; - const { body } = await esClient.transport.request({ - method: 'GET', - path: `/_data_stream/${templateName}-*`, - }); + const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` }); const dataStreams = body.data_streams; if (!dataStreams.length) return; return dataStreams.map((dataStream: any) => ({ From 8e850a3593ff4ada89906f776c88325340488499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 15:11:52 +0100 Subject: [PATCH 32/36] Use specific security API methods --- x-pack/plugins/fleet/server/services/setup.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 8c05e537cdfc2..530f39285375c 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -157,10 +157,7 @@ async function createSetupSideEffects( async function updateFleetRoleIfExists(esClient: ElasticsearchClient) { try { - await esClient.transport.request({ - method: 'GET', - path: `/_security/role/${FLEET_ENROLL_ROLE}`, - }); + await esClient.security.getRole({ name: FLEET_ENROLL_ROLE }); } catch (e) { if (e.statusCode === 404) { return; @@ -173,9 +170,8 @@ async function updateFleetRoleIfExists(esClient: ElasticsearchClient) { } async function putFleetRole(esClient: ElasticsearchClient) { - return await esClient.transport.request({ - method: 'PUT', - path: `/_security/role/${FLEET_ENROLL_ROLE}`, + return await esClient.security.putRole({ + name: FLEET_ENROLL_ROLE, body: { cluster: ['monitor', 'manage_api_key'], indices: [ @@ -203,9 +199,8 @@ export async function setupFleet( } const password = generateRandomPassword(); // Create fleet enroll user - await esClient.transport.request({ - method: 'PUT', - path: `/_security/user/${FLEET_ENROLL_USERNAME}`, + await esClient.security.putUser({ + username: FLEET_ENROLL_USERNAME, body: { password, roles: [FLEET_ENROLL_ROLE], From 9b34fc771c2b9306a7e057df1ef0b8db3113ca8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 15:26:03 +0100 Subject: [PATCH 33/36] Use specific transform API methods --- .../epm/elasticsearch/transform/install.ts | 19 +++----- .../epm/elasticsearch/transform/remove.ts | 46 ++++++------------- 2 files changed, 20 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 3dca064dc964e..20cd48e0c2206 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -122,10 +122,9 @@ async function handleTransformInstall({ }): Promise { try { // defer validation on put if the source index is not available - await esClient.transport.request({ - method: 'PUT', - path: `/_transform/${transform.installationName}`, - querystring: 'defer_validation=true', + await esClient.transform.putTransform({ + transform_id: transform.installationName, + defer_validation: true, body: transform.content, }); } catch (err) { @@ -137,15 +136,9 @@ async function handleTransformInstall({ throw err; } } - await esClient.transport.request( - { - method: 'POST', - path: `/_transform/${transform.installationName}/_start`, - // Ignore error if the transform is already started - }, - { - ignore: [409], - } + await esClient.transform.startTransform( + { transform_id: transform.installationName }, + { ignore: [409] } ); return { id: transform.installationName, type: ElasticsearchAssetType.transform }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts index 3d1b7c6ed59ed..a69416ec1e01a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts @@ -12,15 +12,9 @@ import { appContextService } from '../../../app_context'; export const stopTransforms = async (transformIds: string[], esClient: ElasticsearchClient) => { for (const transformId of transformIds) { - await esClient.transport.request( - { - method: 'POST', - path: `/_transform/${transformId}/_stop`, - querystring: 'force=true', - }, - { - ignore: [404], - } + await esClient.transform.stopTransform( + { transform_id: transformId, force: true }, + { ignore: [404] } ); } }; @@ -32,37 +26,25 @@ export const deleteTransforms = async (esClient: ElasticsearchClient, transformI } await Promise.all( transformIds.map(async (transformId) => { - // get the index the transform - const { body } = await esClient.transport.request( - { - method: 'GET', - path: `/_transform/${transformId}`, - }, - { - ignore: [404], - } - ); - - // `transport.request` doesn't accept generics, so we cast the type here. - const transformResponse = body as { + interface TransformResponse { count: number; transforms?: Array<{ dest: { index: string; }; }>; - }; + } + + // get the index the transform + const { body: transformResponse } = await esClient.transform.getTransform( + { transform_id: transformId }, + { ignore: [404] } + ); await stopTransforms([transformId], esClient); - await esClient.transport.request( - { - method: 'DELETE', - querystring: 'force=true', - path: `/_transform/${transformId}`, - }, - { - ignore: [404], - } + await esClient.transform.deleteTransform( + { force: true, transform_id: transformId }, + { ignore: [404] } ); logger.info(`Deleted: ${transformId}`); if (transformResponse?.transforms) { From fae1df17d39f0a66670040590304af294f6962d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 2 Mar 2021 17:14:05 +0100 Subject: [PATCH 34/36] Fix tests --- .../elasticsearch/transform/transform.test.ts | 102 ++++++++++-------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 63ec268ad4362..78fc5a226e402 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -97,7 +97,7 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - esClient.transport.request.mockReturnValueOnce( + esClient.transform.getTransform.mockReturnValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ @@ -154,30 +154,34 @@ describe('test transform install', () => { savedObjectsClient ); - expect(esClient.transport.request.mock.calls).toEqual([ + expect(esClient.transform.getTransform.mock.calls).toEqual([ [ { - method: 'GET', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', }, { ignore: [404] }, ], + ]); + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0/_stop', - querystring: 'force=true', + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + force: true, }, { ignore: [404] }, ], + ]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - method: 'DELETE', - querystring: 'force=true', - path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + force: true, }, { ignore: [404] }, ], + ]); + + expect(esClient.transport.request.mock.calls).toEqual([ [ { method: 'DELETE', @@ -185,33 +189,34 @@ describe('test transform install', () => { }, { ignore: [404] }, ], + ]); + + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ { - method: 'PUT', - path: '/_transform/endpoint.metadata-default-0.16.0-dev.0', - querystring: 'defer_validation=true', + transform_id: 'endpoint.metadata-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], [ { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - querystring: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], + ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ [ { - method: 'POST', - path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start', + transform_id: 'endpoint.metadata-default-0.16.0-dev.0', }, { ignore: [409] }, ], [ { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', }, { ignore: [409] }, ], @@ -319,23 +324,24 @@ describe('test transform install', () => { savedObjectsClient ); - expect(esClient.transport.request.mock.calls).toEqual([ + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - querystring: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], + ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ [ { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', }, { ignore: [409] }, ], ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', @@ -375,7 +381,7 @@ describe('test transform install', () => { } as unknown) as SavedObject) ); - esClient.transport.request.mockReturnValueOnce( + esClient.transform.getTransform.mockReturnValueOnce( elasticsearchClientMock.createSuccessTransportRequestPromise({ count: 1, transforms: [ @@ -428,30 +434,36 @@ describe('test transform install', () => { savedObjectsClient ); - expect(esClient.transport.request.mock.calls).toEqual([ + expect(esClient.transform.getTransform.mock.calls).toEqual([ [ { - method: 'GET', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', }, { ignore: [404] }, ], + ]); + + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - method: 'POST', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0/_stop', - querystring: 'force=true', + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + force: true, }, { ignore: [404] }, ], + ]); + + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - method: 'DELETE', - querystring: 'force=true', - path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + force: true, }, { ignore: [404] }, ], + ]); + + expect(esClient.transport.request.mock.calls).toEqual([ [ { method: 'DELETE', @@ -460,6 +472,7 @@ describe('test transform install', () => { { ignore: [404] }, ], ]); + expect(savedObjectsClient.update.mock.calls).toEqual([ [ 'epm-packages', @@ -536,22 +549,17 @@ describe('test transform install', () => { savedObjectsClient ); - expect(esClient.transport.request.mock.calls).toEqual([ + expect(esClient.transform.putTransform.mock.calls).toEqual([ [ { - method: 'PUT', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0', - querystring: 'defer_validation=true', + transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0', + defer_validation: true, body: '{"content": "data"}', }, ], - [ - { - method: 'POST', - path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start', - }, - { ignore: [409] }, - ], + ]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [{ transform_id: 'endpoint.metadata_current-default-0.16.0-dev.0' }, { ignore: [409] }], ]); expect(savedObjectsClient.update.mock.calls).toEqual([ From b5f73c3cfcce42e671341501382856de82e37cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 3 Mar 2021 11:57:53 +0100 Subject: [PATCH 35/36] Fix installTemplat tests --- .../elasticsearch/template/install.test.ts | 96 ++++++++----------- 1 file changed, 42 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index fc8a246cff8ff..20b9640e64eb0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -13,16 +13,9 @@ import { installTemplate } from './install'; test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.transport.request.mockImplementation((params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); - } - return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); - }); + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixUnset = { @@ -48,7 +41,11 @@ test('tests installPackage to use correct priority and index_patterns for data s packageName: pkg.name, }); - const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; + expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); @@ -56,16 +53,9 @@ test('tests installPackage to use correct priority and index_patterns for data s test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.transport.request.mockImplementation((params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); - } - return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); - }); + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixFalse = { @@ -92,7 +82,10 @@ test('tests installPackage to use correct priority and index_patterns for data s packageName: pkg.name, }); - const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse); @@ -101,16 +94,9 @@ test('tests installPackage to use correct priority and index_patterns for data s test('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.transport.request.mockImplementation((params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }); - } - return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); - }); + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixTrue = { @@ -136,7 +122,10 @@ test('tests installPackage to use correct priority and index_patterns for data s packageVersion: pkg.version, packageName: pkg.name, }); - const sentTemplate = esClient.transport.request.mock.calls[1][0].body as Record; + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue); @@ -145,26 +134,19 @@ test('tests installPackage to use correct priority and index_patterns for data s test('tests installPackage remove the aliases property if the property existed', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - esClient.transport.request.mockImplementation((params) => { - if ( - params && - params.method === 'GET' && - params.path === '/_index_template/metrics-package.dataset' - ) { - return elasticsearchServiceMock.createSuccessTransportRequestPromise({ - index_templates: [ - { - name: 'metrics-package.dataset', - index_template: { - index_patterns: ['metrics-package.dataset-*'], - template: { aliases: {} }, - }, + esClient.indices.getIndexTemplate.mockImplementation(() => + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + index_templates: [ + { + name: 'metrics-package.dataset', + index_template: { + index_patterns: ['metrics-package.dataset-*'], + template: { aliases: {} }, }, - ], - }); - } - return elasticsearchServiceMock.createSuccessTransportRequestPromise({}); - }); + }, + ], + }) + ); const fields: Field[] = []; const dataStreamDatasetIsPrefixUnset = { @@ -190,10 +172,16 @@ test('tests installPackage remove the aliases property if the property existed', packageName: pkg.name, }); - const removeAliases = esClient.transport.request.mock.calls[1][0].body as Record; + const removeAliases = esClient.indices.putIndexTemplate.mock.calls[0][0]!.body as Record< + string, + any + >; expect(removeAliases.template.aliases).not.toBeDefined(); - const sentTemplate = esClient.transport.request.mock.calls[2][0].body as Record; + const sentTemplate = esClient.indices.putIndexTemplate.mock.calls[1][0]!.body as Record< + string, + any + >; expect(sentTemplate).toBeDefined(); expect(sentTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); From fabd93e6b83db1acac272d8a3300869511ec43b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 3 Mar 2021 15:39:45 +0100 Subject: [PATCH 36/36] Run `node scripts/build_api_docs` locally to try to fix the build --- api_docs/fleet.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_docs/fleet.json b/api_docs/fleet.json index 381818efb9f33..6d0e6f7725ec3 100644 --- a/api_docs/fleet.json +++ b/api_docs/fleet.json @@ -2607,8 +2607,8 @@ "description": [], "source": { "path": "x-pack/plugins/fleet/server/services/package_policy.ts", - "lineNumber": 592, - "link": "https://github.com/elastic/kibana/tree/masterx-pack/plugins/fleet/server/services/package_policy.ts#L592" + "lineNumber": 590, + "link": "https://github.com/elastic/kibana/tree/masterx-pack/plugins/fleet/server/services/package_policy.ts#L590" }, "signature": [ "PackagePolicyService"