From 5c16e07b58416731f5f5307f3a9d3b69baab055b Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 9 Jul 2020 11:46:10 -0400 Subject: [PATCH] Revert "[Ingest Manager ] prepend kibana asset ids with package name (#70502)" This reverts commit 984ea0700ee8b84e69f626792e1dd913607307c9. --- .../services/epm/kibana/assets/install.ts | 119 ---------------- .../tests/__snapshots__/install.test.ts.snap | 133 ------------------ .../epm/kibana/assets/tests/dashboard.json | 129 ----------------- .../epm/kibana/assets/tests/install.test.ts | 35 ----- .../services/epm/packages/get_objects.ts | 32 +++++ .../server/services/epm/packages/index.ts | 2 +- .../server/services/epm/packages/install.ts | 58 +++++++- 7 files changed, 89 insertions(+), 419 deletions(-) delete mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts delete mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap delete mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json delete mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts create mode 100644 x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts deleted file mode 100644 index ae6493d4716e8..0000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - SavedObject, - SavedObjectsBulkCreateObject, - SavedObjectsClientContract, -} from 'src/core/server'; -import * as Registry from '../../registry'; -import { AssetType, KibanaAssetType, AssetReference } from '../../../../types'; - -type SavedObjectToBe = Required & { type: AssetType }; -export type ArchiveAsset = Pick< - SavedObject, - 'id' | 'attributes' | 'migrationVersion' | 'references' -> & { - type: AssetType; -}; - -export async function getKibanaAsset(key: string) { - const buffer = Registry.getAsset(key); - - // cache values are buffers. convert to string / JSON - return JSON.parse(buffer.toString('utf8')); -} - -export function createSavedObjectKibanaAsset( - jsonAsset: ArchiveAsset, - pkgName: string -): SavedObjectToBe { - // convert that to an object - const asset = changeAssetIds(jsonAsset, pkgName); - - return { - type: asset.type, - id: asset.id, - attributes: asset.attributes, - references: asset.references || [], - migrationVersion: asset.migrationVersion || {}, - }; -} - -// modifies id property and the id property of references objects (not index-pattern) -// to be prepended with the package name to distinguish assets from Beats modules' assets -export const changeAssetIds = (asset: ArchiveAsset, pkgName: string): ArchiveAsset => { - const references = asset.references.map((ref) => { - if (ref.type === KibanaAssetType.indexPattern) return ref; - const id = getAssetId(ref.id, pkgName); - return { ...ref, id }; - }); - return { - ...asset, - id: getAssetId(asset.id, pkgName), - references, - }; -}; - -export const getAssetId = (id: string, pkgName: string) => { - return `${pkgName}-${id}`; -}; - -// TODO: make it an exhaustive list -// e.g. switch statement with cases for each enum key returning `never` for default case -export async function installKibanaAssets(options: { - savedObjectsClient: SavedObjectsClientContract; - pkgName: string; - paths: string[]; -}) { - const { savedObjectsClient, paths, pkgName } = options; - - // Only install Kibana assets during package installation. - const kibanaAssetTypes = Object.values(KibanaAssetType); - const installationPromises = kibanaAssetTypes.map((assetType) => - installKibanaSavedObjects({ savedObjectsClient, assetType, paths, pkgName }) - ); - - // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] - // call .flat to flatten into one dimensional array - return Promise.all(installationPromises).then((results) => results.flat()); -} - -async function installKibanaSavedObjects({ - savedObjectsClient, - assetType, - paths, - pkgName, -}: { - savedObjectsClient: SavedObjectsClientContract; - assetType: KibanaAssetType; - paths: string[]; - pkgName: string; -}) { - const isSameType = (path: string) => assetType === Registry.pathParts(path).type; - const pathsOfType = paths.filter((path) => isSameType(path)); - const kibanaAssets = await Promise.all(pathsOfType.map((path) => getKibanaAsset(path))); - const toBeSavedObjects = await Promise.all( - kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset, pkgName)) - ); - - if (toBeSavedObjects.length === 0) { - return []; - } else { - const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { - overwrite: true, - }); - const createdObjects = createResults.saved_objects; - const installed = createdObjects.map(toAssetReference); - return installed; - } -} - -function toAssetReference({ id, type }: SavedObject) { - const reference: AssetReference = { id, type: type as KibanaAssetType }; - - return reference; -} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap deleted file mode 100644 index 638ed4b6118c9..0000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap +++ /dev/null @@ -1,133 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`a kibana asset id and its reference ids are appended with package name changeAssetIds output matches snapshot: dashboard.json 1`] = ` -{ - "attributes": { - "description": "Overview dashboard for the Nginx integration in Metrics", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } - }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "1", - "w": 24, - "x": 24, - "y": 0 - }, - "panelIndex": "1", - "panelRefName": "panel_0", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "2", - "w": 24, - "x": 24, - "y": 12 - }, - "panelIndex": "2", - "panelRefName": "panel_1", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "3", - "w": 24, - "x": 0, - "y": 12 - }, - "panelIndex": "3", - "panelRefName": "panel_2", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "4", - "w": 24, - "x": 0, - "y": 0 - }, - "panelIndex": "4", - "panelRefName": "panel_3", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "5", - "w": 48, - "x": 0, - "y": 24 - }, - "panelIndex": "5", - "panelRefName": "panel_4", - "version": "7.3.0" - } - ], - "timeRestore": false, - "title": "[Metrics Nginx] Overview ECS", - "version": 1 - }, - "id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", - "migrationVersion": { - "dashboard": "7.3.0" - }, - "references": [ - { - "id": "metrics-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type": "index-pattern" - }, - { - "id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_0", - "type": "search" - }, - { - "id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_1", - "type": "map" - }, - { - "id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_2", - "type": "dashboard" - }, - { - "id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_3", - "type": "visualization" - }, - { - "id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_4", - "type": "visualization" - } - ], - "type": "dashboard" -} -`; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json deleted file mode 100644 index e28a61ae5e18c..0000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "attributes": { - "description": "Overview dashboard for the Nginx integration in Metrics", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } - }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "1", - "w": 24, - "x": 24, - "y": 0 - }, - "panelIndex": "1", - "panelRefName": "panel_0", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "2", - "w": 24, - "x": 24, - "y": 12 - }, - "panelIndex": "2", - "panelRefName": "panel_1", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "3", - "w": 24, - "x": 0, - "y": 12 - }, - "panelIndex": "3", - "panelRefName": "panel_2", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "4", - "w": 24, - "x": 0, - "y": 0 - }, - "panelIndex": "4", - "panelRefName": "panel_3", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "5", - "w": 48, - "x": 0, - "y": 24 - }, - "panelIndex": "5", - "panelRefName": "panel_4", - "version": "7.3.0" - } - ], - "timeRestore": false, - "title": "[Metrics Nginx] Overview ECS", - "version": 1 - }, - "id": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", - "migrationVersion": { - "dashboard": "7.3.0" - }, - "references": [ - { - "id": "metrics-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type": "index-pattern" - }, - { - "id": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_0", - "type": "search" - }, - { - "id": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_1", - "type": "map" - }, - { - "id": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_2", - "type": "dashboard" - }, - { - "id": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_3", - "type": "visualization" - }, - { - "id": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", - "name": "panel_4", - "type": "visualization" - } - ], - "type": "dashboard" -} \ No newline at end of file diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts deleted file mode 100644 index f9bc4cdbf203f..0000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { readFileSync } from 'fs'; -import path from 'path'; -import { getAssetId, changeAssetIds } from '../install'; - -expect.addSnapshotSerializer({ - print(val) { - return JSON.stringify(val, null, 2); - }, - - test(val) { - return val; - }, -}); - -describe('a kibana asset id and its reference ids are appended with package name', () => { - const assetPath = path.join(__dirname, './dashboard.json'); - const kibanaAsset = JSON.parse(readFileSync(assetPath, 'utf-8')); - const pkgName = 'nginx'; - const modifiedAssetObject = changeAssetIds(kibanaAsset, pkgName); - - test('changeAssetIds output matches snapshot', () => { - expect(modifiedAssetObject).toMatchSnapshot(path.basename(assetPath)); - }); - - test('getAssetId', () => { - const id = '47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs'; - expect(getAssetId(id, pkgName)).toBe(`${pkgName}-${id}`); - }); -}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts new file mode 100644 index 0000000000000..b623295c5e060 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObject, SavedObjectsBulkCreateObject } from 'src/core/server'; +import { AssetType } from '../../../types'; +import * as Registry from '../registry'; + +type ArchiveAsset = Pick; +type SavedObjectToBe = Required & { type: AssetType }; + +export async function getObject(key: string) { + const buffer = Registry.getAsset(key); + + // cache values are buffers. convert to string / JSON + const json = buffer.toString('utf8'); + // convert that to an object + const asset: ArchiveAsset = JSON.parse(json); + + const { type, file } = Registry.pathParts(key); + const savedObject: SavedObjectToBe = { + type, + id: file.replace('.json', ''), + attributes: asset.attributes, + references: asset.references || [], + migrationVersion: asset.migrationVersion || {}, + }; + + return savedObject; +} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index 57c4f77432455..4bb803dfaf912 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -23,7 +23,7 @@ export { SearchParams, } from './get'; -export { installPackage, ensureInstalledPackage } from './install'; +export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install'; export { removeInstallation } from './remove'; type RequiredPackage = 'system' | 'endpoint'; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 8f73bc9a02765..910283549abdf 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -4,12 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsClientContract } from 'src/core/server'; +import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; import Boom from 'boom'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, Installation, + KibanaAssetType, CallESAsCurrentUser, DefaultPackages, ElasticsearchAssetType, @@ -17,7 +18,7 @@ import { } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; -import { installKibanaAssets } from '../kibana/assets/install'; +import { getObject } from './get_objects'; import { getInstallation, getInstallationObject, isRequiredPackage } from './index'; import { installTemplates } from '../elasticsearch/template/install'; import { generateESIndexPatterns } from '../elasticsearch/template/template'; @@ -120,6 +121,7 @@ export async function installPackage(options: { installKibanaAssets({ savedObjectsClient, pkgName, + pkgVersion, paths, }), installPipelines(registryPackageInfo, paths, callCluster), @@ -183,6 +185,27 @@ export async function installPackage(options: { }); } +// TODO: make it an exhaustive list +// e.g. switch statement with cases for each enum key returning `never` for default case +export async function installKibanaAssets(options: { + savedObjectsClient: SavedObjectsClientContract; + pkgName: string; + pkgVersion: string; + paths: string[]; +}) { + const { savedObjectsClient, paths } = options; + + // Only install Kibana assets during package installation. + const kibanaAssetTypes = Object.values(KibanaAssetType); + const installationPromises = kibanaAssetTypes.map(async (assetType) => + installKibanaSavedObjects({ savedObjectsClient, assetType, paths }) + ); + + // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] + // call .flat to flatten into one dimensional array + return Promise.all(installationPromises).then((results) => results.flat()); +} + export async function saveInstallationReferences(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; @@ -217,3 +240,34 @@ export async function saveInstallationReferences(options: { return toSaveAssetRefs; } + +async function installKibanaSavedObjects({ + savedObjectsClient, + assetType, + paths, +}: { + savedObjectsClient: SavedObjectsClientContract; + assetType: KibanaAssetType; + paths: string[]; +}) { + const isSameType = (path: string) => assetType === Registry.pathParts(path).type; + const pathsOfType = paths.filter((path) => isSameType(path)); + const toBeSavedObjects = await Promise.all(pathsOfType.map(getObject)); + + if (toBeSavedObjects.length === 0) { + return []; + } else { + const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { + overwrite: true, + }); + const createdObjects = createResults.saved_objects; + const installed = createdObjects.map(toAssetReference); + return installed; + } +} + +function toAssetReference({ id, type }: SavedObject) { + const reference: AssetReference = { id, type: type as KibanaAssetType }; + + return reference; +}