From cf52dd9e3992bb4ca7f945b02568fd60c75acf45 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 12 Jun 2023 16:26:50 +0200 Subject: [PATCH 1/5] :sparkles: Add metric migration for default formatter --- .../apis/saved_objects/bulk_get.ts | 4 +- .../api_integration/apis/saved_objects/get.ts | 4 +- .../make_lens_embeddable_factory.ts | 9 +++ .../server/migrations/common_migrations.ts | 66 +++++++++++++++++++ .../migrations/saved_object_migrations.ts | 7 ++ 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/test/api_integration/apis/saved_objects/bulk_get.ts b/test/api_integration/apis/saved_objects/bulk_get.ts index 469fec185282f..62f95c718ab16 100644 --- a/test/api_integration/apis/saved_objects/bulk_get.ts +++ b/test/api_integration/apis/saved_objects/bulk_get.ts @@ -230,10 +230,10 @@ export default function ({ getService }: FtrProviderContext) { type: 'dashboard', namespaces: ['default'], migrationVersion: { - dashboard: '8.7.0', + dashboard: '8.9.0', }, coreMigrationVersion: '8.8.0', - typeMigrationVersion: '8.7.0', + typeMigrationVersion: '8.9.0', updated_at: '2015-01-01T00:00:00.000Z', created_at: '2015-01-01T00:00:00.000Z', version: resp.body.saved_objects[3].version, diff --git a/test/api_integration/apis/saved_objects/get.ts b/test/api_integration/apis/saved_objects/get.ts index 2676aa5b26af3..7a1b272b515fc 100644 --- a/test/api_integration/apis/saved_objects/get.ts +++ b/test/api_integration/apis/saved_objects/get.ts @@ -80,10 +80,10 @@ export default function ({ getService }: FtrProviderContext) { type: 'dashboard', namespaces: ['default'], migrationVersion: { - dashboard: '8.7.0', + dashboard: '8.9.0', }, coreMigrationVersion: '8.8.0', - typeMigrationVersion: '8.7.0', + typeMigrationVersion: '8.9.0', updated_at: resp.body.updated_at, created_at: resp.body.created_at, version: resp.body.version, diff --git a/x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts b/x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts index d687b5bba3050..9262b6054af09 100644 --- a/x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts +++ b/x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts @@ -34,6 +34,7 @@ import { commonMigratePartitionChartGroups, commonMigratePartitionMetrics, commonMigrateIndexPatternDatasource, + commonMigrateMetricFormatter, } from '../migrations/common_migrations'; import { CustomVisualizationMigrations, @@ -42,6 +43,7 @@ import { LensDocShape810, LensDocShape850, LensDocShapePre712, + LensDocShape860, VisState716, VisState810, VisState850, @@ -168,6 +170,13 @@ export const makeLensEmbeddableFactory = attributes: migratedLensState, } as unknown as SerializableRecord; }, + '8.9.0': (state) => { + const lensState = state as unknown as SavedObject; + return { + ...lensState, + attributes: commonMigrateMetricFormatter(lensState.attributes), + } as unknown as SerializableRecord; + }, // FOLLOW THESE GUIDELINES IF YOU ARE ADDING A NEW MIGRATION! // 1. Make sure you are applying migrations for a given version in the same order here as they are applied in x-pack/plugins/lens/server/migrations/saved_object_migrations.ts }), diff --git a/x-pack/plugins/lens/server/migrations/common_migrations.ts b/x-pack/plugins/lens/server/migrations/common_migrations.ts index 10cf0a3847ee9..5073faf3ac177 100644 --- a/x-pack/plugins/lens/server/migrations/common_migrations.ts +++ b/x-pack/plugins/lens/server/migrations/common_migrations.ts @@ -571,3 +571,69 @@ export const commonMigratePartitionMetrics = (attributes: LensDocShape860; }>; }; + +export const commonMigrateMetricFormatter = (attributes: LensDocShape860) => { + if (attributes.visualizationType !== 'lnsMetric') { + return attributes as LensDocShape860; + } + if (!attributes.state.datasourceStates.formBased) { + return attributes as LensDocShape860; + } + + type LayersType = LensDocShape860['state']['datasourceStates']['formBased']['layers']; + + const updatedLayersWithCompactFormatters: LayersType = {}; + for (const [layerId, layer] of Object.entries( + attributes.state.datasourceStates.formBased.layers + )) { + const newColumns: Record> = {}; + for (const [id, column] of Object.entries(layer.columns)) { + const params = column.params as Record< + 'format', + Record }> + >; + if (column.isBucketed) { + newColumns[id] = column; + } else { + // When value formatting is set to Default, assume nothing + if (!params?.format) { + newColumns[id] = column; + } else { + // Metric only support numeric values + newColumns[id] = { + ...column, + params: { + ...params, + format: { + id: 'number', + ...params?.format, + params: { + ...params?.format?.params, + compact: true, + }, + }, + }, + }; + } + } + } + updatedLayersWithCompactFormatters[layerId] = { + ...layer, + columns: newColumns, + }; + } + + return { + ...attributes, + state: { + ...attributes.state, + datasourceStates: { + ...attributes.state.datasourceStates, + formBased: { + ...attributes.state.datasourceStates.formBased, + layers: updatedLayersWithCompactFormatters, + }, + }, + }, + }; +}; diff --git a/x-pack/plugins/lens/server/migrations/saved_object_migrations.ts b/x-pack/plugins/lens/server/migrations/saved_object_migrations.ts index cd119cca100a6..a981dd332d67e 100644 --- a/x-pack/plugins/lens/server/migrations/saved_object_migrations.ts +++ b/x-pack/plugins/lens/server/migrations/saved_object_migrations.ts @@ -62,6 +62,7 @@ import { commonMigratePartitionChartGroups, commonMigratePartitionMetrics, commonMigrateIndexPatternDatasource, + commonMigrateMetricFormatter, } from './common_migrations'; interface LensDocShapePre710 { @@ -562,6 +563,11 @@ const migratePartitionMetrics: SavedObjectMigrationFn = (doc) => ({ + ...doc, + attributes: commonMigrateMetricFormatter(doc.attributes), +}); + const lensMigrations: SavedObjectMigrationMap = { '7.7.0': removeInvalidAccessors, // The order of these migrations matter, since the timefield migration relies on the aggConfigs @@ -584,6 +590,7 @@ const lensMigrations: SavedObjectMigrationMap = { '8.3.0': flow(lockOldMetricVisSettings, preserveOldLegendSizeDefault, fixValueLabelsInXY), '8.5.0': flow(migrateMetricIds, enrichAnnotationLayers, migratePartitionChartGroups), '8.6.0': flow(migrateIndexPatternDatasource, migratePartitionMetrics), + '8.9.0': migrateMetricFormatter, // FOLLOW THESE GUIDELINES IF YOU ARE ADDING A NEW MIGRATION! // 1. Make sure you are applying migrations for a given version in the same order here as they are applied in x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts }; From 9161a7448e3a80e84132c53d69b7a4d10a6e9e2c Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 13 Jun 2023 12:45:11 +0200 Subject: [PATCH 2/5] :white_check_mark: Add migrations tests --- .../migrations/common_migrations.test.ts | 93 ++++++++++++++++++- .../server/migrations/common_migrations.ts | 13 +-- .../saved_object_migrations.test.ts | 1 + 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/lens/server/migrations/common_migrations.test.ts b/x-pack/plugins/lens/server/migrations/common_migrations.test.ts index 85e46d732e935..cb597ec865e62 100644 --- a/x-pack/plugins/lens/server/migrations/common_migrations.test.ts +++ b/x-pack/plugins/lens/server/migrations/common_migrations.test.ts @@ -11,8 +11,9 @@ import { getLensDataViewMigrations, getLensFilterMigrations, commonMigratePartitionChartGroups, + commonMigrateMetricFormatter, } from './common_migrations'; -import { LensDocShape840 } from './types'; +import { LensDocShape840, LensDocShape860 } from './types'; describe('Lens migrations', () => { describe('applying filter migrations', () => { @@ -169,4 +170,94 @@ describe('Lens migrations', () => { `); }); }); + + describe('migrate new metric formatters to uniform configuration', () => { + function createVisualizationSO( + type: string, + lensFormatId: undefined | string, + bucketed: boolean = false + ): { attributes: LensDocShape860 } { + return { + attributes: { + filters: [], + visualizationType: 'lnsMetric', + title: '', + expression: '', + state: { + datasourceMetaData: { filterableIndexPatterns: [] }, + visualization: {}, + filters: [], + query: { query: '', language: 'kuery' }, + datasourceStates: { + formBased: { + currentIndexPatternId: '', + layers: { + test: { + columnOrder: [], + columns: { + col1: { + operationType: 'test', + isBucketed: bucketed, + params: { + ...(lensFormatId ? { format: { id: lensFormatId } } : {}), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }; + } + + it('should not apply the migration non-metric visualization types', () => { + const lensFormatId = 'number'; + const lensVisualizationSavedObject = createVisualizationSO('lnsXY', lensFormatId); + const migratedColumn = commonMigrateMetricFormatter(lensVisualizationSavedObject.attributes) + .state.datasourceStates.formBased.layers.test.columns.col1; + + expect(migratedColumn.params).toEqual({ format: { id: lensFormatId } }); + }); + + it.each` + lensFormat | bucketed | expectedCompact + ${'default'} | ${true} | ${false} + ${'number'} | ${true} | ${false} + ${'number'} | ${false} | ${true} + ${'percentage'} | ${false} | ${true} + ${'bytes'} | ${false} | ${false} + ${'bits'} | ${false} | ${false} + ${'default'} | ${false} | ${false} + `( + 'should migrate lens column (bucketed: $bucketed) format configuration "$lensFormat" into new explicit formatter (compact: $expectedCompact)', + ({ lensFormat, bucketed, expectedCompact }) => { + const lensFormatId = lensFormat === 'default' ? undefined : lensFormat; + const lensVisualizationSavedObject = createVisualizationSO( + 'lnsMetric', + lensFormatId, + bucketed + ); + + const migratedColumn = commonMigrateMetricFormatter(lensVisualizationSavedObject.attributes) + .state.datasourceStates.formBased.layers.test.columns.col1; + + if (bucketed) { + expect(migratedColumn.params).toEqual( + lensFormatId ? { format: { id: lensFormatId } } : {} + ); + } + if (!lensFormatId) { + expect(migratedColumn.params).toEqual({}); + } else if (bucketed || !expectedCompact) { + expect(migratedColumn.params).toEqual({ format: { id: lensFormatId } }); + } else { + expect(migratedColumn.params).toEqual({ + format: { id: lensFormatId, params: { compact: true } }, + }); + } + } + ); + }); }); diff --git a/x-pack/plugins/lens/server/migrations/common_migrations.ts b/x-pack/plugins/lens/server/migrations/common_migrations.ts index 5073faf3ac177..c1be1008ad682 100644 --- a/x-pack/plugins/lens/server/migrations/common_migrations.ts +++ b/x-pack/plugins/lens/server/migrations/common_migrations.ts @@ -588,25 +588,26 @@ export const commonMigrateMetricFormatter = (attributes: LensDocShape860> = {}; for (const [id, column] of Object.entries(layer.columns)) { - const params = column.params as Record< - 'format', - Record }> - >; + const params = column.params as { + format?: { id: string; params: Record }; + }; if (column.isBucketed) { newColumns[id] = column; } else { // When value formatting is set to Default, assume nothing - if (!params?.format) { + // Bytes and bits are already compact + if (!params?.format || ['bytes', 'bits'].includes(params.format.id)) { newColumns[id] = column; } else { // Metric only support numeric values + // suffix is not taken into account as it wasn't possible in metric visualization before this version newColumns[id] = { ...column, params: { ...params, format: { - id: 'number', ...params?.format, + id: params?.format.id || 'number', params: { ...params?.format?.params, compact: true, diff --git a/x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts b/x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts index e80246f5f4c2c..e765ba5f99338 100644 --- a/x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts +++ b/x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts @@ -2537,4 +2537,5 @@ describe('Lens migrations', () => { ); }); }); + // For 8.8.0 tests are already executed at unit level in common_migrations }); From 4e844ecb9b83a1813146cfef55d3a5ccf74ee23d Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 13 Jun 2023 12:47:29 +0200 Subject: [PATCH 3/5] :wrench: manually fix --- .../migrations/group2/check_registered_types.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 97b73ce52f6cb..f1bdfb1660d68 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -66,11 +66,11 @@ describe('checking migration metadata changes on all registered SO types', () => "app_search_telemetry": "36234f19573ad397ac30197c45ac219921cc3106", "application_usage_daily": "20142d23fe5d05ba22b4bc46614d99883bc488f0", "application_usage_totals": "a29ab014edc20382b9ce22ede221b18cee5d93a6", - "canvas-element": "b39dfe54b9ff3ecc4c6bc5bed6a14b0a0fe83644", - "canvas-workpad": "4df66cf25eba8e7e25c061a1b2a5aadbb1f436e9", - "canvas-workpad-template": "52a35f737b579a570510fca361fddd158d2a92ad", + "canvas-element": "cdedc2123eb8a1506b87a56b0bcce60f4ec08bc8", + "canvas-workpad": "9d82aafb19586b119e5c9382f938abe28c26ca5c", + "canvas-workpad-template": "c077b0087346776bb3542b51e1385d172cb24179", "cases": "3d968144040b829dddb8826bad90f9f0ab57a403", - "cases-comments": "ded400d82c5ea26959c2ee8e54896981d499a226", + "cases-comments": "5cb0a421588831c2a950e50f486048b8aabbae25", "cases-configure": "44ed7b8e0f44df39516b8870589b89e32224d2bf", "cases-connector-mappings": "f9d1ac57e484e69506c36a8051e4d61f4a8cfd25", "cases-telemetry": "f219eb7e26772884342487fc9602cfea07b3cedc", @@ -81,7 +81,7 @@ describe('checking migration metadata changes on all registered SO types', () => "connector_token": "5a9ac29fe9c740eb114e9c40517245c71706b005", "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "c151324d5f85178169395eecb12bac6b96064654", - "dashboard": "1635368413415b340ae6f43fcd0a55c5dcdd4f41", + "dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", "epm-packages": "2449bb565f987eff70b1b39578bb17e90c404c6e", @@ -112,7 +112,7 @@ describe('checking migration metadata changes on all registered SO types', () => "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", "legacy-url-alias": "9b8cca3fbb2da46fd12823d3cd38fdf1c9f24bc8", - "lens": "fd6a7938f41888e5ee499fcc5e749489981393aa", + "lens": "5cfa2c52b979b4f8df56dd13c477e152183468b9", "lens-ui-telemetry": "8c47a9e393861f76e268345ecbadfc8a5fb1e0bd", "maintenance-window": "d893544460abad56ff7a0e25b78f78776dfe10d1", "map": "76c71023bd198fb6b1163b31bafd926fe2ceb9da", From 505341df209c3f21bc850b8d65b19129e7fce4f0 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 14 Jun 2023 12:36:29 +0200 Subject: [PATCH 4/5] :white_check_mark: Fix test --- x-pack/plugins/lens/server/migrations/common_migrations.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/server/migrations/common_migrations.test.ts b/x-pack/plugins/lens/server/migrations/common_migrations.test.ts index cb597ec865e62..692494df46b3c 100644 --- a/x-pack/plugins/lens/server/migrations/common_migrations.test.ts +++ b/x-pack/plugins/lens/server/migrations/common_migrations.test.ts @@ -180,7 +180,7 @@ describe('Lens migrations', () => { return { attributes: { filters: [], - visualizationType: 'lnsMetric', + visualizationType: type, title: '', expression: '', state: { From e2c5d2577c7b2898075a18c422c9a612e49455e4 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 14 Jun 2023 12:37:11 +0200 Subject: [PATCH 5/5] :camera_flash: Fix snashot id --- .../migrations/group2/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 984f59ab6253f..f06a2d16aed30 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -69,7 +69,7 @@ describe('checking migration metadata changes on all registered SO types', () => "canvas-element": "cdedc2123eb8a1506b87a56b0bcce60f4ec08bc8", "canvas-workpad": "9d82aafb19586b119e5c9382f938abe28c26ca5c", "canvas-workpad-template": "c077b0087346776bb3542b51e1385d172cb24179", - "cases": "3d968144040b829dddb8826bad90f9f0ab57a403", + "cases": "b43a8ce985c406167e1d115381805a48cb3b0e61", "cases-comments": "5cb0a421588831c2a950e50f486048b8aabbae25", "cases-configure": "44ed7b8e0f44df39516b8870589b89e32224d2bf", "cases-connector-mappings": "f9d1ac57e484e69506c36a8051e4d61f4a8cfd25",