From 0c735e1c7d285ac4c43223b92e83db6a1fabc2e5 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 16 Jan 2020 13:52:57 -0800 Subject: [PATCH 1/4] Sort SavedObjects server-side instead of by _id in ES Sorting/Aggregating by _id was removed in https://github.com/elastic/kibana/issues/52517 Retains sorting as requested by https://github.com/elastic/kibana/issues/29747 Signed-off-by: Tyler Smalley --- .../get_sorted_objects_for_export.test.ts | 121 +++++++++++++++++- .../export/get_sorted_objects_for_export.ts | 10 +- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts index 9a3449b65a941..97accc5c7c998 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts @@ -108,8 +108,6 @@ describe('getSortedObjectsForExport()', () => { "namespace": undefined, "perPage": 500, "search": undefined, - "sortField": "_id", - "sortOrder": "asc", "type": Array [ "index-pattern", "search", @@ -256,8 +254,6 @@ describe('getSortedObjectsForExport()', () => { "namespace": undefined, "perPage": 500, "search": "foo", - "sortField": "_id", - "sortOrder": "asc", "type": Array [ "index-pattern", "search", @@ -345,8 +341,6 @@ describe('getSortedObjectsForExport()', () => { "namespace": "foo", "perPage": 500, "search": undefined, - "sortField": "_id", - "sortOrder": "asc", "type": Array [ "index-pattern", "search", @@ -399,6 +393,121 @@ describe('getSortedObjectsForExport()', () => { ).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`); }); + test('sorts objects within type', async () => { + savedObjectsClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + id: '3', + type: 'index-pattern', + attributes: { + name: 'baz', + }, + references: [], + }, + { + id: '1', + type: 'index-pattern', + attributes: { + name: 'foo', + }, + references: [], + }, + { + id: '2', + type: 'index-pattern', + attributes: { + name: 'bar', + }, + references: [], + }, + ], + }); + const exportStream = await getSortedObjectsForExport({ + exportSizeLimit: 10000, + savedObjectsClient, + types: ['index-pattern'], + objects: [ + { + type: 'index-pattern', + id: '3', + }, + { + type: 'index-pattern', + id: '1', + }, + { + type: 'index-pattern', + id: '2', + }, + ], + }); + const response = await readStreamToCompletion(exportStream); + expect(response).toMatchInlineSnapshot(` + Array [ + Object { + "attributes": Object { + "name": "foo", + }, + "id": "1", + "references": Array [], + "type": "index-pattern", + }, + Object { + "attributes": Object { + "name": "bar", + }, + "id": "2", + "references": Array [], + "type": "index-pattern", + }, + Object { + "attributes": Object { + "name": "baz", + }, + "id": "3", + "references": Array [], + "type": "index-pattern", + }, + Object { + "exportedCount": 3, + "missingRefCount": 0, + "missingReferences": Array [], + }, + ] + `); + expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` + [MockFunction] { + "calls": Array [ + Array [ + Array [ + Object { + "id": "3", + "type": "index-pattern", + }, + Object { + "id": "1", + "type": "index-pattern", + }, + Object { + "id": "2", + "type": "index-pattern", + }, + ], + Object { + "namespace": undefined, + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); + }); + test('exports selected objects and sorts them', async () => { savedObjectsClient.bulkGet.mockResolvedValueOnce({ saved_objects: [ diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts index e1a705a36db75..7fd357d16d76a 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts @@ -19,7 +19,7 @@ import Boom from 'boom'; import { createListStream } from '../../../../legacy/utils/streams'; -import { SavedObjectsClientContract } from '../types'; +import { SavedObjectsClientContract, SavedObject } from '../types'; import { fetchNestedDependencies } from './inject_nested_depdendencies'; import { sortObjects } from './sort_objects'; @@ -69,6 +69,10 @@ export interface SavedObjectsExportResultDetails { }>; } +function compareSavedObjects(a: SavedObject, b: SavedObject) { + return a.id > b.id ? 1 : -1; +} + async function fetchObjectsToExport({ objects, types, @@ -105,8 +109,6 @@ async function fetchObjectsToExport({ const findResponse = await savedObjectsClient.find({ type: types, search, - sortField: '_id', - sortOrder: 'asc', perPage: exportSizeLimit, namespace, }); @@ -137,7 +139,7 @@ export async function getSortedObjectsForExport({ exportSizeLimit, namespace, }); - let exportedObjects = [...rootObjects]; + let exportedObjects = [...rootObjects.sort(compareSavedObjects)]; let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = []; if (includeReferencesDeep) { const fetchResult = await fetchNestedDependencies(rootObjects, savedObjectsClient, namespace); From c821ba3207fdaacd3e9f5d8136c22d5729f625fb Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Fri, 24 Jan 2020 13:13:19 -0800 Subject: [PATCH 2/4] Adds comment Signed-off-by: Tyler Smalley --- .../server/saved_objects/export/get_sorted_objects_for_export.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts index 7fd357d16d76a..9a54d65e1cfb0 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts @@ -69,6 +69,7 @@ export interface SavedObjectsExportResultDetails { }>; } +// sorts objects by id for consistent object sequence function compareSavedObjects(a: SavedObject, b: SavedObject) { return a.id > b.id ? 1 : -1; } From 48883f95f2080fb5d57ee5f02484fec6e0af0527 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Mon, 27 Jan 2020 12:29:04 -0800 Subject: [PATCH 3/4] Sorts as part of fetchObjectsToExport Signed-off-by: Tyler Smalley --- .../get_sorted_objects_for_export.test.ts | 50 ++----------------- .../export/get_sorted_objects_for_export.ts | 20 ++++---- 2 files changed, 15 insertions(+), 55 deletions(-) diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts index 97accc5c7c998..fafa04447ddfe 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts @@ -394,7 +394,10 @@ describe('getSortedObjectsForExport()', () => { }); test('sorts objects within type', async () => { - savedObjectsClient.bulkGet.mockResolvedValueOnce({ + savedObjectsClient.find.mockResolvedValueOnce({ + total: 3, + per_page: 10000, + page: 1, saved_objects: [ { id: '3', @@ -426,20 +429,6 @@ describe('getSortedObjectsForExport()', () => { exportSizeLimit: 10000, savedObjectsClient, types: ['index-pattern'], - objects: [ - { - type: 'index-pattern', - id: '3', - }, - { - type: 'index-pattern', - id: '1', - }, - { - type: 'index-pattern', - id: '2', - }, - ], }); const response = await readStreamToCompletion(exportStream); expect(response).toMatchInlineSnapshot(` @@ -475,37 +464,6 @@ describe('getSortedObjectsForExport()', () => { }, ] `); - expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Array [ - Object { - "id": "3", - "type": "index-pattern", - }, - Object { - "id": "1", - "type": "index-pattern", - }, - Object { - "id": "2", - "type": "index-pattern", - }, - ], - Object { - "namespace": undefined, - }, - ], - ], - "results": Array [ - Object { - "type": "return", - "value": Promise {}, - }, - ], - } - `); }); test('exports selected objects and sorts them', async () => { diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts index 9a54d65e1cfb0..d78eded391f6e 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts @@ -69,11 +69,6 @@ export interface SavedObjectsExportResultDetails { }>; } -// sorts objects by id for consistent object sequence -function compareSavedObjects(a: SavedObject, b: SavedObject) { - return a.id > b.id ? 1 : -1; -} - async function fetchObjectsToExport({ objects, types, @@ -116,7 +111,11 @@ async function fetchObjectsToExport({ if (findResponse.total > exportSizeLimit) { throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`); } - return findResponse.saved_objects; + + // sorts server-side by _id, since it's only available in fielddata + return findResponse.saved_objects.sort((a: SavedObject, b: SavedObject) => + a.id > b.id ? 1 : -1 + ); } else { throw Boom.badRequest('Either `type` or `objects` are required.'); } @@ -140,14 +139,17 @@ export async function getSortedObjectsForExport({ exportSizeLimit, namespace, }); - let exportedObjects = [...rootObjects.sort(compareSavedObjects)]; + let exportedObjects = [...rootObjects]; let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = []; + if (includeReferencesDeep) { const fetchResult = await fetchNestedDependencies(rootObjects, savedObjectsClient, namespace); - exportedObjects = fetchResult.objects; + exportedObjects = sortObjects(fetchResult.objects); missingReferences = fetchResult.missingRefs; + } else { + exportedObjects = sortObjects(rootObjects); } - exportedObjects = sortObjects(exportedObjects); + const exportDetails: SavedObjectsExportResultDetails = { exportedCount: exportedObjects.length, missingRefCount: missingReferences.length, From 6b9beccf72f052316176e73f122cc4d6fa716afb Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 28 Jan 2020 09:37:35 -0800 Subject: [PATCH 4/4] Cleanup object assignment Signed-off-by: Tyler Smalley --- .../saved_objects/export/get_sorted_objects_for_export.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts index d78eded391f6e..a4dfacfd9e34f 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts @@ -139,7 +139,7 @@ export async function getSortedObjectsForExport({ exportSizeLimit, namespace, }); - let exportedObjects = [...rootObjects]; + let exportedObjects = []; let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = []; if (includeReferencesDeep) {