From ae607349eb475b82690ef2fdbae30a7f0ea61921 Mon Sep 17 00:00:00 2001
From: Dmitrii Shevchenko <dmitrii.shevchenko@elastic.co>
Date: Tue, 3 Jan 2023 12:07:17 +0100
Subject: [PATCH] [Fleet] Fix crashes when installing a package with a lot of
 saved objects (#148141)

**Resolves: https://github.com/elastic/kibana/issues/147695,
https://github.com/elastic/kibana/issues/148174**
**Related to: https://github.com/elastic/kibana/pull/145851,
https://github.com/elastic/kibana/issues/137420**

## Summary

This PR improves the stability of the Fleet packages installation
process with many saved objects.

1. Changed mappings of the `installed_kibana` and `package_assets`
fields from `nested` to `object` with `enabled: false`. Values of those
fields were retrieved from `_source`, and no queries or aggregations
were performed against them. So the mappings were unused, while during
the installation of packages containing more than 10,000 saved objects,
an error was thrown due to the nested field limitations:

   ```
Error installing security_detection_engine 8.4.1: The number of nested
documents has exceeded the allowed limit of
   [10000].
This limit can be set by changing the
[index.mapping.nested_objects.limit] index level setting.
   ```
2. Improved the deletion of previous package assets by switching from
sending multiple `savedObjectsClient.delete` requests in parallel to a
single `savedObjectsClient.bulkDelete` request. Multiple parallel
requests were causing the Elasticsearch cluster to stop responding for
some time; see [this
ticket](https://github.com/elastic/kibana/issues/147695) for more info.

**Before**
![Screenshot 2022-12-28 at 11 09
35](https://user-images.githubusercontent.com/1938181/209816219-ade6dd0a-0d56-4acc-929e-b88571f0fe81.png)

**After**
![Screenshot 2022-12-28 at 13 56
44](https://user-images.githubusercontent.com/1938181/209816209-16c69922-4ae2-4589-9aa4-5a28050037f4.png)
---
 .../migrations/check_registered_types.test.ts      |  2 +-
 x-pack/plugins/fleet/server/saved_objects/index.ts | 14 ++++----------
 .../fleet/server/services/epm/archive/storage.ts   |  5 ++---
 .../fleet/server/services/epm/packages/remove.ts   |  5 +----
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts
index 2c6fee861ce7d..048efa5008533 100644
--- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts
+++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts
@@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0",
         "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88",
         "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5",
-        "epm-packages": "2915aee4302d4b00472ed05c21f59b7d498b5206",
+        "epm-packages": "7d80ba3f1fcd80316aa0b112657272034b66d5a8",
         "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771",
         "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd",
         "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c",
diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
index 6c87e894e5777..03f072747a33d 100644
--- a/x-pack/plugins/fleet/server/saved_objects/index.ts
+++ b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -266,19 +266,13 @@ const getSavedObjectTypes = (
           },
         },
         installed_kibana: {
-          type: 'nested',
-          properties: {
-            id: { type: 'keyword' },
-            type: { type: 'keyword' },
-          },
+          type: 'object',
+          enabled: false,
         },
         installed_kibana_space_id: { type: 'keyword' },
         package_assets: {
-          type: 'nested',
-          properties: {
-            id: { type: 'keyword' },
-            type: { type: 'keyword' },
-          },
+          type: 'object',
+          enabled: false,
         },
         install_started_at: { type: 'date' },
         install_version: { type: 'keyword' },
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts
index c6176b94f5fd2..3b6f38971eff7 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts
@@ -102,10 +102,9 @@ export async function removeArchiveEntries(opts: {
 }) {
   const { savedObjectsClient, refs } = opts;
   if (!refs) return;
-  const results = await Promise.all(
-    refs.map((ref) => savedObjectsClient.delete(ASSETS_SAVED_OBJECT_TYPE, ref.id))
+  return savedObjectsClient.bulkDelete(
+    refs.map((ref) => ({ id: ref.id, type: ASSETS_SAVED_OBJECT_TYPE }))
   );
-  return results;
 }
 
 export async function saveArchiveEntries(opts: {
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts
index 61780c7977166..4a4183965913a 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts
@@ -125,11 +125,8 @@ async function deleteKibanaAssets(
   // in the case of a partial install, it is expected that some assets will be not found
   // we filter these out before calling delete
   const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type }));
-  const promises = assetsToDelete.map(async ({ id, type }) => {
-    return savedObjectsClient.delete(type, id, { namespace });
-  });
 
-  return Promise.all(promises);
+  return savedObjectsClient.bulkDelete(assetsToDelete, { namespace });
 }
 
 function deleteESAssets(