From 8052f03f61b2d89f20780caf5641691a1ccece90 Mon Sep 17 00:00:00 2001
From: Josh Dover <1813008+joshdover@users.noreply.github.com>
Date: Mon, 20 Nov 2023 18:47:57 +0100
Subject: [PATCH 01/13] [Fleet] Cap setup attempts to 50 on Serverless
 (#171550)

## Summary

If there is a bug in Fleet setup, we can retrigger rollovers on each
attempt, causing shard explosion in Elasticsearch. We need a sane limit
to prevent this from happening, which this PR introduces.

The impact of Fleet setup failing to complete (Agents may not be able to
be enrolled) is much smaller than causing shard explosion, so this seems
like an acceptable tradeoff.

This is only a small part of the overall solution - in the current
incident we have, it's still unclear why we are failing to rollover the
index and getting into this loop.

### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
---
 x-pack/plugins/fleet/server/plugin.ts | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts
index 0373971c664e2..8333d0257a8f0 100644
--- a/x-pack/plugins/fleet/server/plugin.ts
+++ b/x-pack/plugins/fleet/server/plugin.ts
@@ -524,6 +524,9 @@ export class FleetPlugin
 
     this.policyWatcher.start(licenseService);
 
+    // We only retry when this feature flag is enabled (Serverless)
+    const setupAttempts = this.configInitialValue.internal?.retrySetupOnBoot ? 25 : 1;
+
     const fleetSetupPromise = (async () => {
       try {
         // Fleet remains `available` during setup as to excessively delay Kibana's boot process.
@@ -555,10 +558,9 @@ export class FleetPlugin
             );
           },
           {
-            // We only retry when this feature flag is enabled
-            numOfAttempts: this.configInitialValue.internal?.retrySetupOnBoot ? Infinity : 1,
-            // 250ms initial backoff
-            startingDelay: 250,
+            numOfAttempts: setupAttempts,
+            // 1s initial backoff
+            startingDelay: 1000,
             // 5m max backoff
             maxDelay: 60000 * 5,
             timeMultiple: 2,
@@ -566,7 +568,7 @@ export class FleetPlugin
             jitter: 'full',
             retry: (error: any, attemptCount: number) => {
               const summary = `Fleet setup attempt ${attemptCount} failed, will retry after backoff`;
-              logger.debug(summary, { error: { message: error } });
+              logger.warn(summary, { error: { message: error } });
 
               this.fleetStatus$.next({
                 level: ServiceStatusLevels.available,
@@ -586,7 +588,9 @@ export class FleetPlugin
           summary: 'Fleet is available',
         });
       } catch (error) {
-        logger.warn('Fleet setup failed', { error: { message: error } });
+        logger.warn(`Fleet setup failed after ${setupAttempts} attempts`, {
+          error: { message: error },
+        });
 
         this.fleetStatus$.next({
           // As long as Fleet has a dependency on EPR, we can't reliably set Kibana status to `unavailable` here.

From a723bd0cd71ed99654b9b0f2e82beb4dfda18aa1 Mon Sep 17 00:00:00 2001
From: Cee Chen <549407+cee-chen@users.noreply.github.com>
Date: Mon, 20 Nov 2023 10:01:56 -0800
Subject: [PATCH 02/13] [Unified Search] Change phrase(s) comboboxes to async
 patterns (#171427)

## Summary

This PR does NOT address
https://github.com/elastic/kibana/issues/158876, but helps reduce the
confusion/cognitive dissonance of the async search results only matching
on words from the start of the string, vs. `EuiComboBox` showing search
results from anywhere in the string by default.

Please see https://eui.elastic.co/#/forms/combo-box#async for more
documentation on async searching.

### Before

![image](https://user-images.githubusercontent.com/924948/270275373-0d0df8c3-3a03-40e9-bc79-ea353147e419.gif)

### After


![image](https://user-images.githubusercontent.com/549407/280421058-a7476a75-bfbe-49c6-be81-7bd0b46e048c.gif)

### Checklist

- ~[ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios~ No tests found
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [x] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
---
 .../public/filter_bar/filter_editor/phrase_suggestor.tsx      | 1 +
 .../public/filter_bar/filter_editor/phrase_value_input.tsx    | 4 +++-
 .../public/filter_bar/filter_editor/phrases_values_input.tsx  | 4 +++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_suggestor.tsx b/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_suggestor.tsx
index f25ce0cb28cad..3f542be18d9f8 100644
--- a/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_suggestor.tsx
+++ b/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_suggestor.tsx
@@ -71,6 +71,7 @@ export class PhraseSuggestorUI<T extends PhraseSuggestorProps> extends React.Com
   }
 
   protected onSearchChange = (value: string | number | boolean) => {
+    this.setState({ isLoading: true });
     this.updateSuggestions(`${value}`);
   };
 
diff --git a/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_value_input.tsx b/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_value_input.tsx
index 62648ae50d26a..adb80df6cf543 100644
--- a/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_value_input.tsx
+++ b/src/plugins/unified_search/public/filter_bar/filter_editor/phrase_value_input.tsx
@@ -54,7 +54,7 @@ class PhraseValueInputUI extends PhraseSuggestorUI<PhraseValueInputProps> {
   }
 
   private renderWithSuggestions() {
-    const { suggestions } = this.state;
+    const { suggestions, isLoading } = this.state;
     const { value, intl, onChange, fullWidth } = this.props;
     // there are cases when the value is a number, this would cause an exception
     const valueAsStr = String(value);
@@ -62,6 +62,8 @@ class PhraseValueInputUI extends PhraseSuggestorUI<PhraseValueInputProps> {
     return (
       <div ref={this.comboBoxWrapperRef}>
         <StringComboBox
+          async
+          isLoading={isLoading}
           inputRef={(ref) => {
             this.inputRef = ref;
           }}
diff --git a/src/plugins/unified_search/public/filter_bar/filter_editor/phrases_values_input.tsx b/src/plugins/unified_search/public/filter_bar/filter_editor/phrases_values_input.tsx
index 513dad1445823..500b875f42667 100644
--- a/src/plugins/unified_search/public/filter_bar/filter_editor/phrases_values_input.tsx
+++ b/src/plugins/unified_search/public/filter_bar/filter_editor/phrases_values_input.tsx
@@ -31,13 +31,15 @@ class PhrasesValuesInputUI extends PhraseSuggestorUI<PhrasesValuesInputProps> {
   comboBoxWrapperRef = React.createRef<HTMLDivElement>();
 
   public render() {
-    const { suggestions } = this.state;
+    const { suggestions, isLoading } = this.state;
     const { values, intl, onChange, fullWidth, onParamsUpdate, compressed, disabled } = this.props;
     const options = values ? uniq([...values, ...suggestions]) : suggestions;
 
     return (
       <div ref={this.comboBoxWrapperRef}>
         <StringComboBox
+          async
+          isLoading={isLoading}
           fullWidth={fullWidth}
           compressed={compressed}
           placeholder={intl.formatMessage({

From 9be7fb0d87a5d6073dbea535371d1742b9732dd2 Mon Sep 17 00:00:00 2001
From: Nicolas Chaulet <nicolas.chaulet@elastic.co>
Date: Mon, 20 Nov 2023 13:23:39 -0500
Subject: [PATCH 03/13] [Fleet] Fix invalid kuery when updating integration
 (#171559)

---
 .../epm/screens/detail/settings/settings.tsx     |  6 ++++++
 .../screens/detail/settings/update_button.tsx    | 16 ++++------------
 .../public/hooks/use_request/agent_policy.ts     | 12 ++++++++++++
 x-pack/plugins/fleet/public/types/index.ts       |  1 +
 4 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx
index 9c2ca98479a85..adeb17a5efdcc 100644
--- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx
+++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx
@@ -141,6 +141,11 @@ export const SettingsPage: React.FC<Props> = memo(({ packageInfo, theme$ }: Prop
     [packagePoliciesData]
   );
 
+  const agentPolicyIds = useMemo(
+    () => packagePoliciesData?.items.map(({ policy_id: agentPolicyId }) => agentPolicyId) ?? [],
+    [packagePoliciesData]
+  );
+
   const { data: dryRunData } = useUpgradePackagePolicyDryRunQuery(
     packagePolicyIds ?? [],
     latestVersion,
@@ -329,6 +334,7 @@ export const SettingsPage: React.FC<Props> = memo(({ packageInfo, theme$ }: Prop
                       <UpdateButton
                         {...packageInfo}
                         version={latestVersion}
+                        agentPolicyIds={agentPolicyIds}
                         packagePolicyIds={packagePolicyIds}
                         dryRunData={dryRunData}
                         isUpgradingPackagePolicies={isUpgradingPackagePolicies}
diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/update_button.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/update_button.tsx
index 199d49ff85a4d..3371957c95ab0 100644
--- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/update_button.tsx
+++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/update_button.tsx
@@ -29,7 +29,6 @@ import type {
   PackagePolicy,
 } from '../../../../../types';
 import { InstallStatus } from '../../../../../types';
-import { AGENT_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../../../../constants';
 import {
   useInstallPackage,
   useGetPackageInstallStatus,
@@ -37,12 +36,13 @@ import {
   useAuthz,
   useLink,
   useUpgradePackagePoliciesMutation,
-  useGetAgentPoliciesQuery,
+  useBulkGetAgentPoliciesQuery,
 } from '../../../../../hooks';
 
 interface UpdateButtonProps extends Pick<PackageInfo, 'name' | 'title' | 'version'> {
   dryRunData?: UpgradePackagePolicyDryRunResponse | null;
   packagePolicyIds?: string[];
+  agentPolicyIds: string[];
   isUpgradingPackagePolicies?: boolean;
   setIsUpgradingPackagePolicies?: React.Dispatch<React.SetStateAction<boolean>>;
   theme$: Observable<CoreTheme>;
@@ -73,6 +73,7 @@ export const UpdateButton: React.FunctionComponent<UpdateButtonProps> = ({
   isUpgradingPackagePolicies = false,
   name,
   packagePolicyIds = [],
+  agentPolicyIds = [],
   setIsUpgradingPackagePolicies = () => {},
   title,
   version,
@@ -92,16 +93,7 @@ export const UpdateButton: React.FunctionComponent<UpdateButtonProps> = ({
   const [isUpdateModalVisible, setIsUpdateModalVisible] = useState<boolean>(false);
   const [upgradePackagePolicies, setUpgradePackagePolicies] = useState<boolean>(true);
 
-  const { data: agentPolicyData } = useGetAgentPoliciesQuery({
-    perPage: SO_SEARCH_LIMIT,
-    page: 1,
-    // Fetch all agent policies that include one of the eligible package policies
-    kuery: packagePolicyIds.length
-      ? `${AGENT_POLICY_SAVED_OBJECT_TYPE}.package_policies:${packagePolicyIds
-          .map((id) => `"${id}"`)
-          .join(' or ')}`
-      : '',
-  });
+  const { data: agentPolicyData } = useBulkGetAgentPoliciesQuery(agentPolicyIds, { full: true });
 
   const packagePolicyCount = useMemo(() => packagePolicyIds.length, [packagePolicyIds]);
 
diff --git a/x-pack/plugins/fleet/public/hooks/use_request/agent_policy.ts b/x-pack/plugins/fleet/public/hooks/use_request/agent_policy.ts
index ee0481bd373af..6ce49febdeca2 100644
--- a/x-pack/plugins/fleet/public/hooks/use_request/agent_policy.ts
+++ b/x-pack/plugins/fleet/public/hooks/use_request/agent_policy.ts
@@ -22,6 +22,7 @@ import type {
   CopyAgentPolicyResponse,
   DeleteAgentPolicyRequest,
   DeleteAgentPolicyResponse,
+  BulkGetAgentPoliciesResponse,
 } from '../../types';
 
 import { useRequest, sendRequest, useConditionalRequest, sendRequestForRq } from './use_request';
@@ -47,6 +48,17 @@ export const useGetAgentPoliciesQuery = (query?: GetAgentPoliciesRequest['query'
   );
 };
 
+export const useBulkGetAgentPoliciesQuery = (ids: string[], options?: { full?: boolean }) => {
+  return useQuery<BulkGetAgentPoliciesResponse, RequestError>(['agentPolicies', ids], () =>
+    sendRequestForRq<BulkGetAgentPoliciesResponse>({
+      path: agentPolicyRouteService.getBulkGetPath(),
+      method: 'post',
+      body: JSON.stringify({ ids, full: options?.full }),
+      version: API_VERSIONS.public.v1,
+    })
+  );
+};
+
 export const sendGetAgentPolicies = (query?: GetAgentPoliciesRequest['query']) => {
   return sendRequest<GetAgentPoliciesResponse>({
     path: agentPolicyRouteService.getListPath(),
diff --git a/x-pack/plugins/fleet/public/types/index.ts b/x-pack/plugins/fleet/public/types/index.ts
index 8c7d3ddcf5b45..63837dc809559 100644
--- a/x-pack/plugins/fleet/public/types/index.ts
+++ b/x-pack/plugins/fleet/public/types/index.ts
@@ -137,6 +137,7 @@ export type {
   KibanaSavedObjectType,
   GetInputsTemplatesRequest,
   GetInputsTemplatesResponse,
+  BulkGetAgentPoliciesResponse,
 } from '../../common/types';
 export {
   entries,

From 936a1a40ac852920eb8353ca4b8d2ff1009b1bcf Mon Sep 17 00:00:00 2001
From: Gloria Hornero <gloria.hornero@elastic.co>
Date: Mon, 20 Nov 2023 19:39:57 +0100
Subject: [PATCH 04/13] unskipping entity analytics tests (#170712)

---
 .../cypress/e2e/explore/dashboards/entity_analytics.cy.ts      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts
index 7bb492d51d25c..7210bacd1aa77 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/dashboards/entity_analytics.cy.ts
@@ -169,8 +169,7 @@ describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () =>
         cy.get(HOSTS_TABLE_ALERT_CELL).should('have.length', 5);
       });
 
-      // FLAKY: https://github.com/elastic/kibana/issues/168490
-      it.skip('filters by risk level', () => {
+      it('filters by risk level', () => {
         openRiskTableFilterAndSelectTheLowOption();
 
         cy.get(HOSTS_DONUT_CHART).should('include.text', '1Total');

From 8fd18fbb05ad988cb30cba2a2f949b911e80a897 Mon Sep 17 00:00:00 2001
From: Maryam Saeidi <maryam.saeidi@elastic.co>
Date: Mon, 20 Nov 2023 20:21:35 +0100
Subject: [PATCH 05/13] [Custom threshold] Add an adHoc data view API
 integration test (#171508)

Closes #170438

## Summary

This PR adds an ad hoc data view API integration test by changing one of
the existing custom threshold API integration tests.
---
 .../custom_threshold_rule/avg_pct_fired.ts    | 31 ++++++++++---------
 .../helpers/alerting_api_helper.ts            |  6 +++-
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
index b38b4b0ee672c..e3ed2144876ad 100644
--- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
+++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
@@ -16,7 +16,6 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/cus
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
 import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
-import { createDataView, deleteDataView } from '../helpers/data_view';
 import {
   waitForAlertInIndex,
   waitForDocumentInIndex,
@@ -38,8 +37,19 @@ export default function ({ getService }: FtrProviderContext) {
     // DATE_VIEW should match the index template:
     // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json
     const DATE_VIEW = 'kbn-data-forge-fake_hosts';
-    const DATE_VIEW_NAME = 'data-view-name';
+    const DATE_VIEW_NAME = 'ad-hoc-data-view-name';
     const DATA_VIEW_ID = 'data-view-id';
+    const MOCKED_AD_HOC_DATA_VIEW = {
+      id: DATA_VIEW_ID,
+      title: DATE_VIEW,
+      timeFieldName: '@timestamp',
+      sourceFilters: [],
+      fieldFormats: {},
+      runtimeFieldMap: {},
+      allowNoIndex: false,
+      name: DATE_VIEW_NAME,
+      allowHidden: false,
+    };
     let infraDataIndex: string;
     let actionId: string;
     let ruleId: string;
@@ -48,12 +58,6 @@ export default function ({ getService }: FtrProviderContext) {
 
     before(async () => {
       infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
-      await createDataView({
-        supertest,
-        name: DATE_VIEW_NAME,
-        id: DATA_VIEW_ID,
-        title: DATE_VIEW,
-      });
     });
 
     after(async () => {
@@ -67,10 +71,6 @@ export default function ({ getService }: FtrProviderContext) {
         index: '.kibana-event-log-*',
         query: { term: { 'kibana.alert.rule.consumer': 'logs' } },
       });
-      await deleteDataView({
-        supertest,
-        id: DATA_VIEW_ID,
-      });
       await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
       await cleanup({ esClient, logger });
     });
@@ -109,7 +109,7 @@ export default function ({ getService }: FtrProviderContext) {
                 query: '',
                 language: 'kuery',
               },
-              index: DATA_VIEW_ID,
+              index: MOCKED_AD_HOC_DATA_VIEW,
             },
           },
           actions: [
@@ -199,7 +199,10 @@ export default function ({ getService }: FtrProviderContext) {
             ],
             alertOnNoData: true,
             alertOnGroupDisappear: true,
-            searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
+            searchConfiguration: {
+              index: MOCKED_AD_HOC_DATA_VIEW,
+              query: { query: '', language: 'kuery' },
+            },
           });
       });
 
diff --git a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
index d56b91dda5515..57b5721701a40 100644
--- a/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
+++ b/x-pack/test/alerting_api_integration/observability/helpers/alerting_api_helper.ts
@@ -5,8 +5,9 @@
  * 2.0.
  */
 
-import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types';
 import type { SuperTest, Test } from 'supertest';
+import expect from '@kbn/expect';
+import { ThresholdParams } from '@kbn/observability-plugin/common/custom_threshold_rule/types';
 
 export async function createIndexConnector({
   supertest,
@@ -64,5 +65,8 @@ export async function createRule<Params = ThresholdParams>({
       rule_type_id: ruleTypeId,
       actions,
     });
+  if (body.statusCode) {
+    expect(body.statusCode).eql(200, body.message);
+  }
   return body;
 }

From 5d8c0508267ada42175299f07b8777e58feb4de5 Mon Sep 17 00:00:00 2001
From: Rickyanto Ang <rickyangwyn@gmail.com>
Date: Mon, 20 Nov 2023 11:22:21 -0800
Subject: [PATCH 06/13] [Cloud Security][Bug Fix] Dashboard and Findings page
 now will only show result within retention period (#160313)

## Summary

Currently Findings and Dashboard page kept showing data even if the data
past its retention period. This PR will make it so that, it will only
show data within that retention period, If theres no more data past that
retention period it will show No Agent deployed prompt instead

Our solution/workaround for this will be adding a retention periods for
each kind of posture type:
**KSPM** and **CSPM** : 26 hours
**CNVM** and **all** : 3 days or 72 hours
if posture type doesnt exist : 72 hours (this is to cover the situation
where findings came before CSPM gets implemented back then)

Before when having Findings outside retention period:
<img width="1391" alt="Screenshot 2023-11-07 at 5 03 15 PM"
src="https://github.com/elastic/kibana/assets/8703149/26f74ebf-8002-4da9-b4f1-8238e81cea5e">

After: (27 hours later)
<img width="1452" alt="Screenshot 2023-11-07 at 4 52 09 PM"
src="https://github.com/elastic/kibana/assets/8703149/98a33379-dc21-4edf-97b6-32de584598e9">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
---
 .../server/lib/check_index_status.ts          | 43 ++++++---
 .../server/routes/status/status.ts            | 65 ++++++++++---
 .../apis/cloud_security_posture/mock_data.ts  |  8 ++
 .../status/status_unprivileged.ts             |  4 +-
 .../mocks/vulnerabilities_latest_mock.ts      |  4 +-
 .../page_objects/findings_page.ts             |  7 ++
 .../pages/findings.ts                         | 10 +-
 .../pages/findings_old_data.ts                | 94 +++++++++++++++++++
 .../pages/index.ts                            |  1 +
 9 files changed, 201 insertions(+), 35 deletions(-)
 create mode 100644 x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts

diff --git a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts
index 95dca041521e0..ebce74d89b2dc 100644
--- a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts
+++ b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts
@@ -9,25 +9,45 @@ import { ElasticsearchClient, type Logger } from '@kbn/core/server';
 import { getSafePostureTypeRuntimeMapping } from '../../common/runtime_mappings/get_safe_posture_type_runtime_mapping';
 import { IndexStatus, PostureTypes } from '../../common/types';
 
+export interface PostureTypeAndRetention {
+  postureType?: PostureTypes;
+  retentionTime?: string;
+}
+
 export const checkIndexStatus = async (
   esClient: ElasticsearchClient,
   index: string,
   logger: Logger,
-  postureType?: PostureTypes
+  PostureTypeAndRetention?: PostureTypeAndRetention
 ): Promise<IndexStatus> => {
-  const query =
-    !postureType || postureType === 'all' || postureType === 'vuln_mgmt'
-      ? undefined
-      : {
-          bool: {
-            filter: {
-              term: {
-                safe_posture_type: postureType,
+  const isNotKspmOrCspm =
+    !PostureTypeAndRetention?.postureType ||
+    PostureTypeAndRetention?.postureType === 'all' ||
+    PostureTypeAndRetention?.postureType === 'vuln_mgmt';
+
+  const query = {
+    bool: {
+      filter: [
+        ...(isNotKspmOrCspm
+          ? []
+          : [
+              {
+                term: {
+                  safe_posture_type: PostureTypeAndRetention?.postureType,
+                },
               },
+            ]),
+        {
+          range: {
+            '@timestamp': {
+              gte: `now-${PostureTypeAndRetention?.retentionTime}`,
+              lte: 'now',
             },
           },
-        };
-
+        },
+      ],
+    },
+  };
   try {
     const queryResult = await esClient.search({
       index,
@@ -37,7 +57,6 @@ export const checkIndexStatus = async (
       query,
       size: 1,
     });
-
     return queryResult.hits.hits.length ? 'not-empty' : 'empty';
   } catch (e) {
     logger.debug(e);
diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
index 86b0d0a66802b..ed3bcd99746fc 100644
--- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
+++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
@@ -29,6 +29,9 @@ import {
   POSTURE_TYPES,
   LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
   VULN_MGMT_POLICY_TEMPLATE,
+  POSTURE_TYPE_ALL,
+  LATEST_VULNERABILITIES_RETENTION_POLICY,
+  LATEST_FINDINGS_RETENTION_POLICY,
 } from '../../../common/constants';
 import type {
   CspApiRequestHandlerContext,
@@ -168,20 +171,53 @@ export const getCspStatus = async ({
     installedPackagePoliciesVulnMgmt,
     installedPolicyTemplates,
   ] = await Promise.all([
-    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger),
-    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger),
-    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger),
-
-    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, 'cspm'),
-    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, 'cspm'),
-    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'cspm'),
-
-    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, 'kspm'),
-    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, 'kspm'),
-    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'kspm'),
-
-    checkIndexStatus(esClient, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, logger),
-    checkIndexStatus(esClient, VULNERABILITIES_INDEX_PATTERN, logger, VULN_MGMT_POLICY_TEMPLATE),
+    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
+      postureType: POSTURE_TYPE_ALL,
+      retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
+      postureType: POSTURE_TYPE_ALL,
+      retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
+      postureType: POSTURE_TYPE_ALL,
+      retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
+    }),
+
+    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
+      postureType: CSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
+      postureType: CSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
+      postureType: CSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+
+    checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
+      postureType: KSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
+      postureType: KSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
+      postureType: KSPM_POLICY_TEMPLATE,
+      retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
+    }),
+
+    checkIndexStatus(esClient, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, logger, {
+      postureType: VULN_MGMT_POLICY_TEMPLATE,
+      retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
+    }),
+    checkIndexStatus(esClient, VULNERABILITIES_INDEX_PATTERN, logger, {
+      postureType: VULN_MGMT_POLICY_TEMPLATE,
+      retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
+    }),
 
     packageService.asInternalUser.getInstallation(CLOUD_SECURITY_POSTURE_PACKAGE_NAME),
     packageService.asInternalUser.fetchFindLatestPackage(CLOUD_SECURITY_POSTURE_PACKAGE_NAME),
@@ -295,6 +331,7 @@ export const getCspStatus = async ({
     {
       latest: vulnerabilitiesLatestIndexStatus,
       stream: vulnerabilitiesIndexStatus,
+      score: scoreIndexStatus,
     },
     installation,
     healthyAgentsVulMgmt,
diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts b/x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts
index 4ebfacb4d10c9..b4daf5172b164 100644
--- a/x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts
+++ b/x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts
@@ -24,6 +24,10 @@ export const findingsMockData = [
       type: 'process',
     },
     cluster_id: 'Upper case cluster id',
+    event: {
+      ingested: '2023-08-19T18:20:41Z',
+      created: '2023-08-19T18:17:15.609124281Z',
+    },
   },
   {
     resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' },
@@ -40,6 +44,10 @@ export const findingsMockData = [
       type: 'process',
     },
     cluster_id: 'Another Upper case cluster id',
+    event: {
+      ingested: '2023-08-19T18:20:41Z',
+      created: '2023-08-19T18:17:15.609124281Z',
+    },
   },
 ];
 
diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts
index 6a3f8fbbfeda5..06f80adf46034 100644
--- a/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts
+++ b/x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts
@@ -247,8 +247,8 @@ export default function (providerContext: FtrProviderContext) {
           `expected unprivileged but got ${res.cspm.status} instead`
         );
         expect(res.vuln_mgmt.status).to.eql(
-          'not-installed',
-          `expected not-installed but got ${res.vuln_mgmt.status} instead`
+          'unprivileged',
+          `expected unprivileged but got ${res.vuln_mgmt.status} instead`
         );
 
         assertIndexStatus(res.indicesDetails, LATEST_FINDINGS_INDEX_DEFAULT_NS, 'unprivileged');
diff --git a/x-pack/test/cloud_security_posture_functional/mocks/vulnerabilities_latest_mock.ts b/x-pack/test/cloud_security_posture_functional/mocks/vulnerabilities_latest_mock.ts
index 813c72743e7fc..f0ffd4aebb388 100644
--- a/x-pack/test/cloud_security_posture_functional/mocks/vulnerabilities_latest_mock.ts
+++ b/x-pack/test/cloud_security_posture_functional/mocks/vulnerabilities_latest_mock.ts
@@ -83,7 +83,7 @@ export const vulnerabilitiesLatestMock = [
         id: '704479110758',
       },
     },
-    '@timestamp': '2023-06-29T02:08:44.993Z',
+    '@timestamp': (Date.now() - 249200000).toString(),
     cloudbeat: {
       commit_sha: '4d990caa0c9c1594441da6bf24a685599aeb2bd5',
       commit_time: '2023-05-15T14:48:10Z',
@@ -189,7 +189,7 @@ export const vulnerabilitiesLatestMock = [
         id: '704479110758',
       },
     },
-    '@timestamp': '2023-06-29T02:08:16.535Z',
+    '@timestamp': (Date.now() - 249200000).toString(),
     ecs: {
       version: '8.6.0',
     },
diff --git a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
index 49f4ab0a6d12f..0fd0e463e7087 100644
--- a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
+++ b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
@@ -410,6 +410,12 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
     },
   });
 
+  const isLatestFindingsTableThere = async () => {
+    const table = await testSubjects.findAll('docTable');
+    const trueOrFalse = table.length > 0 ? true : false;
+    return trueOrFalse;
+  };
+
   return {
     navigateToLatestFindingsPage,
     navigateToVulnerabilities,
@@ -426,5 +432,6 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
     misconfigurationsFlyout,
     toastMessage,
     detectionRuleApi,
+    isLatestFindingsTableThere,
   };
 }
diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings.ts b/x-pack/test/cloud_security_posture_functional/pages/findings.ts
index 69c1fd6949f51..4ec0240f735c9 100644
--- a/x-pack/test/cloud_security_posture_functional/pages/findings.ts
+++ b/x-pack/test/cloud_security_posture_functional/pages/findings.ts
@@ -17,12 +17,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const retry = getService('retry');
   const pageObjects = getPageObjects(['common', 'findings', 'header']);
   const chance = new Chance();
+  const timeFiveHoursAgo = (Date.now() - 18000000).toString();
 
   // We need to use a dataset for the tests to run
   // We intentionally make some fields start with a capital letter to test that the query bar is case-insensitive/case-sensitive
   const data = [
     {
-      '@timestamp': '1695819664234',
+      '@timestamp': timeFiveHoursAgo,
       resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
       result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
       rule: {
@@ -39,7 +40,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       cluster_id: 'Upper case cluster id',
     },
     {
-      '@timestamp': '1695819673242',
+      '@timestamp': timeFiveHoursAgo,
       resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' },
       result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
       rule: {
@@ -56,7 +57,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       cluster_id: 'Another Upper case cluster id',
     },
     {
-      '@timestamp': '1695819676242',
+      '@timestamp': timeFiveHoursAgo,
       resource: { id: chance.guid(), name: `process`, sub_type: 'another lower case type' },
       result: { evaluation: 'passed' },
       rule: {
@@ -73,7 +74,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       cluster_id: 'lower case cluster id',
     },
     {
-      '@timestamp': '1695819680202',
+      '@timestamp': timeFiveHoursAgo,
       resource: { id: chance.guid(), name: `process`, sub_type: 'Upper case type again' },
       result: { evaluation: 'failed' },
       rule: {
@@ -122,7 +123,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       await findings.index.add(data);
 
       await findings.navigateToLatestFindingsPage();
-
       await retry.waitFor(
         'Findings table to be loaded',
         async () => (await latestFindingsTable.getRowsCount()) === data.length
diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts
new file mode 100644
index 0000000000000..fc12d593333b2
--- /dev/null
+++ b/x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import Chance from 'chance';
+import type { FtrProviderContext } from '../ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const pageObjects = getPageObjects(['common', 'findings', 'header']);
+  const chance = new Chance();
+  const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000;
+
+  const dataOldKspm = [
+    {
+      '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
+      resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        name: 'Upper case rule name',
+        section: 'Upper case section',
+        benchmark: {
+          id: 'cis_k8s',
+          posture_type: 'kspm',
+          name: 'CIS Kubernetes V1.23',
+          version: 'v1.0.0',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Upper case cluster id',
+    },
+  ];
+
+  const dataOldCspm = [
+    {
+      '@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
+      resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
+      result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
+      rule: {
+        name: 'Upper case rule name',
+        section: 'Upper case section',
+        benchmark: {
+          id: 'cis_aws',
+          posture_type: 'cspm',
+          name: 'CIS AWS V1.23',
+          version: 'v1.0.0',
+        },
+        type: 'process',
+      },
+      cluster_id: 'Upper case cluster id',
+    },
+  ];
+
+  describe('Old Data', function () {
+    this.tags(['cloud_security_posture_findings']);
+    let findings: typeof pageObjects.findings;
+
+    before(async () => {
+      findings = pageObjects.findings;
+
+      // Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
+      await findings.waitForPluginInitialized();
+    });
+
+    after(async () => {
+      await findings.index.remove();
+    });
+
+    describe('Findings page with old data', () => {
+      it('returns no Findings KSPM', async () => {
+        // Prepare mocked findings
+        await findings.index.remove();
+        await findings.index.add(dataOldKspm);
+
+        await findings.navigateToLatestFindingsPage();
+        pageObjects.header.waitUntilLoadingHasFinished();
+        expect(await findings.isLatestFindingsTableThere()).to.be(false);
+      });
+      it('returns no Findings CSPM', async () => {
+        // Prepare mocked findings
+        await findings.index.remove();
+        await findings.index.add(dataOldCspm);
+
+        await findings.navigateToLatestFindingsPage();
+        pageObjects.header.waitUntilLoadingHasFinished();
+        expect(await findings.isLatestFindingsTableThere()).to.be(false);
+      });
+    });
+  });
+}
diff --git a/x-pack/test/cloud_security_posture_functional/pages/index.ts b/x-pack/test/cloud_security_posture_functional/pages/index.ts
index 9d4e17ec0c88c..f1bb7f8fb7875 100644
--- a/x-pack/test/cloud_security_posture_functional/pages/index.ts
+++ b/x-pack/test/cloud_security_posture_functional/pages/index.ts
@@ -16,5 +16,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./compliance_dashboard'));
     loadTestFile(require.resolve('./vulnerability_dashboard'));
     loadTestFile(require.resolve('./cis_integration'));
+    loadTestFile(require.resolve('./findings_old_data'));
   });
 }

From e15b5e69af467d1f285ada84cabe91c9c3081ca6 Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:38:52 +0000
Subject: [PATCH 07/13] skip flaky suite (#171489)

---
 .../apps/integrations/artifact_entries_list.ts                 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
index 67c21417beed8..6694274ad476e 100644
--- a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
+++ b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
@@ -246,7 +246,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
     };
 
     for (const testData of getArtifactsListTestsData()) {
-      describe(`When on the ${testData.title} entries list`, function () {
+      // FLAKY: https://github.com/elastic/kibana/issues/171489
+      describe.skip(`When on the ${testData.title} entries list`, function () {
         beforeEach(async () => {
           policyInfo = await policyTestResources.createPolicy();
           await removeAllArtifacts();

From f045d9d1302e426af60ef7d46a8c9f6d792cb6ec Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:39:57 +0000
Subject: [PATCH 08/13] skip flaky suite (#171476)

---
 .../apps/integrations/artifact_entries_list.ts                 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
index 6694274ad476e..ff0e58b779852 100644
--- a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
+++ b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
@@ -333,7 +333,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
       });
     }
 
-    describe('Should check artifacts are correctly generated when multiple entries', function () {
+    // FLAKY: https://github.com/elastic/kibana/issues/171476
+    describe.skip('Should check artifacts are correctly generated when multiple entries', function () {
       let firstPolicy: PolicyTestResourceInfo;
       let secondPolicy: PolicyTestResourceInfo;
 

From edf89560535437fe475567fe9f16bcad36c3816d Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:41:11 +0000
Subject: [PATCH 09/13] skip flaky suite (#171481)

---
 .../apps/integrations/trusted_apps_list.ts                     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/security_solution_endpoint/apps/integrations/trusted_apps_list.ts b/x-pack/test/security_solution_endpoint/apps/integrations/trusted_apps_list.ts
index fd925a975ba14..1d48a415b1577 100644
--- a/x-pack/test/security_solution_endpoint/apps/integrations/trusted_apps_list.ts
+++ b/x-pack/test/security_solution_endpoint/apps/integrations/trusted_apps_list.ts
@@ -16,7 +16,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const browser = getService('browser');
   const endpointTestResources = getService('endpointTestResources');
 
-  describe('When on the Trusted Apps list', function () {
+  // FLAKY: https://github.com/elastic/kibana/issues/171481
+  describe.skip('When on the Trusted Apps list', function () {
     targetTags(this, ['@ess', '@serverless']);
 
     let indexedData: IndexedHostsAndAlertsResponse;

From 0e565e197c96778d8b4f9d8b69688c39b712cea7 Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:42:17 +0000
Subject: [PATCH 10/13] skip flaky suite (#171475)

---
 .../apps/integrations/artifact_entries_list.ts                   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
index ff0e58b779852..45dbff31170f8 100644
--- a/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
+++ b/x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts
@@ -247,6 +247,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
 
     for (const testData of getArtifactsListTestsData()) {
       // FLAKY: https://github.com/elastic/kibana/issues/171489
+      // FLAKY: https://github.com/elastic/kibana/issues/171475
       describe.skip(`When on the ${testData.title} entries list`, function () {
         beforeEach(async () => {
           policyInfo = await policyTestResources.createPolicy();

From b54d00debb9f29874c948bd25683a5b4437d8c5a Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:43:28 +0000
Subject: [PATCH 11/13] skip flaky suite (#171493)

---
 .../test/functional/apps/infra/logs/ml_job_id_formats/tests.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts
index 4e0e12a386dc9..ec16fe567f4ea 100644
--- a/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts
+++ b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts
@@ -205,7 +205,8 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => {
       });
     });
 
-    describe('creation and recreation', () => {
+    // FLAKY: https://github.com/elastic/kibana/issues/171493
+    describe.skip('creation and recreation', () => {
       it('create first ML job', async () => {
         await logsUi.logEntryRatePage.navigateTo();
         await requestTracker.install();

From 0a09215f0c1521e7b879c10df6f56b25da5a21a1 Mon Sep 17 00:00:00 2001
From: Tiago Costa <tiago.costa@elastic.co>
Date: Mon, 20 Nov 2023 19:46:04 +0000
Subject: [PATCH 12/13] skip flaky suite (#169866)

---
 .../cypress/e2e/investigations/timelines/creation.cy.ts        | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts
index 1b69b314daf75..546cdaa6b64c1 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/creation.cy.ts
@@ -75,7 +75,8 @@ describe('Timelines', (): void => {
     deleteTimelines();
   });
 
-  describe('Toggle create timeline from plus icon', () => {
+  // FLAKY: https://github.com/elastic/kibana/issues/169866
+  describe.skip('Toggle create timeline from plus icon', () => {
     context('Privileges: CRUD', { tags: '@ess' }, () => {
       beforeEach(() => {
         login();

From 029b3ba81d340ca0d8394397aeee127841f190c5 Mon Sep 17 00:00:00 2001
From: Thomas Watson <watson@elastic.co>
Date: Mon, 20 Nov 2023 21:47:48 +0100
Subject: [PATCH 13/13] Upgrade to Node.js 20 (#162696)

Co-authored-by: Jonathan Budzenski <jon@elastic.co>
---
 .buildkite/pipelines/artifacts.yml            |  2 +-
 .ci/Dockerfile                                |  2 +-
 .node-version                                 |  2 +-
 .nvmrc                                        |  2 +-
 WORKSPACE.bazel                               | 12 ++---
 package.json                                  |  6 +--
 .../discovery/plugin_manifest_parser.test.ts  |  2 +-
 .../src/discovery/plugins_discovery.test.ts   |  2 +-
 .../src/lib/archives/parse.test.ts            |  2 +-
 .../integration_tests/__fixtures__/es_bin.js  |  4 +-
 packages/kbn-es/src/utils/docker.test.ts      | 15 +++---
 packages/kbn-es/src/utils/docker.ts           | 11 ++++-
 .../build/tasks/patch_native_modules_task.ts  | 20 ++++----
 .../create_streaming_batched_function.test.ts |  2 +-
 .../public/search/expressions/esdsl.test.ts   |  2 +-
 src/setup_node_env/exit_on_warning.js         |  7 +++
 src/setup_node_env/heap_snapshot.js           | 46 -------------------
 src/setup_node_env/setup_env.js               |  2 -
 .../user_actions/connector_id.test.ts         |  6 +--
 .../document_creation_logic.test.ts           |  2 +-
 .../server/browsers/download/fetch.test.ts    |  4 +-
 .../server/browsers/download/fetch.ts         | 12 ++---
 .../server/browsers/download/index.test.ts    | 10 ++--
 .../server/browsers/download/index.ts         | 10 +++-
 .../server/browsers/extract/unzip.test.ts     |  4 +-
 .../lib/check_for_json_errors.test.ts         |  2 +-
 .../api/rules/import_rules/route.test.ts      |  2 +-
 .../create_rules_stream_from_ndjson.test.ts   |  4 +-
 .../server/saved_objects/migrations.test.ts   |  2 +-
 yarn.lock                                     | 15 ++++--
 30 files changed, 99 insertions(+), 115 deletions(-)
 delete mode 100644 src/setup_node_env/heap_snapshot.js

diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
index 4a52ee7402823..5b219952c8cf1 100644
--- a/.buildkite/pipelines/artifacts.yml
+++ b/.buildkite/pipelines/artifacts.yml
@@ -88,7 +88,7 @@ steps:
       - exit_status: -1
     agents:
       queue: n2-2
-    timeout_in_minutes: 30
+    timeout_in_minutes: 60
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
       automatic:
diff --git a/.ci/Dockerfile b/.ci/Dockerfile
index 109b9ffab3cc5..91e0d5bf36a9d 100644
--- a/.ci/Dockerfile
+++ b/.ci/Dockerfile
@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=18.18.2
+ARG NODE_VERSION=20.9.0
 
 FROM node:${NODE_VERSION} AS base
 
diff --git a/.node-version b/.node-version
index 87ec8842b158d..f3f52b42d3da9 100644
--- a/.node-version
+++ b/.node-version
@@ -1 +1 @@
-18.18.2
+20.9.0
diff --git a/.nvmrc b/.nvmrc
index 87ec8842b158d..f3f52b42d3da9 100644
--- a/.nvmrc
+++ b/.nvmrc
@@ -1 +1 @@
-18.18.2
+20.9.0
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
index e614bdff172f9..38802ebef5b1d 100644
--- a/WORKSPACE.bazel
+++ b/WORKSPACE.bazel
@@ -22,13 +22,13 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install
 # Setup the Node.js toolchain for the architectures we want to support
 node_repositories(
   node_repositories = {
-    "18.18.2-darwin_amd64": ("node-v18.18.2-darwin-x64.tar.gz", "node-v18.18.2-darwin-x64", "5bb8da908ed590e256a69bf2862238c8a67bc4600119f2f7721ca18a7c810c0f"),
-    "18.18.2-darwin_arm64": ("node-v18.18.2-darwin-arm64.tar.gz", "node-v18.18.2-darwin-arm64", "9f982cc91b28778dd8638e4f94563b0c2a1da7aba62beb72bd427721035ab553"),
-    "18.18.2-linux_arm64": ("node-v18.18.2-linux-arm64.tar.xz", "node-v18.18.2-linux-arm64", "8a5a03f6a742159c9aa0ae3a99b368cd938cf62f3a5522a2e5acbe6313710efe"),
-    "18.18.2-linux_amd64": ("node-v18.18.2-linux-x64.tar.xz", "node-v18.18.2-linux-x64", "f7cf590bc7153f3beaa9e1138d00e50d74df223f0bec61f63e7df65f7315b76a"),
-    "18.18.2-windows_amd64": ("node-v18.18.2-win-x64.zip", "node-v18.18.2-win-x64", "3bb0e51e579a41a22b3bf6cb2f3e79c03801aa17acbe0ca00fc555d1282e7acd"),
+    "20.9.0-darwin_amd64": ("node-v20.9.0-darwin-x64.tar.gz", "node-v20.9.0-darwin-x64", "fc5b73f2a78c17bbe926cdb1447d652f9f094c79582f1be6471b4b38a2e1ccc8"),
+    "20.9.0-darwin_arm64": ("node-v20.9.0-darwin-arm64.tar.gz", "node-v20.9.0-darwin-arm64", "31d2d46ae8d8a3982f54e2ff1e60c2e4a8e80bf78a3e8b46dcaac95ac5d7ce6a"),
+    "20.9.0-linux_arm64": ("node-v20.9.0-linux-arm64.tar.xz", "node-v20.9.0-linux-arm64", "79c07c41c9f2410e35fd8dec61491ba63762e428bffa2ee0ff3aec1afe05d4b1"),
+    "20.9.0-linux_amd64": ("node-v20.9.0-linux-x64.tar.xz", "node-v20.9.0-linux-x64", "d11a5e06d6fda8d0cb1a759365d2b5e33c609f3c9f333fdc63e0522475dc0c89"),
+    "20.9.0-windows_amd64": ("node-v20.9.0-win-x64.zip", "node-v20.9.0-win-x64", "70d87dad2378c63216ff83d5a754c61d2886fc39d32ce0d2ea6de763a22d3780"),
   },
-  node_version = "18.18.2",
+  node_version = "20.9.0",
   node_urls = [
     "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v{version}/{filename}",
   ],
diff --git a/package.json b/package.json
index 0f8260566b536..73aff64a7f932 100644
--- a/package.json
+++ b/package.json
@@ -74,12 +74,12 @@
     "url": "https://github.com/elastic/kibana.git"
   },
   "engines": {
-    "node": "18.18.2",
+    "node": "20.9.0",
     "yarn": "^1.22.19"
   },
   "resolutions": {
     "**/@hello-pangea/dnd": "16.2.0",
-    "**/@types/node": "18.18.5",
+    "**/@types/node": "20.9.0",
     "**/@typescript-eslint/utils": "5.62.0",
     "**/chokidar": "^3.5.3",
     "**/globule/minimatch": "^3.1.2",
@@ -1379,7 +1379,7 @@
     "@types/multistream": "^4.1.0",
     "@types/mustache": "^0.8.31",
     "@types/nock": "^10.0.3",
-    "@types/node": "18.18.5",
+    "@types/node": "20.9.0",
     "@types/node-fetch": "2.6.4",
     "@types/node-forge": "^1.3.1",
     "@types/nodemailer": "^6.4.0",
diff --git a/packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_parser.test.ts b/packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_parser.test.ts
index f940e49805cd5..202cef2ca09d4 100644
--- a/packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_parser.test.ts
+++ b/packages/core/plugins/core-plugins-server-internal/src/discovery/plugin_manifest_parser.test.ts
@@ -59,7 +59,7 @@ test('return error when manifest content is not a valid JSON', async () => {
   });
 
   await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
-    message: `Unexpected token o in JSON at position 1 (invalid-manifest, ${pluginManifestPath})`,
+    message: `Unexpected token 'o', "not-json" is not valid JSON (invalid-manifest, ${pluginManifestPath})`,
     type: PluginDiscoveryErrorType.InvalidManifest,
     path: pluginManifestPath,
   });
diff --git a/packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.test.ts b/packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.test.ts
index 9ab50d8786cca..93657a1f2533e 100644
--- a/packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.test.ts
+++ b/packages/core/plugins/core-plugins-server-internal/src/discovery/plugins_discovery.test.ts
@@ -278,7 +278,7 @@ describe('plugins discovery system', () => {
       .toPromise();
 
     expect(errors).toContain(
-      `Error: Unexpected token o in JSON at position 1 (invalid-manifest, ${manifestPath(
+      `Error: Unexpected token 'o', "not-json" is not valid JSON (invalid-manifest, ${manifestPath(
         'plugin_a'
       )})`
     );
diff --git a/packages/kbn-es-archiver/src/lib/archives/parse.test.ts b/packages/kbn-es-archiver/src/lib/archives/parse.test.ts
index b87e6f71400e0..9e0d5c02238b8 100644
--- a/packages/kbn-es-archiver/src/lib/archives/parse.test.ts
+++ b/packages/kbn-es-archiver/src/lib/archives/parse.test.ts
@@ -98,7 +98,7 @@ describe('esArchiver createParseArchiveStreams', () => {
           ] as [Readable, ...Writable[]]);
           throw new Error('should have failed');
         } catch (err) {
-          expect(err.message).toEqual(expect.stringContaining('Unexpected number'));
+          expect(err.message).toEqual(`Expected property name or '}' in JSON at position 1`);
         }
       });
     });
diff --git a/packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js b/packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js
index 5ef6145698811..0d44b8a775b25 100644
--- a/packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js
+++ b/packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js
@@ -87,11 +87,11 @@ const { ES_KEY_PATH, ES_CERT_PATH } = require('@kbn/dev-utils');
     }
   );
 
-  // setup server auto close after 1 second of silence
+  // setup server auto close after 5 second of silence
   let serverCloseTimer;
   const delayServerClose = () => {
     clearTimeout(serverCloseTimer);
-    serverCloseTimer = setTimeout(() => server.close(), 1000);
+    serverCloseTimer = setTimeout(() => server.close(), 5000);
   };
   server.on('request', delayServerClose);
   server.on('listening', delayServerClose);
diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts
index 2d71a4e628e11..2def181febb58 100644
--- a/packages/kbn-es/src/utils/docker.test.ts
+++ b/packages/kbn-es/src/utils/docker.test.ts
@@ -7,8 +7,7 @@
  */
 import mockFs from 'mock-fs';
 
-import { existsSync } from 'fs';
-import { stat } from 'fs/promises';
+import Fsp from 'fs/promises';
 import { basename } from 'path';
 
 import {
@@ -109,7 +108,7 @@ const volumeCmdTest = async (volumeCmd: string[]) => {
 
   // extract only permission from mode
   // eslint-disable-next-line no-bitwise
-  expect((await stat(serverlessObjectStorePath)).mode & 0o777).toBe(0o777);
+  expect((await Fsp.stat(serverlessObjectStorePath)).mode & 0o777).toBe(0o777);
 };
 
 describe('resolveDockerImage()', () => {
@@ -442,7 +441,7 @@ describe('setupServerlessVolumes()', () => {
     const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath });
 
     volumeCmdTest(volumeCmd);
-    expect(existsSync(serverlessObjectStorePath)).toBe(true);
+    await expect(Fsp.access(serverlessObjectStorePath)).resolves.not.toThrow();
   });
 
   test('should use an existing object store', async () => {
@@ -451,7 +450,9 @@ describe('setupServerlessVolumes()', () => {
     const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath });
 
     volumeCmdTest(volumeCmd);
-    expect(existsSync(`${serverlessObjectStorePath}/cluster_state/lease`)).toBe(true);
+    await expect(
+      Fsp.access(`${serverlessObjectStorePath}/cluster_state/lease`)
+    ).resolves.not.toThrow();
   });
 
   test('should remove an existing object store when clean is passed', async () => {
@@ -460,7 +461,9 @@ describe('setupServerlessVolumes()', () => {
     const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, clean: true });
 
     volumeCmdTest(volumeCmd);
-    expect(existsSync(`${serverlessObjectStorePath}/cluster_state/lease`)).toBe(false);
+    await expect(
+      Fsp.access(`${serverlessObjectStorePath}/cluster_state/lease`)
+    ).rejects.toThrowError();
   });
 
   test('should add SSL volumes when ssl is passed', async () => {
diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts
index 1c89339e1a567..2981acb56d0f4 100644
--- a/packages/kbn-es/src/utils/docker.ts
+++ b/packages/kbn-es/src/utils/docker.ts
@@ -486,12 +486,19 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
   log.info(chalk.bold(`Checking for local serverless ES object store at ${objectStorePath}`));
   log.indent(4);
 
-  if (clean && fs.existsSync(objectStorePath)) {
+  let exists = null;
+  try {
+    await Fsp.access(objectStorePath);
+    exists = true;
+  } catch (e) {
+    exists = false;
+  }
+  if (clean && exists) {
     log.info('Cleaning existing object store.');
     await Fsp.rm(objectStorePath, { recursive: true, force: true });
   }
 
-  if (clean || !fs.existsSync(objectStorePath)) {
+  if (clean || !exists) {
     await Fsp.mkdir(objectStorePath, { recursive: true }).then(() =>
       log.info('Created new object store.')
     );
diff --git a/src/dev/build/tasks/patch_native_modules_task.ts b/src/dev/build/tasks/patch_native_modules_task.ts
index 24ab98d132203..c928d8a5b4d06 100644
--- a/src/dev/build/tasks/patch_native_modules_task.ts
+++ b/src/dev/build/tasks/patch_native_modules_task.ts
@@ -47,8 +47,8 @@ const packages: Package[] = [
     extractMethod: 'gunzip',
     archives: {
       'linux-x64': {
-        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/linux-x64-108.gz',
-        sha256: 'e14f274f73ede22f170bfe9e57a0645ebf7ed320042a27361fa158bc239a5563',
+        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/linux-x64-115.gz',
+        sha256: '7a4821ef7e9ddbafe5bba6beb54b100c233242f0dbf5a7268f55beea5f845f97',
       },
       // Linux ARM builds are currently done manually as Github Actions used in upstream project
       // do not natively support an Linux ARM target.
@@ -63,20 +63,20 @@ const packages: Package[] = [
       // * capture the sha256 with: `shasum -a 256 linux-arm64-*`
       // * upload the `linux-arm64-*.gz` artifact to the `yarn-prebuilt-artifacts` bucket in GCS using the correct version number
       'linux-arm64': {
-        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/linux-arm64-108.gz',
-        sha256: 'cbdf3f75a331c601ac0bd34715814d0a1fd17612c6d6b5269f176d46044defd5',
+        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/linux-arm64-115.gz',
+        sha256: '8d753d6ac15d95d6d236dce2f986f4a6b2f9945ba0d927ab972eb82da68d14b1',
       },
       'darwin-x64': {
-        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/darwin-x64-108.gz',
-        sha256: 'f88c09e98f152ac15c593b3b923b7fbe28d448cfde5986da40c34461bede5a09',
+        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/darwin-x64-115.gz',
+        sha256: '91823077c510c6da9c428038bfd210846373bcd0ab6851f7408add67864785a9',
       },
       'darwin-arm64': {
-        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/darwin-arm64-108.gz',
-        sha256: '80700aecbe63052149aba721449a8ce30c24d884e414025124bb4602efe708be',
+        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/darwin-arm64-115.gz',
+        sha256: '935e2a5590e93e6f52f41d40ae4115fbd2f130a4d61afb0a6549ed17adb1dd84',
       },
       'win32-x64': {
-        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/win32-x64-108.gz',
-        sha256: 'cadc4713907f3ad1de45f470810ec8e13e08f32c1a1e45e5d5ab5e9d7fcb9763',
+        url: 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/node-re2/uhop/node-re2/releases/download/1.20.1/win32-x64-115.gz',
+        sha256: '120c2663bcab4803f8405d9aa2cb97fa181d90b4ff176827cc7295667aa2e9c2',
       },
     },
   },
diff --git a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
index fb9db4490144b..f4eeca4bc5ba6 100644
--- a/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
+++ b/src/plugins/bfetch/public/batching/create_streaming_batched_function.test.ts
@@ -744,7 +744,7 @@ describe('createStreamingBatchedFunction()', () => {
       const [, error1] = await promise1;
       const [result1] = await promise2;
       expect(error1).toMatchObject({
-        message: 'Unexpected token N in JSON at position 0',
+        message: `Unexpected token 'N', "Not a JSON\n" is not valid JSON`,
         code: 'STREAM',
       });
       expect(result1).toMatchObject({
diff --git a/src/plugins/data/public/search/expressions/esdsl.test.ts b/src/plugins/data/public/search/expressions/esdsl.test.ts
index 2bf2ef1148507..603dbf1e8f3eb 100644
--- a/src/plugins/data/public/search/expressions/esdsl.test.ts
+++ b/src/plugins/data/public/search/expressions/esdsl.test.ts
@@ -63,7 +63,7 @@ describe('esdsl', () => {
       } catch (error) {
         errorMessage = error.message;
       }
-      expect(errorMessage).toEqual('Unexpected token i in JSON at position 0');
+      expect(errorMessage).toEqual(`Unexpected token 'i', "invalid json" is not valid JSON`);
     });
 
     test('adds filters', async () => {
diff --git a/src/setup_node_env/exit_on_warning.js b/src/setup_node_env/exit_on_warning.js
index 5e7bae8254c04..dc6e321074224 100644
--- a/src/setup_node_env/exit_on_warning.js
+++ b/src/setup_node_env/exit_on_warning.js
@@ -46,6 +46,13 @@ var IGNORE_WARNINGS = [
     //  We need to discard that warning
     name: 'ProductNotSupportedSecurityError',
   },
+  // https://github.com/browserify/browserify-rsa/pull/20
+  {
+    name: 'DeprecationWarning',
+    code: 'DEP0170',
+    message:
+      'The URL https://github.com:crypto-browserify/browserify-rsa.git is invalid. Future versions of Node.js will throw an error.',
+  },
 ];
 
 if (process.noProcessWarnings !== true) {
diff --git a/src/setup_node_env/heap_snapshot.js b/src/setup_node_env/heap_snapshot.js
deleted file mode 100644
index 94e4b35e2f887..0000000000000
--- a/src/setup_node_env/heap_snapshot.js
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-var getopts = require('getopts');
-var path = require('path');
-var v8 = require('node:v8');
-var worker = require('node:worker_threads');
-
-var execOpts = getopts(process.execArgv);
-var envOpts = getopts(process.env.NODE_OPTIONS ? process.env.NODE_OPTIONS.split(/\s+/) : []);
-var diagnosticDir = execOpts['diagnostic-dir'] || envOpts['diagnostic-dir'];
-var heapSnapshotSignal = execOpts['heapsnapshot-signal'] || envOpts['heapsnapshot-signal'];
-var heapSnapshotSerial = 0;
-
-function getHeapSnapshotPath() {
-  var now = new Date();
-
-  var year = now.getFullYear();
-  var month = String(now.getMonth() + 1).padStart(2, '0');
-  var day = String(now.getDate()).padStart(2, '0');
-  var hours = String(now.getHours()).padStart(2, '0');
-  var minutes = String(now.getMinutes()).padStart(2, '0');
-  var seconds = String(now.getSeconds()).padStart(2, '0');
-
-  var date = `${year}${month}${day}`;
-  var time = `${hours}${minutes}${seconds}`;
-  var pid = process.pid;
-  var threadId = worker.threadId;
-  var serial = (++heapSnapshotSerial).toString().padStart(3, '0');
-
-  return path.join(diagnosticDir, `Heap.${date}.${time}.${pid}.${threadId}.${serial}.heapsnapshot`);
-}
-
-if (diagnosticDir && heapSnapshotSignal) {
-  process.removeAllListeners(heapSnapshotSignal);
-
-  process.on(heapSnapshotSignal, function () {
-    var heapSnapshotPath = getHeapSnapshotPath();
-    v8.writeHeapSnapshot(heapSnapshotPath);
-  });
-}
diff --git a/src/setup_node_env/setup_env.js b/src/setup_node_env/setup_env.js
index d3076a2c3b9cf..7b37d98011cfb 100644
--- a/src/setup_node_env/setup_env.js
+++ b/src/setup_node_env/setup_env.js
@@ -11,8 +11,6 @@ require('./exit_on_warning');
 require('./harden');
 // The following require statements MUST be executed before any others - END
 
-// @todo Remove when migrated to Node 20 (#162696)
-require('./heap_snapshot');
 require('symbol-observable');
 require('source-map-support').install();
 require('./node_version_validator');
diff --git a/x-pack/plugins/cases/server/saved_object_types/migrations/user_actions/connector_id.test.ts b/x-pack/plugins/cases/server/saved_object_types/migrations/user_actions/connector_id.test.ts
index ed43cc40649c6..2ed37837f5100 100644
--- a/x-pack/plugins/cases/server/saved_object_types/migrations/user_actions/connector_id.test.ts
+++ b/x-pack/plugins/cases/server/saved_object_types/migrations/user_actions/connector_id.test.ts
@@ -254,7 +254,7 @@ describe('user action migrations', () => {
           const log = context.log as jest.Mocked<SavedObjectsMigrationLogger>;
           expect(log.error.mock.calls[0]).toMatchInlineSnapshot(`
             Array [
-              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Unexpected token a in JSON at position 1",
+              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Expected property name or '}' in JSON at position 1",
               Object {
                 "migrations": Object {
                   "userAction": Object {
@@ -448,7 +448,7 @@ describe('user action migrations', () => {
           const log = context.log as jest.Mocked<SavedObjectsMigrationLogger>;
           expect(log.error.mock.calls[0]).toMatchInlineSnapshot(`
             Array [
-              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Unexpected token b in JSON at position 1",
+              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Expected property name or '}' in JSON at position 1",
               Object {
                 "migrations": Object {
                   "userAction": Object {
@@ -644,7 +644,7 @@ describe('user action migrations', () => {
           const log = context.log as jest.Mocked<SavedObjectsMigrationLogger>;
           expect(log.error.mock.calls[0]).toMatchInlineSnapshot(`
             Array [
-              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Unexpected token e in JSON at position 1",
+              "Failed to migrate user action connector with doc id: 1 version: 8.0.0 error: Unexpected token 'e', \\"new json value\\" is not valid JSON",
               Object {
                 "migrations": Object {
                   "userAction": Object {
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/document_creation/document_creation_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/document_creation/document_creation_logic.test.ts
index 3e39c50ff8de7..925f5af35ff48 100644
--- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/document_creation/document_creation_logic.test.ts
+++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/document_creation/document_creation_logic.test.ts
@@ -404,7 +404,7 @@ describe('DocumentCreationLogic', () => {
           DocumentCreationLogic.actions.onSubmitJson();
 
           expect(DocumentCreationLogic.actions.setErrors).toHaveBeenCalledWith([
-            'Unexpected token i in JSON at position 0',
+            `Unexpected token 'i', "invalid JSON" is not valid JSON`,
           ]);
           expect(DocumentCreationLogic.actions.uploadDocuments).not.toHaveBeenCalled();
         });
diff --git a/x-pack/plugins/screenshotting/server/browsers/download/fetch.test.ts b/x-pack/plugins/screenshotting/server/browsers/download/fetch.test.ts
index cc22f152216af..b32f98aeffe20 100644
--- a/x-pack/plugins/screenshotting/server/browsers/download/fetch.test.ts
+++ b/x-pack/plugins/screenshotting/server/browsers/download/fetch.test.ts
@@ -8,7 +8,7 @@
 import mockFs from 'mock-fs';
 import axios from 'axios';
 import { createHash } from 'crypto';
-import { readFileSync } from 'fs';
+import { readFile } from 'fs/promises';
 import { resolve as resolvePath } from 'path';
 import { Readable } from 'stream';
 import { fetch } from './fetch';
@@ -38,7 +38,7 @@ describe('fetch', () => {
   test('downloads the url to the path', async () => {
     await fetch('url', TEMP_FILE);
 
-    expect(readFileSync(TEMP_FILE, 'utf8')).toEqual('foobar');
+    await expect(readFile(TEMP_FILE, 'utf8')).resolves.toBe('foobar');
   });
 
   test('returns the md5 hex hash of the http body', async () => {
diff --git a/x-pack/plugins/screenshotting/server/browsers/download/fetch.ts b/x-pack/plugins/screenshotting/server/browsers/download/fetch.ts
index ba3e083816937..e248aad94c14e 100644
--- a/x-pack/plugins/screenshotting/server/browsers/download/fetch.ts
+++ b/x-pack/plugins/screenshotting/server/browsers/download/fetch.ts
@@ -7,7 +7,8 @@
 
 import Axios from 'axios';
 import { createHash } from 'crypto';
-import { closeSync, mkdirSync, openSync, writeSync } from 'fs';
+import { mkdir, open } from 'fs/promises';
+import { writeSync } from 'fs';
 import { dirname } from 'path';
 import { finished, Readable } from 'stream';
 import { promisify } from 'util';
@@ -21,9 +22,8 @@ export async function fetch(url: string, path: string, logger?: Logger): Promise
 
   const hash = createHash('md5');
 
-  mkdirSync(dirname(path), { recursive: true });
-  const handle = openSync(path, 'w');
-
+  await mkdir(dirname(path), { recursive: true });
+  const handle = await open(path, 'w');
   try {
     const response = await Axios.request<Readable>({
       url,
@@ -32,7 +32,7 @@ export async function fetch(url: string, path: string, logger?: Logger): Promise
     });
 
     response.data.on('data', (chunk: Buffer) => {
-      writeSync(handle, chunk);
+      writeSync(handle.fd, chunk);
       hash.update(chunk);
     });
 
@@ -43,7 +43,7 @@ export async function fetch(url: string, path: string, logger?: Logger): Promise
 
     throw new Error(`Unable to download ${url}: ${error}`);
   } finally {
-    closeSync(handle);
+    await handle.close();
   }
 
   return hash.digest('hex');
diff --git a/x-pack/plugins/screenshotting/server/browsers/download/index.test.ts b/x-pack/plugins/screenshotting/server/browsers/download/index.test.ts
index 887a631a2c1d7..8b8115c5a6164 100644
--- a/x-pack/plugins/screenshotting/server/browsers/download/index.test.ts
+++ b/x-pack/plugins/screenshotting/server/browsers/download/index.test.ts
@@ -7,7 +7,7 @@
 
 import path from 'path';
 import mockFs from 'mock-fs';
-import { existsSync, readdirSync } from 'fs';
+import { access, readdir } from 'fs/promises';
 import { ChromiumArchivePaths, PackageInfo } from '../chromium';
 import { fetch } from './fetch';
 import { md5 } from './checksum';
@@ -55,8 +55,8 @@ describe('ensureDownloaded', () => {
 
     await download(paths, pkg);
 
-    expect(existsSync(unexpectedPath1)).toBe(false);
-    expect(existsSync(unexpectedPath2)).toBe(false);
+    await expect(access(unexpectedPath1)).rejects.toThrow();
+    await expect(access(unexpectedPath2)).rejects.toThrow();
   });
 
   it('should reject when download fails', async () => {
@@ -84,14 +84,14 @@ describe('ensureDownloaded', () => {
       await download(paths, pkg);
 
       expect(fetch).not.toHaveBeenCalled();
-      expect(readdirSync(path.resolve(`${paths.archivesPath}/x64`))).toEqual(
+      await expect(readdir(path.resolve(`${paths.archivesPath}/x64`))).resolves.toEqual(
         expect.arrayContaining([
           'chrome-mac.zip',
           'chrome-win.zip',
           expect.stringMatching(/^chromium-[0-9a-f]{7}-locales-linux_x64\.zip$/),
         ])
       );
-      expect(readdirSync(path.resolve(`${paths.archivesPath}/arm64`))).toEqual(
+      await expect(readdir(path.resolve(`${paths.archivesPath}/arm64`))).resolves.toEqual(
         expect.arrayContaining([
           'chrome-mac.zip',
           expect.stringMatching(/^chromium-[0-9a-f]{7}-locales-linux_arm64\.zip$/),
diff --git a/x-pack/plugins/screenshotting/server/browsers/download/index.ts b/x-pack/plugins/screenshotting/server/browsers/download/index.ts
index ff8de66df743a..61b46ac84c588 100644
--- a/x-pack/plugins/screenshotting/server/browsers/download/index.ts
+++ b/x-pack/plugins/screenshotting/server/browsers/download/index.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { existsSync } from 'fs';
+import { access } from 'fs/promises';
 import del from 'del';
 import type { Logger } from '@kbn/core/server';
 import type { ChromiumArchivePaths, PackageInfo } from '../chromium';
@@ -42,7 +42,13 @@ export async function download(
   const resolvedPath = paths.resolvePath(pkg);
   const foundChecksum = await md5(resolvedPath).catch(() => 'MISSING');
 
-  const pathExists = existsSync(resolvedPath);
+  let pathExists = null;
+  try {
+    await access(resolvedPath);
+    pathExists = true;
+  } catch (e) {
+    pathExists = false;
+  }
   if (pathExists && foundChecksum === archiveChecksum) {
     logger?.debug(
       `Browser archive for ${pkg.platform}/${pkg.architecture} already found in ${resolvedPath} with matching checksum.`
diff --git a/x-pack/plugins/screenshotting/server/browsers/extract/unzip.test.ts b/x-pack/plugins/screenshotting/server/browsers/extract/unzip.test.ts
index 4af457a0c3a6e..08eac21450f26 100644
--- a/x-pack/plugins/screenshotting/server/browsers/extract/unzip.test.ts
+++ b/x-pack/plugins/screenshotting/server/browsers/extract/unzip.test.ts
@@ -6,7 +6,7 @@
  */
 
 import mockFs from 'mock-fs';
-import { readFileSync } from 'fs';
+import { readFile } from 'fs/promises';
 import { ExtractError } from './extract_error';
 import { unzip } from './unzip';
 
@@ -28,7 +28,7 @@ describe('unzip', () => {
   it('should extract zipped contents', async () => {
     await unzip('/test.zip', '/output');
 
-    expect(readFileSync('/output/test.txt').toString()).toEqual('test');
+    await expect(readFile('/output/test.txt', 'utf8')).resolves.toBe('test');
   });
 
   it('should reject on invalid archive', async () => {
diff --git a/x-pack/plugins/searchprofiler/public/application/lib/check_for_json_errors.test.ts b/x-pack/plugins/searchprofiler/public/application/lib/check_for_json_errors.test.ts
index ccd8761c031b2..606b302fc8559 100644
--- a/x-pack/plugins/searchprofiler/public/application/lib/check_for_json_errors.test.ts
+++ b/x-pack/plugins/searchprofiler/public/application/lib/check_for_json_errors.test.ts
@@ -12,7 +12,7 @@ describe('checkForParseErrors', function () {
   it('returns error from bad JSON', function () {
     const json = '{"foo": {"bar": {"baz": "buzz}}}';
     const result = checkForParseErrors(json);
-    expect(result.error.message).to.be(`Unexpected end of JSON input`);
+    expect(result.error.message).to.be(`Unterminated string in JSON at position 32`);
   });
 
   it('returns parsed value from good JSON', function () {
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts
index f383a9d11cc00..da4f9e67d7b8f 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts
@@ -167,7 +167,7 @@ describe('Import rules route', () => {
         errors: [
           {
             error: {
-              message: 'Unexpected token h in JSON at position 1',
+              message: `Unexpected token 'h', "this is not"... is not valid JSON`,
               status_code: 400,
             },
             rule_id: '(unknown id)',
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts
index be9561598fd08..50b8d5e7ee6fc 100644
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts
@@ -234,7 +234,9 @@ describe('create_rules_stream_from_ndjson', () => {
         type: 'query',
         immutable: false,
       });
-      expect(resultOrError[1].message).toEqual('Unexpected token , in JSON at position 1');
+      expect(resultOrError[1].message).toEqual(
+        `Expected property name or '}' in JSON at position 1`
+      );
       expect(resultOrError[2]).toEqual({
         rule_id: 'rule-2',
         output_index: '.siem-signals',
diff --git a/x-pack/plugins/task_manager/server/saved_objects/migrations.test.ts b/x-pack/plugins/task_manager/server/saved_objects/migrations.test.ts
index 081918673d82a..794a9c466906f 100644
--- a/x-pack/plugins/task_manager/server/saved_objects/migrations.test.ts
+++ b/x-pack/plugins/task_manager/server/saved_objects/migrations.test.ts
@@ -282,7 +282,7 @@ describe('handles errors during migrations', () => {
         migration800(taskInstance, migrationContext);
       }).toThrowError();
       expect(migrationContext.log.error).toHaveBeenCalledWith(
-        `savedObject 8.0.0 migration failed for task instance ${taskInstance.id} with error: Unexpected token s in JSON at position 2`,
+        `savedObject 8.0.0 migration failed for task instance ${taskInstance.id} with error: Expected property name or '}' in JSON at position 2`,
         {
           migrations: {
             taskInstanceDocument: {
diff --git a/yarn.lock b/yarn.lock
index 8016568cd10f5..9c1d247a8f5fb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -9549,10 +9549,12 @@
   dependencies:
     "@types/node" "*"
 
-"@types/node@*", "@types/node@18.18.5", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10 || ^16.0.0", "@types/node@^14.14.20 || ^16.0.0", "@types/node@^18.11.18", "@types/node@^18.17.5":
-  version "18.18.5"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.5.tgz#afc0fd975df946d6e1add5bbf98264225b212244"
-  integrity sha512-4slmbtwV59ZxitY4ixUZdy1uRLf9eSIvBWPQxNjhHYWEtn0FryfKpyS2cvADYXTayWdKEIsJengncrVvkI4I6A==
+"@types/node@*", "@types/node@20.9.0", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10 || ^16.0.0", "@types/node@^14.14.20 || ^16.0.0", "@types/node@^18.11.18", "@types/node@^18.17.5":
+  version "20.9.0"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.9.0.tgz#bfcdc230583aeb891cf51e73cfdaacdd8deae298"
+  integrity sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==
+  dependencies:
+    undici-types "~5.26.4"
 
 "@types/nodemailer@^6.4.0":
   version "6.4.0"
@@ -29384,6 +29386,11 @@ unc-path-regex@^0.1.2:
   resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa"
   integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo=
 
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
+
 undici@^5.21.2, undici@^5.22.1:
   version "5.26.3"
   resolved "https://registry.yarnpkg.com/undici/-/undici-5.26.3.tgz#ab3527b3d5bb25b12f898dfd22165d472dd71b79"