Merge branch 'main' into task/dw-serverless-endpoint-exceptions-pli-w…

…ith-plugin-sub-features

.github/CODEOWNERS

@@ -827,6 +827,12 @@ packages/kbn-yarn-lock-validator @elastic/kibana-operations
 /x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js @elastic/kibana-data-discovery
 /x-pack/test/stack_functional_integration/apps/management/_index_pattern_create.js @elastic/kibana-data-discovery
 /x-pack/test/upgrade/apps/discover @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/data_views @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/data_view_field_editor @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/kql_telemetry @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/scripts_tests @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/search_oss @elastic/kibana-data-discovery
+/x-pack/test_serverless/api_integration/test_suites/common/search_xpack @elastic/kibana-data-discovery
 
 # Visualizations
 /src/plugins/visualize/ @elastic/kibana-visualizations

packages/kbn-securitysolution-utils/src/transform_data_to_ndjson/index.test.ts

@@ -20,7 +20,6 @@ const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE) => ({
   enabled: true,
   false_positives: ['false positive 1', 'false positive 2'],
   from: 'now-6m',
-  investigation_fields: ['custom.field1', 'custom.field2'],
   immutable: false,
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',

src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.ts

@@ -271,6 +271,7 @@ export class ContentStream extends Duplex {
    * of holding, at most, 2 full chunks in memory.
    */
   private indexRequestBuffer: undefined | IndexRequestParams;
+
   private async writeChunk(data: Buffer) {
     const chunkId = this.getChunkId(this.chunksWritten);
 

src/plugins/files/server/blob_storage_service/adapters/es/es.ts

@@ -183,11 +183,19 @@ export class ElasticsearchBlobStorageClient implements BlobStorageClient {
   }
 
   public async download({ id, size }: { id: string; size?: number }): Promise<Readable> {
+    // The refresh interval is set to 10s. To avoid throwing an error if the user tries to download a file
+    // right after uploading it, we refresh the index before downloading the file.
+    await this.esClient.indices.refresh({ index: this.index });
+
     return this.getReadableContentStream(id, size);
   }
 
   public async delete(id: string): Promise<void> {
     try {
+      // The refresh interval is set to 10s. To avoid throwing an error if the user tries to delete a file
+      // right after uploading it, we refresh the index before deleting the file.
+      await this.esClient.indices.refresh({ index: this.index });
+
       const dest = getWritableContentStream({
         id,
         client: this.esClient,

src/plugins/files/server/blob_storage_service/adapters/es/integration_tests/es.test.ts

@@ -22,6 +22,7 @@ describe('Elasticsearch blob storage', () => {
   let esBlobStorage: ElasticsearchBlobStorageClient;
   let esClient: ElasticsearchClient;
   let esGetSpy: jest.SpyInstance;
+  let esRefreshIndexSpy: jest.SpyInstance;
 
   beforeAll(async () => {
     ElasticsearchBlobStorageClient.configureConcurrentUpload(Infinity);
@@ -48,6 +49,7 @@ describe('Elasticsearch blob storage', () => {
   beforeEach(() => {
     esBlobStorage = createEsBlobStorage();
     esGetSpy = jest.spyOn(esClient, 'get');
+    esRefreshIndexSpy = jest.spyOn(esClient.indices, 'refresh');
   });
 
   afterEach(async () => {
@@ -105,7 +107,9 @@ describe('Elasticsearch blob storage', () => {
     esBlobStorage = createEsBlobStorage({ chunkSize: '1024B' });
     const { id } = await esBlobStorage.upload(Readable.from([fileString]));
     expect(await getAllDocCount()).toMatchObject({ count: 37 });
+    esRefreshIndexSpy.mockReset();
     const rs = await esBlobStorage.download({ id });
+    expect(esRefreshIndexSpy).toHaveBeenCalled(); // Make sure we refresh the index before downloading the chunks
     const chunks: string[] = [];
     for await (const chunk of rs) {
       chunks.push(chunk);
@@ -137,7 +141,9 @@ describe('Elasticsearch blob storage', () => {
     const { id } = await esBlobStorage.upload(Readable.from([fileString]));
     const { id: id2 } = await esBlobStorage.upload(Readable.from([fileString2]));
     expect(await getAllDocCount()).toMatchObject({ count: 10 });
+    esRefreshIndexSpy.mockReset();
     await esBlobStorage.delete(id);
+    expect(esRefreshIndexSpy).toHaveBeenCalled(); // Make sure we refresh the index before deleting the chunks
     expect(await getAllDocCount()).toMatchObject({ count: 2 });
     // Now we check that the other file is still intact
     const rs = await esBlobStorage.download({ id: id2 });

test/functional/services/lib/web_element_wrapper/web_element_wrapper.ts

@@ -730,7 +730,32 @@ export class WebElementWrapper {
   }
 
   /**
-   * Scroll the element into view, avoiding the fixed header if necessary
+   * Scroll the element into view
+   *
+   * @param {ScrollIntoViewOptions} scrollIntoViewOptions
+   * @return {Promise<void>}
+   */
+  public scrollIntoView(scrollIntoViewOptions?: ScrollIntoViewOptions) {
+    return this.driver.executeScript<void>(
+      (target: HTMLElement, options: ScrollIntoViewOptions) => target.scrollIntoView(options),
+      this._webElement,
+      scrollIntoViewOptions
+    );
+  }
+
+  /**
+   * Scroll the element into view if it is not already, avoiding the fixed header if necessary
+   * This method is a variation of the scrollIntoView method, where we only scroll into an element
+   * if it is not part of the "scrollable view".
+   * This implies a specific behavior, since the "scrollable view" of the view is identified by
+   * the `document.scrollingElement`, which always results to the html or body tag.
+   *
+   * Use cases:
+   * - An element (a section, a footer) is not visible in the whole page and we need to scroll into it.
+   * - An element is covered by the fixed header and we need to scroll into it ensuring is not covered.
+   *
+   * In case you have a scrollable list smaller that the size of the HTML document and you need
+   * to scroll into an element of that list, prefer using the `.scrollIntoView` method.
    *
    * @nonstandard
    * @return {Promise<void>}

x-pack/plugins/cloud_security_posture/common/constants.ts

@@ -27,6 +27,7 @@ export const FIND_CSP_RULE_TEMPLATE_ROUTE_PATH = '/internal/cloud_security_postu
 export const FIND_CSP_RULE_TEMPLATE_API_CURRENT_VERSION = '1';
 
 export const DETECTION_RULE_ALERTS_STATUS_API_CURRENT_VERSION = '1';
+export const DETECTION_RULE_RULES_API_CURRENT_VERSION = '2023-10-31';
 
 export const GET_DETECTION_RULE_ALERTS_STATUS_PATH =
   '/internal/cloud_security_posture/detection_engine_rules/alerts/_status';

x-pack/plugins/cloud_security_posture/public/common/api/create_detection_rule.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 import { HttpSetup } from '@kbn/core/public';
+import { DETECTION_RULE_RULES_API_CURRENT_VERSION } from '../../../common/constants';
 import { RuleCreateProps, RuleResponse } from '../types';
 
 const DETECTION_ENGINE_URL = '/api/detection_engine' as const;
@@ -18,6 +19,7 @@ export const createDetectionRule = async ({
   rule: RuleCreateProps;
 }): Promise<RuleResponse> => {
   const res = await http.post<RuleCreateProps>(DETECTION_ENGINE_RULES_URL, {
+    version: DETECTION_RULE_RULES_API_CURRENT_VERSION,
     body: JSON.stringify(rule),
   });
 

x-pack/plugins/cloud_security_posture/public/common/api/use_fetch_detection_rules_by_tags.ts

@@ -8,6 +8,7 @@
 import { CoreStart } from '@kbn/core/public';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { useQuery } from '@tanstack/react-query';
+import { DETECTION_RULE_RULES_API_CURRENT_VERSION } from '../../../common/constants';
 import { RuleResponse } from '../types';
 import { DETECTION_ENGINE_RULES_KEY } from '../constants';
 
@@ -47,6 +48,7 @@ export const useFetchDetectionRulesByTags = (tags: string[]) => {
   return useQuery([DETECTION_ENGINE_RULES_KEY, tags], () =>
     http.fetch<FetchRulesResponse>(DETECTION_ENGINE_RULES_URL_FIND, {
       method: 'GET',
+      version: DETECTION_RULE_RULES_API_CURRENT_VERSION,
       query,
     })
   );

x-pack/plugins/infra/public/components/asset_details/asset_details.tsx

@@ -28,15 +28,12 @@ export const AssetDetails = ({
   tabs,
   links,
   renderMode,
-  activeTabId,
   metricAlias,
   ...props
 }: AssetDetailsProps) => {
   return (
     <ContextProviders props={{ ...props, renderMode }}>
-      <TabSwitcherProvider
-        initialActiveTabId={tabs.length > 0 ? activeTabId ?? tabs[0].id : undefined}
-      >
+      <TabSwitcherProvider defaultActiveTabId={tabs[0]?.id}>
         <DataViewsProvider metricAlias={metricAlias}>
           <ContentTemplate header={{ tabs, links }} renderMode={renderMode} />
         </DataViewsProvider>

x-pack/plugins/infra/public/components/asset_details/asset_details_embeddable.tsx

@@ -71,14 +71,12 @@ export class AssetDetailsEmbeddable extends Embeddable<AssetDetailsEmbeddableInp
         <EuiThemeProvider>
           <div style={{ width: '100%' }}>
             <LazyAssetDetailsWrapper
-              activeTabId={this.input.activeTabId}
               dateRange={this.input.dateRange}
               asset={this.input.asset}
               assetType={this.input.assetType}
               overrides={this.input.overrides}
               renderMode={this.input.renderMode}
               tabs={this.input.tabs}
-              onTabsStateChange={this.input.onTabsStateChange}
               links={this.input.links}
               metricAlias={this.input.metricAlias}
             />

x-pack/plugins/infra/public/components/asset_details/context_providers.tsx

@@ -6,7 +6,7 @@
  */
 
 import React from 'react';
-import { AssetDetailsStateProvider } from './hooks/use_asset_details_state';
+import { AssetDetailsRenderPropsProvider } from './hooks/use_asset_details_render_props';
 import { DateRangeProvider } from './hooks/use_date_range';
 import { MetadataStateProvider } from './hooks/use_metadata_state';
 import { AssetDetailsProps } from './types';
@@ -17,21 +17,20 @@ export const ContextProviders = ({
 }: { props: Omit<AssetDetailsProps, 'links' | 'tabs' | 'activeTabId' | 'metricAlias'> } & {
   children: React.ReactNode;
 }) => {
-  const { asset, dateRange, overrides, onTabsStateChange, assetType = 'host', renderMode } = props;
+  const { asset, dateRange, overrides, assetType = 'host', renderMode } = props;
   return (
     <DateRangeProvider initialDateRange={dateRange}>
       <MetadataStateProvider asset={asset} assetType={assetType}>
-        <AssetDetailsStateProvider
-          state={{
+        <AssetDetailsRenderPropsProvider
+          props={{
             asset,
             assetType,
             overrides,
-            onTabsStateChange,
             renderMode,
           }}
         >
           {children}
-        </AssetDetailsStateProvider>
+        </AssetDetailsRenderPropsProvider>
       </MetadataStateProvider>
     </DateRangeProvider>
   );

x-pack/plugins/infra/public/components/asset_details/date_picker/date_picker.tsx

@@ -7,22 +7,17 @@
 
 import { EuiSuperDatePicker, type OnTimeChangeProps } from '@elastic/eui';
 import React, { useCallback } from 'react';
-import { useAssetDetailsStateContext } from '../hooks/use_asset_details_state';
 import { useDateRangeProviderContext } from '../hooks/use_date_range';
 
 export const DatePicker = () => {
-  const { onTabsStateChange } = useAssetDetailsStateContext();
   const { dateRange, setDateRange } = useDateRangeProviderContext();
   const onTimeChange = useCallback(
     ({ start, end, isInvalid }: OnTimeChangeProps) => {
       if (!isInvalid) {
         setDateRange({ from: start, to: end });
-        if (onTabsStateChange) {
-          onTabsStateChange({ dateRange: { from: start, to: end } });
-        }
       }
     },
-    [onTabsStateChange, setDateRange]
+    [setDateRange]
   );
 
   return (

x-pack/plugins/infra/public/components/asset_details/hooks/use_asset_details_state.ts → x-pack/plugins/infra/public/components/asset_details/hooks/use_asset_details_render_props.ts

@@ -9,16 +9,13 @@ import createContainer from 'constate';
 import type { AssetDetailsProps } from '../types';
 import { useMetadataStateProviderContext } from './use_metadata_state';
 
-export interface UseAssetDetailsStateProps {
-  state: Pick<
-    AssetDetailsProps,
-    'asset' | 'assetType' | 'overrides' | 'onTabsStateChange' | 'renderMode'
-  >;
+export interface UseAssetDetailsRenderProps {
+  props: Pick<AssetDetailsProps, 'asset' | 'assetType' | 'overrides' | 'renderMode'>;
 }
 
-export function useAssetDetailsState({ state }: UseAssetDetailsStateProps) {
+export function useAssetDetailsRenderProps({ props }: UseAssetDetailsRenderProps) {
   const { metadata } = useMetadataStateProviderContext();
-  const { asset, assetType, onTabsStateChange, overrides, renderMode } = state;
+  const { asset, assetType, overrides, renderMode } = props;
 
   // When the asset asset.name is known we can load the page faster
   // Otherwise we need to use metadata response.
@@ -30,12 +27,12 @@ export function useAssetDetailsState({ state }: UseAssetDetailsStateProps) {
       name: asset.name || metadata?.name || 'asset-name',
     },
     assetType,
-    onTabsStateChange,
     overrides,
     renderMode,
     loading,
   };
 }
 
-export const AssetDetailsState = createContainer(useAssetDetailsState);
-export const [AssetDetailsStateProvider, useAssetDetailsStateContext] = AssetDetailsState;
+export const AssetDetailsRenderProps = createContainer(useAssetDetailsRenderProps);
+export const [AssetDetailsRenderPropsProvider, useAssetDetailsRenderPropsContext] =
+  AssetDetailsRenderProps;

x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_host_flyout_url_state.ts → x-pack/plugins/infra/public/components/asset_details/hooks/use_asset_details_url_state.ts

@@ -9,28 +9,27 @@ import * as rt from 'io-ts';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { fold } from 'fp-ts/lib/Either';
 import { constant, identity } from 'fp-ts/lib/function';
-import { FlyoutTabIds } from '../../../../components/asset_details/types';
-import { useUrlState } from '../../../../utils/use_url_state';
+import { FlyoutTabIds } from '../types';
+import { useUrlState } from '../../../utils/use_url_state';
 
-export const DEFAULT_STATE: HostFlyout = {
-  itemId: '',
+export const DEFAULT_STATE: AssetDetailsState = {
   tabId: FlyoutTabIds.OVERVIEW,
   processSearch: undefined,
   metadataSearch: undefined,
 };
-const HOST_FLYOUT_URL_STATE_KEY = 'flyout';
+const ASSET_DETAILS_URL_STATE_KEY = 'asset_details';
 
-type SetHostFlyoutState = (newProp: Payload | null) => void;
+type SetAssetDetailsState = (newProp: Payload | null) => void;
 
-export const useHostFlyoutUrlState = (): [HostFlyoutUrl, SetHostFlyoutState] => {
-  const [urlState, setUrlState] = useUrlState<HostFlyoutUrl>({
+export const useAssetDetailsUrlState = (): [AssetDetailsUrl, SetAssetDetailsState] => {
+  const [urlState, setUrlState] = useUrlState<AssetDetailsUrl>({
     defaultState: null,
     decodeUrlState,
     encodeUrlState,
-    urlStateKey: HOST_FLYOUT_URL_STATE_KEY,
+    urlStateKey: ASSET_DETAILS_URL_STATE_KEY,
   });
 
-  const setHostFlyoutState = (newProps: Payload | null) => {
+  const setAssetDetailsState = (newProps: Payload | null) => {
     if (!newProps) {
       setUrlState(DEFAULT_STATE);
     } else {
@@ -41,10 +40,10 @@ export const useHostFlyoutUrlState = (): [HostFlyoutUrl, SetHostFlyoutState] =>
     }
   };
 
-  return [urlState as HostFlyoutUrl, setHostFlyoutState];
+  return [urlState as AssetDetailsUrl, setAssetDetailsState];
 };
 
-const FlyoutTabIdRT = rt.union([
+const TabIdRT = rt.union([
   rt.literal(FlyoutTabIds.OVERVIEW),
   rt.literal(FlyoutTabIds.METADATA),
   rt.literal(FlyoutTabIds.PROCESSES),
@@ -53,10 +52,9 @@ const FlyoutTabIdRT = rt.union([
   rt.literal(FlyoutTabIds.OSQUERY),
 ]);
 
-const HostFlyoutStateRT = rt.intersection([
+const AssetDetailsStateRT = rt.intersection([
   rt.type({
-    itemId: rt.string,
-    tabId: FlyoutTabIdRT,
+    tabId: TabIdRT,
   }),
   rt.partial({
     dateRange: rt.type({
@@ -69,14 +67,13 @@ const HostFlyoutStateRT = rt.intersection([
   }),
 ]);
 
-const HostFlyoutUrlRT = rt.union([HostFlyoutStateRT, rt.null]);
+const AssetDetailsUrlRT = rt.union([AssetDetailsStateRT, rt.null]);
 
-type HostFlyoutState = rt.TypeOf<typeof HostFlyoutStateRT>;
-type HostFlyoutUrl = rt.TypeOf<typeof HostFlyoutUrlRT>;
-type Payload = Partial<HostFlyoutState>;
-export type HostFlyout = rt.TypeOf<typeof HostFlyoutStateRT>;
+export type AssetDetailsState = rt.TypeOf<typeof AssetDetailsStateRT>;
+type AssetDetailsUrl = rt.TypeOf<typeof AssetDetailsUrlRT>;
+type Payload = Partial<AssetDetailsState>;
 
-const encodeUrlState = HostFlyoutUrlRT.encode;
+const encodeUrlState = AssetDetailsUrlRT.encode;
 const decodeUrlState = (value: unknown) => {
-  return pipe(HostFlyoutUrlRT.decode(value), fold(constant(undefined), identity));
+  return pipe(AssetDetailsUrlRT.decode(value), fold(constant(undefined), identity));
 };