[One Discover] Push Smart Fields / Virtual Columns down into Discover (…

…#181870)

## Summary

Implements **part one** of
#181674.

The main focus here is to get the code for Smart Fields and Virtual
Columns located within Discover. Whilst some hooks have been added to
decide what to register / render these should most definitely be treated
as "subject to change" once the real data type context awareness lands
in Discover. For now these hooks rely on the old extension points to say
"yes, this is a logs context". As far as I could see other solutions are
not using the extension points that have been temporarily altered here.

As this is functionality related to a data type (logs) and not a
solution (observability, for example) this is all treated as a first
class citizen. Some code has been moved to the `discover-utils`.

## Reviewer notes

- Nothing should really have *changed*, this is mostly about verifying
that the Smart Fields and Virtual Columns still work as expected when
using Logs Explorer (and that core Discover still works as expected).
Please also check the Flyout still works as expected as this was rebased
on top of https://github.com/elastic/kibana/pull/180262/files and there
were some conflicts between file relocation in the two.

- There are a couple of components now duplicated in Discover and Logs
Explorer as the `actions` column still uses them and hasn't been moved
across yet.

<img width="306" alt="Screenshot 2024-04-28 at 15 39 15"
src="https://github.com/elastic/kibana/assets/471693/7b56df75-8b0b-41a7-8f15-f1066ef269d7">
<img width="1183" alt="Screenshot 2024-04-28 at 15 39 21"
src="https://github.com/elastic/kibana/assets/471693/167ab53c-9e86-464d-9622-86d87b311b46">

---------

Co-authored-by: Achyut Jhunjhunwala <[email protected]>

packages/kbn-discover-utils/src/data_types/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './logs';

packages/kbn-discover-utils/src/data_types/logs/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './types';
+export * from './utils';

x-pack/plugins/observability_solution/logs_explorer/common/document.ts → packages/kbn-discover-utils/src/data_types/logs/types.ts

@@ -1,11 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-import type { DataTableRecord } from '@kbn/discover-utils/src/types';
+import { DataTableRecord } from '../../types';
 
 export interface LogDocument extends DataTableRecord {
   flattened: {
@@ -40,7 +41,7 @@ export interface LogDocument extends DataTableRecord {
   };
 }
 
-export interface FlyoutDoc {
+export interface LogFlyoutDoc {
   '@timestamp': string;
   'log.level'?: string;
   message?: string;

x-pack/plugins/observability_solution/logs_explorer/public/utils/get_field_from_flattened_doc.ts → packages/kbn-discover-utils/src/data_types/logs/utils/get_field_from_doc.ts

@@ -1,11 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-import { LogDocument } from '../../common/document';
+import { LogDocument } from '../types';
 
 type Field = keyof LogDocument['flattened'];
 

packages/kbn-discover-utils/src/data_types/logs/utils/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './get_field_from_doc';

packages/kbn-discover-utils/src/index.ts

@@ -10,3 +10,4 @@ export * from './constants';
 export * as fieldConstants from './field_constants';
 export * from './hooks';
 export * from './utils';
+export * from './data_types';

src/plugins/discover/common/data_types/logs/constants.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { fieldConstants } from '@kbn/discover-utils';
+import { SmartFieldGridColumnOptions } from './display_options';
+
+export * from '@kbn/discover-utils/src/field_constants';
+
+export const LOGS_EXPLORER_PROFILE_ID = 'logs-explorer';
+
+// Virtual column fields
+export const CONTENT_FIELD = 'content';
+export const RESOURCE_FIELD = 'resource';
+
+// Sizing
+export const DATA_GRID_COLUMN_WIDTH_SMALL = 240;
+export const DATA_GRID_COLUMN_WIDTH_MEDIUM = 320;
+export const ACTIONS_COLUMN_WIDTH = 80;
+
+export const RESOURCE_FIELD_CONFIGURATION: SmartFieldGridColumnOptions = {
+  type: 'smart-field',
+  smartField: RESOURCE_FIELD,
+  fallbackFields: [fieldConstants.HOST_NAME_FIELD, fieldConstants.SERVICE_NAME_FIELD],
+  width: DATA_GRID_COLUMN_WIDTH_MEDIUM,
+};
+
+export const CONTENT_FIELD_CONFIGURATION: SmartFieldGridColumnOptions = {
+  type: 'smart-field',
+  smartField: CONTENT_FIELD,
+  fallbackFields: [fieldConstants.MESSAGE_FIELD],
+};
+
+export const SMART_FALLBACK_FIELDS = {
+  [CONTENT_FIELD]: CONTENT_FIELD_CONFIGURATION,
+  [RESOURCE_FIELD]: RESOURCE_FIELD_CONFIGURATION,
+};
+
+// UI preferences
+export const DEFAULT_COLUMNS = [RESOURCE_FIELD_CONFIGURATION, CONTENT_FIELD_CONFIGURATION];
+export const DEFAULT_ROWS_PER_PAGE = 100;
+
+// List of prefixes which needs to be filtered out for Display in Content Column
+export const FILTER_OUT_FIELDS_PREFIXES_FOR_CONTENT = [
+  '_', // Filter fields like '_id', '_score'
+  '@timestamp',
+  'agent.',
+  'elastic_agent.',
+  'data_stream.',
+  'ecs.',
+  'host.',
+  'container.',
+  'cloud.',
+  'kubernetes.',
+  'orchestrator.',
+  'log.',
+  'service.',
+];
+
+export const DEFAULT_ALLOWED_DATA_VIEWS = ['logs', 'auditbeat', 'filebeat', 'winlogbeat'];
+export const DEFAULT_ALLOWED_LOGS_DATA_VIEWS = ['logs', 'auditbeat', 'filebeat', 'winlogbeat'];

src/plugins/discover/common/data_types/logs/display_options.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export interface SmartFieldGridColumnOptions {
+  type: 'smart-field';
+  smartField: 'content' | 'resource';
+  fallbackFields: string[];
+  width?: number;
+}

src/plugins/discover/public/application/main/components/layout/discover_documents.test.tsx

@@ -109,26 +109,14 @@ describe('Discover documents layout', () => {
   });
 
   test('should render customisations', async () => {
-    const customCellRenderer = {
-      content: () => <span className="custom-renderer-test">Test</span>,
-    };
-
-    const customGridColumnsConfiguration = {
-      content: () => ({
-        id: 'content',
-        displayText: <span className="custom-column-test">Column</span>,
-      }),
-    };
-
     const customControlColumnsConfiguration = () => ({
       leadingControlColumns: [],
       trailingControlColumns: [],
     });
 
     const customization: DiscoverCustomization = {
       id: 'data_table',
-      customCellRenderer,
-      customGridColumnsConfiguration,
+      logsEnabled: true,
       customControlColumnsConfiguration,
     };
 
@@ -137,12 +125,10 @@ describe('Discover documents layout', () => {
     const discoverGridComponent = component.find(DiscoverGrid);
     expect(discoverGridComponent.exists()).toBeTruthy();
 
-    expect(discoverGridComponent.prop('externalCustomRenderers')).toEqual(customCellRenderer);
-    expect(discoverGridComponent.prop('customGridColumnsConfiguration')).toEqual(
-      customGridColumnsConfiguration
-    );
     expect(discoverGridComponent.prop('customControlColumnsConfiguration')).toEqual(
       customControlColumnsConfiguration
     );
+    expect(discoverGridComponent.prop('externalCustomRenderers')).toBeDefined();
+    expect(discoverGridComponent.prop('customGridColumnsConfiguration')).toBeDefined();
   });
 });

src/plugins/discover/public/application/main/components/layout/discover_documents.tsx

@@ -67,6 +67,7 @@ import { useFetchMoreRecords } from './use_fetch_more_records';
 import { SelectedVSAvailableCallout } from './selected_vs_available_callout';
 import { useDiscoverCustomization } from '../../../../customizations';
 import { onResizeGridColumn } from '../../../../utils/on_resize_grid_column';
+import { useContextualGridCustomisations } from '../../hooks/grid_customisations';
 
 const containerStyles = css`
   position: relative;
@@ -258,11 +259,9 @@ function DiscoverDocumentsComponent({
     [dataView, onAddColumn, onAddFilter, onRemoveColumn, query, savedSearch.id, setExpandedDoc]
   );
 
-  const {
-    customCellRenderer: externalCustomRenderers,
-    customGridColumnsConfiguration,
-    customControlColumnsConfiguration,
-  } = useDiscoverCustomization('data_table') || {};
+  const { customControlColumnsConfiguration } = useDiscoverCustomization('data_table') || {};
+  const { customCellRenderer, customGridColumnsConfiguration } =
+    useContextualGridCustomisations() || {};
 
   const documents = useObservable(stateContainer.dataState.data$.documents$);
 
@@ -432,7 +431,7 @@ function DiscoverDocumentsComponent({
                   totalHits={totalHits}
                   onFetchMoreRecords={onFetchMoreRecords}
                   componentsTourSteps={TOUR_STEPS}
-                  externalCustomRenderers={externalCustomRenderers}
+                  externalCustomRenderers={customCellRenderer}
                   customGridColumnsConfiguration={customGridColumnsConfiguration}
                   customControlColumnsConfiguration={customControlColumnsConfiguration}
                 />

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx

@@ -37,7 +37,6 @@ import { buildDataTableRecord } from '@kbn/discover-utils';
 import type { DataTableRecord } from '@kbn/discover-utils/types';
 import type { DiscoverCustomizationId } from '../../../../customizations/customization_service';
 import { FieldListCustomization, SearchBarCustomization } from '../../../../customizations';
-import { DataViewField } from '@kbn/data-views-plugin/common';
 
 const mockSearchBarCustomization: SearchBarCustomization = {
   id: 'search_bar',
@@ -46,22 +45,9 @@ const mockSearchBarCustomization: SearchBarCustomization = {
     .mockName('CustomDataViewPickerMock'),
 };
 
-const smartFields = [
-  new DataViewField({
-    name: 'mock_field',
-    type: 'mock_field',
-    searchable: false,
-    aggregatable: false,
-  }),
-];
-
-const additionalFieldGroups = {
-  smartFields,
-};
-
 const mockFieldListCustomisation: FieldListCustomization = {
   id: 'field_list',
-  additionalFieldGroups,
+  logsFieldsEnabled: true,
 };
 
 let mockUseCustomizations = false;
@@ -786,7 +772,7 @@ describe('discover responsive sidebar', function () {
 
       const smartFieldsCount = findTestSubject(comp, 'fieldListGroupedSmartFields-count');
 
-      expect(smartFieldsCount.text()).toBe('1');
+      expect(smartFieldsCount.text()).toBe('2');
     });
   });
 });

src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx

@@ -38,6 +38,7 @@ import {
   DiscoverSidebarReducerStatus,
 } from './lib/sidebar_reducer';
 import { useDiscoverCustomization } from '../../../../customizations';
+import { useAdditionalFieldGroups } from '../../hooks/sidebar/use_additional_field_groups';
 
 const EMPTY_FIELD_COUNTS = {};
 
@@ -331,7 +332,7 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
   );
 
   const searchBarCustomization = useDiscoverCustomization('search_bar');
-  const fieldListCustomization = useDiscoverCustomization('field_list');
+  const additionalFieldGroups = useAdditionalFieldGroups();
   const CustomDataViewPicker = searchBarCustomization?.CustomDataViewPicker;
 
   const createField = unifiedFieldListSidebarContainerApi?.createField;
@@ -408,7 +409,7 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
             onAddFilter={onAddFilter}
             onFieldEdited={onFieldEdited}
             prependInFlyout={prependDataViewPickerForMobile}
-            additionalFieldGroups={fieldListCustomization?.additionalFieldGroups}
+            additionalFieldGroups={additionalFieldGroups}
           />
         ) : null}
       </EuiFlexItem>

src/plugins/discover/public/application/main/hooks/grid_customisations/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { useMemo } from 'react';
+import { useDiscoverServices } from '../../../../hooks/use_discover_services';
+import { useDiscoverCustomization } from '../../../../customizations';
+import { getLogsVirtualColumnsConfiguration } from './logs';
+
+export * from './logs';
+
+export const useContextualGridCustomisations = () => {
+  const { data } = useDiscoverServices();
+  // TODO / NOTE: This will eventually rely on Discover's context resolution to determine which fields
+  // are returned based on the data type.
+  const isLogsContext = useDiscoverCustomization('data_table')?.logsEnabled;
+
+  const virtualColumnsConfiguration = useMemo(() => {
+    if (!isLogsContext) return null;
+    if (isLogsContext) return getLogsVirtualColumnsConfiguration(data);
+  }, [data, isLogsContext]);
+
+  return virtualColumnsConfiguration;
+};

src/plugins/discover/public/application/main/hooks/grid_customisations/logs.tsx

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { DataPublicPluginStart } from '@kbn/data-plugin/public';
+import { CONTENT_FIELD, RESOURCE_FIELD } from '../../../../../common/data_types/logs/constants';
+import { renderCell } from '../../../../components/discover_grid/virtual_columns/logs/cell_renderer';
+import { renderColumn } from '../../../../components/discover_grid/virtual_columns/logs/column';
+
+export const getLogsVirtualColumnsConfiguration = (data: DataPublicPluginStart) => {
+  return {
+    customCellRenderer: createCustomCellRenderer({ data }),
+    customGridColumnsConfiguration: createCustomGridColumnsConfiguration(),
+  };
+};
+
+export const createCustomCellRenderer = ({ data }: { data: DataPublicPluginStart }) => {
+  return {
+    [CONTENT_FIELD]: renderCell(CONTENT_FIELD, { data }),
+    [RESOURCE_FIELD]: renderCell(RESOURCE_FIELD, { data }),
+  };
+};
+
+export const createCustomGridColumnsConfiguration = () => ({
+  [CONTENT_FIELD]: renderColumn(CONTENT_FIELD),
+  [RESOURCE_FIELD]: renderColumn(RESOURCE_FIELD),
+});

src/plugins/discover/public/application/main/hooks/grid_customisations/use_virtual_column_services.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import createContainer from 'constate';
+import type { DataView } from '@kbn/data-views-plugin/common';
+import { DataPublicPluginStart } from '@kbn/data-plugin/public';
+
+export interface UseVirtualColumnServices {
+  services: {
+    data: DataPublicPluginStart;
+    dataView: DataView;
+  };
+}
+
+const useVirtualColumns = ({ services }: UseVirtualColumnServices) => services;
+
+export const [VirtualColumnServiceProvider, useVirtualColumnServiceContext] =
+  createContainer(useVirtualColumns);