[Security Solution] [Sourcerer] [Feature Branch] Update to use Kibana Data Views

src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -5,7 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
 import { IndexPatternsFetcher } from '.';
 import { ElasticsearchClient } from 'kibana/server';
 import * as indexNotFoundException from './index_not_found_exception.json';
@@ -15,36 +14,36 @@ describe('Index Pattern Fetcher - server', () => {
   let esClient: ElasticsearchClient;
   const emptyResponse = {
     body: {
-      count: 0,
+      indices: [],
     },
   };
   const response = {
     body: {
-      count: 1115,
+      indices: ['b'],
+      fields: [{ name: 'foo' }, { name: 'bar' }, { name: 'baz' }],
     },
   };
   const patternList = ['a', 'b', 'c'];
   beforeEach(() => {
+    jest.clearAllMocks();
     esClient = {
-      count: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValueOnce(emptyResponse).mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
   });
-
   it('Removes pattern without matching indices', async () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(['b', 'c']);
   });
-
   it('Returns all patterns when all match indices', async () => {
     esClient = {
-      count: jest.fn().mockResolvedValue(response),
+      fieldCaps: jest.fn().mockResolvedValue(response),
     } as unknown as ElasticsearchClient;
     indexPatterns = new IndexPatternsFetcher(esClient);
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual(patternList);
   });
-  it('Removes pattern when "index_not_found_exception" error is thrown', async () => {
+  it('Removes pattern when error is thrown', async () => {
     class ServerError extends Error {
       public body?: Record<string, any>;
       constructor(
@@ -56,9 +55,8 @@ describe('Index Pattern Fetcher - server', () => {
         this.body = errBody;
       }
     }
-
     esClient = {
-      count: jest
+      fieldCaps: jest
         .fn()
         .mockResolvedValueOnce(response)
         .mockRejectedValue(
@@ -69,4 +67,22 @@ describe('Index Pattern Fetcher - server', () => {
     const result = await indexPatterns.validatePatternListActive(patternList);
     expect(result).toEqual([patternList[0]]);
   });
+  it('When allowNoIndices is false, run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(4);
+  });
+  it('When allowNoIndices is true, do not run validatePatternListActive', async () => {
+    const fieldCapsMock = jest.fn();
+    esClient = {
+      fieldCaps: fieldCapsMock.mockResolvedValue(response),
+    } as unknown as ElasticsearchClient;
+    indexPatterns = new IndexPatternsFetcher(esClient, true);
+    await indexPatterns.getFieldsForWildcard({ pattern: patternList });
+    expect(fieldCapsMock.mock.calls).toHaveLength(1);
+  });
 });

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -36,12 +36,10 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
   private elasticsearchClient: ElasticsearchClient;
   private allowNoIndices: boolean;
-
   constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
     this.elasticsearchClient = elasticsearchClient;
     this.allowNoIndices = allowNoIndices;
   }
-
   /**
    *  Get a list of field objects for an index pattern that may contain wildcards
    *
@@ -60,23 +58,22 @@ export class IndexPatternsFetcher {
   }): Promise<FieldDescriptor[]> {
     const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
     const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
+    const allowNoIndices = fieldCapsOptions
+      ? fieldCapsOptions.allow_no_indices
+      : this.allowNoIndices;
     let patternListActive: string[] = patternList;
     // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
-    if (patternList.length > 1) {
+    if (patternList.length > 1 && !allowNoIndices) {
       patternListActive = await this.validatePatternListActive(patternList);
     }
     const fieldCapsResponse = await getFieldCapabilities(
       this.elasticsearchClient,
-      // if none of the patterns are active, pass the original list to get an error
-      patternListActive.length > 0 ? patternListActive : patternList,
+      patternListActive,
       metaFields,
       {
-        allow_no_indices: fieldCapsOptions
-          ? fieldCapsOptions.allow_no_indices
-          : this.allowNoIndices,
+        allow_no_indices: allowNoIndices,
       }
     );
-
     if (type === 'rollup' && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
       const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -87,13 +84,11 @@ export class IndexPatternsFetcher {
         ).body
       )[rollupIndex].aggs;
       const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
-
       // Keep meta fields
       metaFields!.forEach(
         (field: string) =>
           fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
       );
-
       return mergeCapabilitiesWithFields(
         rollupIndexCapabilities,
         fieldCapsResponseObj,
@@ -137,23 +132,20 @@ export class IndexPatternsFetcher {
   async validatePatternListActive(patternList: string[]) {
     const result = await Promise.all(
       patternList
-        .map((pattern) =>
-          this.elasticsearchClient.count({
-            index: pattern,
-          })
-        )
-        .map((p) =>
-          p.catch((e) => {
-            if (e.body.error.type === 'index_not_found_exception') {
-              return { body: { count: 0 } };
-            }
-            throw e;
-          })
-        )
+        .map(async (index) => {
+          const searchResponse = await this.elasticsearchClient.fieldCaps({
+            index,
+            fields: '_id',
+            ignore_unavailable: true,
+            allow_no_indices: false,
+          });
+          return searchResponse.body.indices.length > 0;
+        })
+        .map((p) => p.catch(() => false))
     );
     return result.reduce(
-      (acc: string[], { body: { count } }, patternListIndex) =>
-        count > 0 ? [...acc, patternList[patternListIndex]] : acc,
+      (acc: string[], isValid, patternListIndex) =>
+        isValid ? [...acc, patternList[patternListIndex]] : acc,
       []
     );
   }

x-pack/plugins/observability/public/pages/alerts/alerts_table_t_grid.tsx

@@ -412,6 +412,8 @@ export function AlertsTableTGrid(props: AlertsTableTGridProps) {
       },
       renderCellValue: getRenderCellValue({ setFlyoutAlert }),
       rowRenderers: NO_ROW_RENDER,
+      // TODO: implement Kibana data view runtime fields in observability
+      runtimeMappings: {},
       start: rangeFrom,
       setRefetch,
       sort: [

x-pack/plugins/security_solution/common/constants.ts

@@ -10,9 +10,19 @@ import { ENABLE_ITOM } from '../../actions/server/constants/connectors';
 import type { TransformConfigSchema } from './transforms/types';
 import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
 import { METADATA_TRANSFORMS_PATTERN } from './endpoint/constants';
+import { SavedObjectReference } from '../../../../src/core/types';
+import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../../../src/plugins/data/common';
+
+/**
+ * as const
+ *
+ * The const assertion ensures that type widening does not occur
+ * https://mariusschulz.com/blog/literal-type-widening-in-typescript
+ * Please follow this convention when adding to this file
+ */
 
 export const APP_ID = 'securitySolution' as const;
-export const APP_UI_ID = 'securitySolutionUI';
+export const APP_UI_ID = 'securitySolutionUI' as const;
 export const CASES_FEATURE_ID = 'securitySolutionCases' as const;
 export const SERVER_APP_ID = 'siem' as const;
 export const APP_NAME = 'Security' as const;
@@ -26,6 +36,8 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const;
 export const DEFAULT_DARK_MODE = 'theme:darkMode' as const;
 export const DEFAULT_INDEX_KEY = 'securitySolution:defaultIndex' as const;
 export const DEFAULT_NUMBER_FORMAT = 'format:number:defaultPattern' as const;
+export const DEFAULT_DATA_VIEW_ID = 'security-solution' as const;
+export const DEFAULT_TIME_FIELD = '@timestamp' as const;
 export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults' as const;
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults' as const;
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const;
@@ -51,7 +63,6 @@ export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges' as const
 export const DEFAULT_TRANSFORMS = 'securitySolution:transforms' as const;
 export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled' as const;
 export const GLOBAL_HEADER_HEIGHT = 96 as const; // px
-export const GLOBAL_HEADER_HEIGHT_WITH_GLOBAL_BANNER = 128 as const; // px
 export const FILTERS_GLOBAL_HEIGHT = 109 as const; // px
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled' as const;
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51' as const;
@@ -152,6 +163,12 @@ export const DEFAULT_INDEX_PATTERN = [
   'winlogbeat-*',
 ];
 
+export const defaultDataViewRef: SavedObjectReference = {
+  id: DEFAULT_DATA_VIEW_ID,
+  name: 'dataViewId',
+  type: DATA_VIEW_SAVED_OBJECT_TYPE,
+};
+
 export const DEFAULT_INDEX_PATTERN_EXPERIMENTAL = [
   // TODO: Steph/ueba TEMP for testing UEBA data
   'ml_host_risk_score_*',
@@ -256,6 +273,7 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged` as const;
 
 export const NOTE_URL = '/api/note' as const;
 export const PINNED_EVENT_URL = '/api/pinned_event' as const;
+export const SOURCERER_API_URL = '/api/sourcerer' as const;
 
 /**
  * Default signals index key for kibana.dev.yml
@@ -343,7 +361,7 @@ export const ELASTIC_NAME = 'estc' as const;
 
 export const METADATA_TRANSFORM_STATS_URL = `/api/transform/transforms/${METADATA_TRANSFORMS_PATTERN}/_stats`;
 
-export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_';
+export const RISKY_HOSTS_INDEX_PREFIX = 'ml_host_risk_score_latest_' as const;
 
 export const TRANSFORM_STATES = {
   ABORTING: 'aborting',

x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts

@@ -5,79 +5,15 @@
  * 2.0.
  */
 
-import type { IFieldSubType } from '@kbn/es-query';
-
-import type {
-  IEsSearchRequest,
-  IEsSearchResponse,
-  IIndexPattern,
-} from '../../../../../../src/plugins/data/common';
-import type { DocValueFields, Maybe } from '../common';
-
-interface FieldInfo {
-  category: string;
-  description?: string;
-  example?: string | number;
-  format?: string;
-  name: string;
-  type?: string;
-}
-
-export interface IndexField {
-  /** Where the field belong */
-  category: string;
-  /** Example of field's value */
-  example?: Maybe<string | number>;
-  /** whether the field's belong to an alias index */
-  indexes: Array<Maybe<string>>;
-  /** The name of the field */
-  name: string;
-  /** The type of the field's values as recognized by Kibana */
-  type: string;
-  /** Whether the field's values can be efficiently searched for */
-  searchable: boolean;
-  /** Whether the field's values can be aggregated */
-  aggregatable: boolean;
-  /** Description of the field */
-  description?: Maybe<string>;
-  format?: Maybe<string>;
-  /** the elastic type as mapped in the index */
-  esTypes?: string[];
-  subType?: IFieldSubType;
-  readFromDocValues: boolean;
-}
-
-export type BeatFields = Record<string, FieldInfo>;
-
-export interface IndexFieldsStrategyRequest extends IEsSearchRequest {
-  indices: string[];
-  onlyCheckIfIndicesExist: boolean;
-}
-
-export interface IndexFieldsStrategyResponse extends IEsSearchResponse {
-  indexFields: IndexField[];
-  indicesExist: string[];
-}
-
-export interface BrowserField {
-  aggregatable: boolean;
-  category: string;
-  description: string | null;
-  example: string | number | null;
-  fields: Readonly<Record<string, Partial<BrowserField>>>;
-  format: string;
-  indexes: string[];
-  name: string;
-  searchable: boolean;
-  type: string;
-  subType?: IFieldSubType;
-}
-
-export type BrowserFields = Readonly<Record<string, Partial<BrowserField>>>;
-
-export const EMPTY_BROWSER_FIELDS = {};
-export const EMPTY_DOCVALUE_FIELD: DocValueFields[] = [];
-export const EMPTY_INDEX_PATTERN: IIndexPattern = {
-  fields: [],
-  title: '',
-};
+export {
+  FieldInfo,
+  IndexField,
+  BeatFields,
+  IndexFieldsStrategyRequest,
+  IndexFieldsStrategyResponse,
+  BrowserField,
+  BrowserFields,
+  EMPTY_BROWSER_FIELDS,
+  EMPTY_DOCVALUE_FIELD,
+  EMPTY_INDEX_PATTERN,
+} from '../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/events/last_event_time/index.ts

@@ -10,6 +10,7 @@ export { LastEventIndexKey } from '../../../../../../timelines/common';
 export type {
   LastTimeDetails,
   TimelineEventsLastEventTimeStrategyResponse,
+  TimelineKpiStrategyRequest,
   TimelineKpiStrategyResponse,
   TimelineEventsLastEventTimeRequestOptions,
 } from '../../../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { IEsSearchRequest } from '../../../../../../src/plugins/data/common';
 import { ESQuery } from '../../typed_json';
 import {
@@ -41,6 +42,7 @@ export interface TimelineRequestBasicOptions extends IEsSearchRequest {
   defaultIndex: string[];
   docValueFields?: DocValueFields[];
   factoryQueryType?: TimelineFactoryQueryTypes;
+  runtimeMappings: MappingRuntimeFields;
 }
 
 export interface TimelineRequestSortField<Field = string> extends SortField<Field> {
@@ -171,6 +173,7 @@ export interface SortTimelineInput {
 export interface TimelineInput {
   columns?: Maybe<ColumnHeaderInput[]>;
   dataProviders?: Maybe<DataProviderInput[]>;
+  dataViewId?: Maybe<string>;
   description?: Maybe<string>;
   eqlOptions?: Maybe<EqlOptionsInput>;
   eventType?: Maybe<string>;

x-pack/plugins/security_solution/common/test/index.ts

@@ -7,12 +7,12 @@
 
 // For the source of these roles please consult the PR these were introduced https://github.com/elastic/kibana/pull/81866#issue-511165754
 export enum ROLES {
+  soc_manager = 'soc_manager',
   reader = 'reader',
   t1_analyst = 't1_analyst',
   t2_analyst = 't2_analyst',
   hunter = 'hunter',
   rule_author = 'rule_author',
-  soc_manager = 'soc_manager',
   platform_engineer = 'platform_engineer',
   detections_admin = 'detections_admin',
 }