[Security Solution] [Sourcerer] [Feature Branch] Update to use Kibana Data Views

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -36,12 +36,10 @@ interface FieldSubType {
 export class IndexPatternsFetcher {
   private elasticsearchClient: ElasticsearchClient;
   private allowNoIndices: boolean;
-
   constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) {
     this.elasticsearchClient = elasticsearchClient;
     this.allowNoIndices = allowNoIndices;
   }
-
   /**
    *  Get a list of field objects for an index pattern that may contain wildcards
    *
@@ -60,23 +58,16 @@ export class IndexPatternsFetcher {
   }): Promise<FieldDescriptor[]> {
     const { pattern, metaFields, fieldCapsOptions, type, rollupIndex } = options;
     const patternList = Array.isArray(pattern) ? pattern : pattern.split(',');
-    let patternListActive: string[] = patternList;
-    // if only one pattern, don't bother with validation. We let getFieldCapabilities fail if the single pattern is bad regardless
-    if (patternList.length > 1) {
-      patternListActive = await this.validatePatternListActive(patternList);
-    }
     const fieldCapsResponse = await getFieldCapabilities(
       this.elasticsearchClient,
-      // if none of the patterns are active, pass the original list to get an error
-      patternListActive.length > 0 ? patternListActive : patternList,
+      patternList,
       metaFields,
       {
         allow_no_indices: fieldCapsOptions
           ? fieldCapsOptions.allow_no_indices
           : this.allowNoIndices,
       }
     );
-
     if (type === 'rollup' && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
       const rollupIndexCapabilities = getCapabilitiesForRollupIndices(
@@ -87,13 +78,11 @@ export class IndexPatternsFetcher {
         ).body
       )[rollupIndex].aggs;
       const fieldCapsResponseObj = keyBy(fieldCapsResponse, 'name');
-
       // Keep meta fields
       metaFields!.forEach(
         (field: string) =>
           fieldCapsResponseObj[field] && rollupFields.push(fieldCapsResponseObj[field])
       );
-
       return mergeCapabilitiesWithFields(
         rollupIndexCapabilities,
         fieldCapsResponseObj,
@@ -127,34 +116,4 @@ export class IndexPatternsFetcher {
     }
     return await getFieldCapabilities(this.elasticsearchClient, indices, metaFields);
   }
-
-  /**
-   *  Returns an index pattern list of only those index pattern strings in the given list that return indices
-   *
-   *  @param patternList string[]
-   *  @return {Promise<string[]>}
-   */
-  async validatePatternListActive(patternList: string[]) {
-    const result = await Promise.all(
-      patternList
-        .map((pattern) =>
-          this.elasticsearchClient.count({
-            index: pattern,
-          })
-        )
-        .map((p) =>
-          p.catch((e) => {
-            if (e.body.error.type === 'index_not_found_exception') {
-              return { body: { count: 0 } };
-            }
-            throw e;
-          })
-        )
-    );
-    return result.reduce(
-      (acc: string[], { body: { count } }, patternListIndex) =>
-        count > 0 ? [...acc, patternList[patternListIndex]] : acc,
-      []
-    );
-  }
 }

x-pack/plugins/observability/public/pages/alerts/alerts_table_t_grid.tsx

@@ -412,6 +412,8 @@ export function AlertsTableTGrid(props: AlertsTableTGridProps) {
       },
       renderCellValue: getRenderCellValue({ setFlyoutAlert }),
       rowRenderers: NO_ROW_RENDER,
+      // TODO: implement Kibana data view runtime fields in observability
+      runtimeMappings: {},
       start: rangeFrom,
       setRefetch,
       sort: [

x-pack/plugins/security_solution/common/constants.ts

@@ -8,6 +8,8 @@
 import type { TransformConfigSchema } from './transforms/types';
 import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
 import { METADATA_TRANSFORMS_PATTERN } from './endpoint/constants';
+import { SavedObjectReference } from '../../../../src/core/types';
+import { DATA_VIEW_SAVED_OBJECT_TYPE } from '../../../../src/plugins/data/common';
 
 export const APP_ID = 'securitySolution';
 export const CASES_FEATURE_ID = 'securitySolutionCases';
@@ -23,6 +25,8 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz';
 export const DEFAULT_DARK_MODE = 'theme:darkMode';
 export const DEFAULT_INDEX_KEY = 'securitySolution:defaultIndex';
 export const DEFAULT_NUMBER_FORMAT = 'format:number:defaultPattern';
+export const DEFAULT_DATA_VIEW_ID = 'security-solution';
+export const DEFAULT_TIME_FIELD = '@timestamp';
 export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults';
 export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults';
 export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults';
@@ -37,7 +41,6 @@ export const DEFAULT_MAX_SIGNALS = 100;
 export const DEFAULT_SEARCH_AFTER_PAGE_SIZE = 100;
 export const DEFAULT_ANOMALY_SCORE = 'securitySolution:defaultAnomalyScore';
 export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000;
-export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled';
 export const DEFAULT_FROM = 'now/d';
 export const DEFAULT_TO = 'now/d';
 export const DEFAULT_INTERVAL_PAUSE = true;
@@ -146,6 +149,12 @@ export const DEFAULT_INDEX_PATTERN = [
   'winlogbeat-*',
 ];
 
+export const defaultDataViewRef: SavedObjectReference = {
+  id: DEFAULT_DATA_VIEW_ID,
+  name: 'dataViewId',
+  type: DATA_VIEW_SAVED_OBJECT_TYPE,
+};
+
 export const DEFAULT_INDEX_PATTERN_EXPERIMENTAL = [
   // TODO: Steph/ueba TEMP for testing UEBA data
   'ml_host_risk_score_*',
@@ -258,6 +267,7 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged`;
 
 export const NOTE_URL = '/api/note';
 export const PINNED_EVENT_URL = '/api/pinned_event';
+export const SOURCERER_API_URL = '/api/sourcerer';
 
 /**
  * Default signals index key for kibana.dev.yml

x-pack/plugins/security_solution/common/search_strategy/index_fields/index.ts

@@ -5,79 +5,15 @@
  * 2.0.
  */
 
-import type { IFieldSubType } from '@kbn/es-query';
-
-import type {
-  IEsSearchRequest,
-  IEsSearchResponse,
-  IIndexPattern,
-} from '../../../../../../src/plugins/data/common';
-import type { DocValueFields, Maybe } from '../common';
-
-interface FieldInfo {
-  category: string;
-  description?: string;
-  example?: string | number;
-  format?: string;
-  name: string;
-  type?: string;
-}
-
-export interface IndexField {
-  /** Where the field belong */
-  category: string;
-  /** Example of field's value */
-  example?: Maybe<string | number>;
-  /** whether the field's belong to an alias index */
-  indexes: Array<Maybe<string>>;
-  /** The name of the field */
-  name: string;
-  /** The type of the field's values as recognized by Kibana */
-  type: string;
-  /** Whether the field's values can be efficiently searched for */
-  searchable: boolean;
-  /** Whether the field's values can be aggregated */
-  aggregatable: boolean;
-  /** Description of the field */
-  description?: Maybe<string>;
-  format?: Maybe<string>;
-  /** the elastic type as mapped in the index */
-  esTypes?: string[];
-  subType?: IFieldSubType;
-  readFromDocValues: boolean;
-}
-
-export type BeatFields = Record<string, FieldInfo>;
-
-export interface IndexFieldsStrategyRequest extends IEsSearchRequest {
-  indices: string[];
-  onlyCheckIfIndicesExist: boolean;
-}
-
-export interface IndexFieldsStrategyResponse extends IEsSearchResponse {
-  indexFields: IndexField[];
-  indicesExist: string[];
-}
-
-export interface BrowserField {
-  aggregatable: boolean;
-  category: string;
-  description: string | null;
-  example: string | number | null;
-  fields: Readonly<Record<string, Partial<BrowserField>>>;
-  format: string;
-  indexes: string[];
-  name: string;
-  searchable: boolean;
-  type: string;
-  subType?: IFieldSubType;
-}
-
-export type BrowserFields = Readonly<Record<string, Partial<BrowserField>>>;
-
-export const EMPTY_BROWSER_FIELDS = {};
-export const EMPTY_DOCVALUE_FIELD: DocValueFields[] = [];
-export const EMPTY_INDEX_PATTERN: IIndexPattern = {
-  fields: [],
-  title: '',
-};
+export {
+  FieldInfo,
+  IndexField,
+  BeatFields,
+  IndexFieldsStrategyRequest,
+  IndexFieldsStrategyResponse,
+  BrowserField,
+  BrowserFields,
+  EMPTY_BROWSER_FIELDS,
+  EMPTY_DOCVALUE_FIELD,
+  EMPTY_INDEX_PATTERN,
+} from '../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/events/last_event_time/index.ts

@@ -10,6 +10,7 @@ export { LastEventIndexKey } from '../../../../../../timelines/common';
 export type {
   LastTimeDetails,
   TimelineEventsLastEventTimeStrategyResponse,
+  TimelineKpiStrategyRequest,
   TimelineKpiStrategyResponse,
   TimelineEventsLastEventTimeRequestOptions,
 } from '../../../../../../timelines/common';

x-pack/plugins/security_solution/common/search_strategy/timeline/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/api/types';
 import { IEsSearchRequest } from '../../../../../../src/plugins/data/common';
 import { ESQuery } from '../../typed_json';
 import {
@@ -41,6 +42,7 @@ export interface TimelineRequestBasicOptions extends IEsSearchRequest {
   defaultIndex: string[];
   docValueFields?: DocValueFields[];
   factoryQueryType?: TimelineFactoryQueryTypes;
+  runtimeMappings: MappingRuntimeFields;
 }
 
 export interface TimelineRequestSortField<Field = string> extends SortField<Field> {
@@ -171,6 +173,7 @@ export interface SortTimelineInput {
 export interface TimelineInput {
   columns?: Maybe<ColumnHeaderInput[]>;
   dataProviders?: Maybe<DataProviderInput[]>;
+  dataViewId?: Maybe<string>;
   description?: Maybe<string>;
   eqlOptions?: Maybe<EqlOptionsInput>;
   eventType?: Maybe<string>;

x-pack/plugins/security_solution/common/types/timeline/index.ts

@@ -272,6 +272,7 @@ export type TimelineTypeLiteralWithNull = runtimeTypes.TypeOf<typeof TimelineTyp
 export const SavedTimelineRuntimeType = runtimeTypes.partial({
   columns: unionWithNullType(runtimeTypes.array(SavedColumnHeaderRuntimeType)),
   dataProviders: unionWithNullType(runtimeTypes.array(SavedDataProviderRuntimeType)),
+  dataViewId: unionWithNullType(runtimeTypes.string),
   description: unionWithNullType(runtimeTypes.string),
   eqlOptions: unionWithNullType(EqlOptionsRuntimeType),
   eventType: unionWithNullType(runtimeTypes.string),
@@ -719,6 +720,7 @@ export interface TimelineResult {
   created?: Maybe<number>;
   createdBy?: Maybe<string>;
   dataProviders?: Maybe<DataProviderResult[]>;
+  dataViewId?: Maybe<string>;
   dateRange?: Maybe<DateRangePickerResult>;
   description?: Maybe<string>;
   eqlOptions?: Maybe<EqlOptionsResult>;

x-pack/plugins/security_solution/common/types/timeline/store.ts

@@ -38,19 +38,20 @@ export interface SortColumnTimeline {
 }
 
 export interface TimelinePersistInput {
-  id: string;
+  columns: ColumnHeaderOptions[];
   dataProviders?: DataProvider[];
+  dataViewId: string;
   dateRange?: {
     start: string;
     end: string;
   };
+  defaultColumns?: ColumnHeaderOptions[];
   excludedRowRendererIds?: RowRendererId[];
   expandedDetail?: TimelineExpandedDetail;
   filters?: Filter[];
-  columns: ColumnHeaderOptions[];
-  defaultColumns?: ColumnHeaderOptions[];
-  itemsPerPage?: number;
+  id: string;
   indexNames: string[];
+  itemsPerPage?: number;
   kqlQuery?: {
     filterQuery: SerializedFilterQuery | null;
   };

x-pack/plugins/security_solution/cypress/cypress.json

@@ -12,5 +12,10 @@
   "video": false,
   "videosFolder": "../../../target/kibana-security-solution/cypress/videos",
   "viewportHeight": 900,
-  "viewportWidth": 1440
+  "viewportWidth": 1440,
+  "env": {
+    "protocol": "http",
+    "hostname": "localhost",
+    "configport": "5601"
+  }
 }

x-pack/plugins/security_solution/cypress/downloads/timelines_export.ndjson

@@ -0,0 +1 @@
+{"savedObjectId":"46cca0e0-2580-11ec-8e56-9dafa0b0343b","version":"WzIyNjIzNCwxXQ==","columns":[{"id":"@timestamp"},{"id":"user.name"},{"id":"event.category"},{"id":"event.action"},{"id":"host.name"}],"kqlMode":"filter","kqlQuery":{"filterQuery":{"kuery":{"expression":"host.name: *","kind":"kuery"}}},"dateRange":{"start":"1514809376000","end":"1577881376000"},"description":"This is the best timeline","title":"Security Timeline","created":1633399341550,"createdBy":"elastic","updated":1633399341550,"updatedBy":"elastic","savedQueryId":null,"dataViewId":null,"timelineType":"default","sort":[],"eventNotes":[],"globalNotes":[],"pinnedEventIds":[]}