[Security Solution][Detection Engine] move lists to data stream

packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx

@@ -211,6 +211,7 @@ describe('AutocompleteFieldListsComponent', () => {
 
     await waitFor(() => {
       expect(mockOnChange).toHaveBeenCalledWith({
+        '@timestamp': DATE_NOW,
         _version: undefined,
         created_at: DATE_NOW,
         created_by: 'some user',

packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts

@@ -42,6 +42,7 @@ export const NAME = 'some name';
 // TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715
 // import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

packages/kbn-securitysolution-es-utils/index.ts

@@ -8,8 +8,10 @@
 
 export * from './src/bad_request_error';
 export * from './src/create_boostrap_index';
+export * from './src/create_data_stream';
 export * from './src/decode_version';
 export * from './src/delete_all_index';
+export * from './src/delete_data_stream';
 export * from './src/delete_index_template';
 export * from './src/delete_policy';
 export * from './src/delete_template';
@@ -18,11 +20,15 @@ export * from './src/get_bootstrap_index_exists';
 export * from './src/get_index_aliases';
 export * from './src/get_index_count';
 export * from './src/get_index_exists';
+export * from './src/get_data_stream_exists';
 export * from './src/get_index_template_exists';
 export * from './src/get_policy_exists';
 export * from './src/get_template_exists';
+export * from './src/migrate_to_data_stream';
 export * from './src/read_index';
 export * from './src/read_privileges';
+export * from './src/put_mappings';
+export * from './src/remove_policy_from_index';
 export * from './src/set_index_template';
 export * from './src/set_policy';
 export * from './src/set_template';

packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * creates data stream
+ * @param esClient
+ * @param name
+ */
+export const createDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return esClient.indices.createDataStream({
+    name,
+  });
+};

packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * deletes data stream
+ * @param esClient
+ * @param name
+ */
+export const deleteDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<boolean> => {
+  return (
+    await esClient.indices.deleteDataStream(
+      {
+        name,
+      },
+      { meta: true }
+    )
+  ).body.acknowledged;
+};

packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * checks if data stream exists
+ * @param esClient
+ * @param name
+ */
+export const getDataStreamExists = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<boolean> => {
+  try {
+    const body = await esClient.indices.getDataStream({ name, expand_wildcards: 'all' });
+    return body.data_streams.length > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else if (
+      // if index already created, _data_stream/${name} request will produce the following error
+      // data stream does not exist at this point, so we can return false
+      err?.body?.error?.reason?.includes(
+        `The provided expression [${name}] matches an alias, specify the corresponding concrete indices instead.`
+      )
+    ) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};

packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * migrate to data stream
+ * @param esClient
+ * @param name
+ */
+export const migrateToDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return esClient.indices.migrateToDataStream({
+    name,
+  });
+};

packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * update mappings of index
+ * @param esClient
+ * @param index
+ * @param mappings
+ */
+export const putMappings = async (
+  esClient: ElasticsearchClient,
+  index: string,
+  mappings: Record<string, MappingProperty>
+): Promise<unknown> => {
+  return await esClient.indices.putMapping({
+    index,
+    properties: mappings,
+  });
+};

packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+export const removePolicyFromIndex = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<unknown> => {
+  return (await esClient.ilm.removePolicy({ index }, { meta: true })).body;
+};

packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts

@@ -56,6 +56,7 @@ export * from './sort_field';
 export * from './sort_order';
 export * from './tags';
 export * from './tie_breaker_id';
+export * from './timestamp';
 export * from './total';
 export * from './type';
 export * from './underscore_version';

packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts

@@ -12,3 +12,6 @@ export const meta = t.object;
 export type Meta = t.TypeOf<typeof meta>;
 export const metaOrUndefined = t.union([meta, t.undefined]);
 export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
+
+export const nullableMetaOrUndefined = t.union([metaOrUndefined, t.null]);
+export type NullableMetaOrUndefined = t.TypeOf<typeof nullableMetaOrUndefined>;

packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+import { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+
+export const timestamp = IsoDateString;
+export const timestampOrUndefined = t.union([IsoDateString, t.undefined]);

packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts

@@ -21,6 +21,7 @@ import {
 
 export const getListItemResponseMock = (): ListItemSchema => ({
   _version: undefined,
+  '@timestamp': DATE_NOW,
   created_at: DATE_NOW,
   created_by: USER,
   deserializer: undefined,

packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts

@@ -11,6 +11,7 @@ import * as t from 'io-ts';
 import { _versionOrUndefined } from '../../common/underscore_version';
 import { deserializerOrUndefined } from '../../common/deserializer';
 import { metaOrUndefined } from '../../common/meta';
+import { timestampOrUndefined } from '../../common/timestamp';
 import { serializerOrUndefined } from '../../common/serializer';
 import { created_at } from '../../common/created_at';
 import { created_by } from '../../common/created_by';
@@ -25,6 +26,7 @@ import { value } from '../../common/value';
 export const listItemSchema = t.exact(
   t.type({
     _version: _versionOrUndefined,
+    '@timestamp': timestampOrUndefined,
     created_at,
     created_by,
     deserializer: deserializerOrUndefined,

packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts

@@ -23,6 +23,7 @@ import {
 
 export const getListResponseMock = (): ListSchema => ({
   _version: undefined,
+  '@timestamp': DATE_NOW,
   created_at: DATE_NOW,
   created_by: USER,
   description: DESCRIPTION,

packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts

@@ -13,6 +13,7 @@ import { deserializerOrUndefined } from '../../common/deserializer';
 import { metaOrUndefined } from '../../common/meta';
 import { serializerOrUndefined } from '../../common/serializer';
 import { created_at } from '../../common/created_at';
+import { timestampOrUndefined } from '../../common/timestamp';
 import { created_by } from '../../common/created_by';
 import { description } from '../../common/description';
 import { id } from '../../common/id';
@@ -26,6 +27,7 @@ import { updated_by } from '../../common/updated_by';
 export const listSchema = t.exact(
   t.type({
     _version: _versionOrUndefined,
+    '@timestamp': timestampOrUndefined,
     created_at,
     created_by,
     description,

packages/kbn-securitysolution-list-api/src/api/index.ts

@@ -37,7 +37,7 @@ import {
 
 import {
   ENDPOINT_LIST_URL,
-  EXCEPTION_FILTER,
+  INTERNAL_EXCEPTION_FILTER,
   EXCEPTION_LIST_ITEM_URL,
   EXCEPTION_LIST_URL,
 } from '@kbn/securitysolution-list-constants';
@@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({
   http,
   signal,
 }: GetExceptionFilterFromExceptionListIdsProps): Promise<ExceptionFilterResponse> =>
-  http.fetch(EXCEPTION_FILTER, {
+  http.fetch(INTERNAL_EXCEPTION_FILTER, {
     method: 'POST',
     body: JSON.stringify({
       exception_list_ids: exceptionListIds,
@@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({
   chunkSize,
   signal,
 }: GetExceptionFilterFromExceptionsProps): Promise<ExceptionFilterResponse> =>
-  http.fetch(EXCEPTION_FILTER, {
+  http.fetch(INTERNAL_EXCEPTION_FILTER, {
     method: 'POST',
     body: JSON.stringify({
       exceptions,

packages/kbn-securitysolution-list-api/src/list_api/index.ts

@@ -37,7 +37,7 @@ import {
   LIST_ITEM_URL,
   LIST_PRIVILEGES_URL,
   LIST_URL,
-  FIND_LISTS_BY_SIZE,
+  INTERNAL_FIND_LISTS_BY_SIZE,
 } from '@kbn/securitysolution-list-constants';
 import { toError, toPromise } from '../fp_utils';
 
@@ -115,7 +115,7 @@ const findListsBySize = async ({
   per_page,
   signal,
 }: ApiParams & FindListSchemaEncoded): Promise<FoundListsBySizeSchema> => {
-  return http.fetch(`${FIND_LISTS_BY_SIZE}`, {
+  return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, {
     method: 'GET',
     query: {
       cursor,

packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts

@@ -23,6 +23,7 @@ import {
 } from '../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

packages/kbn-securitysolution-list-constants/index.ts

@@ -20,8 +20,8 @@ export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`;
  * Internal value list routes
  */
 export const INTERNAL_LIST_URL = '/internal/lists';
-export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
-export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
+export const INTERNAL_FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
+export const INTERNAL_EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
 
 /**
  * Exception list routes

packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts

@@ -23,6 +23,7 @@ import {
 } from '../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts

@@ -20,6 +20,7 @@ import {
 } from '../../constants.mock';
 
 export const getListItemResponseMock = (): ListItemSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts

@@ -22,6 +22,7 @@ import {
 } from '../../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,