From 245ba5769914cf8b978034d1bb36619182ccacab Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 5 May 2023 11:42:05 +0100 Subject: [PATCH 01/41] add test element --- .../components/rules/step_about_rule_details/index.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx index 4d9c23240a9b2..ff916eeccb8f3 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx @@ -94,6 +94,14 @@ const StepAboutRuleToggleDetailsComponent: React.FC = ({ {stepData != null && stepDataDetails != null && ( +
+ TEST +
{toggleOptions.length > 0 && ( Date: Fri, 5 May 2023 12:49:41 +0100 Subject: [PATCH 02/41] Revert "add test element" This reverts commit 245ba5769914cf8b978034d1bb36619182ccacab. --- .../components/rules/step_about_rule_details/index.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx index ff916eeccb8f3..4d9c23240a9b2 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule_details/index.tsx @@ -94,14 +94,6 @@ const StepAboutRuleToggleDetailsComponent: React.FC = ({ {stepData != null && stepDataDetails != null && ( -
- TEST -
{toggleOptions.length > 0 && ( Date: Tue, 25 Jul 2023 17:30:12 +0100 Subject: [PATCH 03/41] [Security Solution][Detection Engine] move lists to data stream --- .../kbn-securitysolution-es-utils/index.ts | 2 + .../src/create_data_stream/index.ts | 24 ++++++++++ .../src/get_data_stream_exists/index.ts | 39 ++++++++++++++++ .../server/routes/create_list_index_route.ts | 22 ++++----- .../server/routes/read_list_index_route.ts | 18 ++++---- .../services/items/get_list_item_template.ts | 3 +- .../services/lists/get_list_template.ts | 3 +- .../server/services/lists/list_client.ts | 46 +++++++++++++++++++ 8 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts index 94aed92969627..65762c55af982 100644 --- a/packages/kbn-securitysolution-es-utils/index.ts +++ b/packages/kbn-securitysolution-es-utils/index.ts @@ -8,6 +8,7 @@ export * from './src/bad_request_error'; export * from './src/create_boostrap_index'; +export * from './src/create_data_stream'; export * from './src/decode_version'; export * from './src/delete_all_index'; export * from './src/delete_index_template'; @@ -18,6 +19,7 @@ export * from './src/get_bootstrap_index_exists'; export * from './src/get_index_aliases'; export * from './src/get_index_count'; export * from './src/get_index_exists'; +export * from './src/get_data_stream_exists'; export * from './src/get_index_template_exists'; export * from './src/get_policy_exists'; export * from './src/get_template_exists'; diff --git a/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts new file mode 100644 index 0000000000000..28b65f5882f94 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * creates data stream + * @param esClient + * @param name + * @returns + */ +export const createDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return esClient.indices.createDataStream({ + name, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts new file mode 100644 index 0000000000000..2496aa285d4a2 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * creates data stream + * @param esClient + * @param name + * @returns + */ +export const getDataStreamExists = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + try { + const body = await esClient.indices.getDataStream({ name, expand_wildcards: 'all' }); + return body.data_streams.length > 0; + } catch (err) { + if (err.body != null && err.body.status === 404) { + return false; + } else if ( + // if index already created, _data_stream/${name} request will produce the following error + // data stream does not exist at this point, so we can return false + err?.body?.error?.reason?.includes( + `The provided expression [${name}] matches an alias, specify the corresponding concrete indices instead.` + ) + ) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts index 5fe903fba39b8..1315b6455c178 100644 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_index_route.ts @@ -30,9 +30,9 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { try { const lists = await getListClient(context); - const listIndexExists = await lists.getListIndexExists(); - const listItemIndexExists = await lists.getListItemIndexExists(); + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); const policyExists = await lists.getListPolicyExists(); const policyListItemExists = await lists.getListItemPolicyExists(); @@ -43,16 +43,16 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { await lists.setListItemPolicy(); } - const templateExists = await lists.getListTemplateExists(); + const templateListExists = await lists.getListTemplateExists(); const templateListItemsExists = await lists.getListItemTemplateExists(); const legacyTemplateExists = await lists.getLegacyListTemplateExists(); const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); - if (!templateExists) { + if (!templateListExists || !listDataStreamExists) { await lists.setListTemplate(); } - if (!templateListItemsExists) { + if (!templateListItemsExists || !listItemDataStreamExists) { await lists.setListItemTemplate(); } @@ -70,17 +70,17 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { } } - if (listIndexExists && listItemIndexExists) { + if (listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ - body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, + body: `data stream: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, statusCode: 409, }); } else { - if (!listIndexExists) { - await lists.createListBootStrapIndex(); + if (!listDataStreamExists) { + await lists.createListDataStream(); } - if (!listItemIndexExists) { - await lists.createListItemBootStrapIndex(); + if (!listItemDataStreamExists) { + await lists.createListItemDataStream(); } const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); diff --git a/x-pack/plugins/lists/server/routes/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/read_list_index_route.ts index 061c8486db358..93c28135dc0ef 100644 --- a/x-pack/plugins/lists/server/routes/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/read_list_index_route.ts @@ -30,12 +30,12 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { try { const lists = await getListClient(context); - const listIndexExists = await lists.getListIndexExists(); - const listItemIndexExists = await lists.getListItemIndexExists(); + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - if (listIndexExists || listItemIndexExists) { + if (listDataStreamExists || listItemDataStreamExists) { const [validated, errors] = validate( - { list_index: listIndexExists, list_item_index: listItemIndexExists }, + { list_index: listDataStreamExists, list_item_index: listItemDataStreamExists }, listItemIndexExistSchema ); if (errors != null) { @@ -43,19 +43,19 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { } else { return response.ok({ body: validated ?? {} }); } - } else if (!listIndexExists && listItemIndexExists) { + } else if (!listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ - body: `index ${lists.getListIndex()} does not exist`, + body: `data stream ${lists.getListIndex()} does not exist`, statusCode: 404, }); - } else if (!listItemIndexExists && listIndexExists) { + } else if (!listItemDataStreamExists && listDataStreamExists) { return siemResponse.error({ - body: `index ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListItemIndex()} does not exist`, statusCode: 404, }); } else { return siemResponse.error({ - body: `index ${lists.getListIndex()} and index ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListIndex()} and data stream ${lists.getListItemIndex()} does not exist`, statusCode: 404, }); } diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts index d42844e3e133a..eb69563af41bf 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts @@ -9,7 +9,8 @@ import listsItemsMappings from './list_item_mappings.json'; export const getListItemTemplate = (index: string): Record => { const template = { - index_patterns: [`${index}-*`], + data_stream: {}, + index_patterns: [index], template: { mappings: listsItemsMappings, settings: { diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.ts index 8b3cfa8d1df6e..34f140b1c5002 100644 --- a/x-pack/plugins/lists/server/services/lists/get_list_template.ts +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.ts @@ -8,7 +8,8 @@ import listMappings from './list_mappings.json'; export const getListTemplate = (index: string): Record => ({ - index_patterns: [`${index}-*`], + data_stream: {}, + index_patterns: [index], template: { mappings: listMappings, settings: { diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index b6dd681c8802c..abdc19e58c9f6 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -8,11 +8,13 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { createBootstrapIndex, + createDataStream, deleteAllIndex, deleteIndexTemplate, deletePolicy, deleteTemplate, getBootstrapIndexExists, + getDataStreamExists, getIndexTemplateExists, getPolicyExists, getTemplateExists, @@ -245,6 +247,7 @@ export class ListClient { /** * True if the list index exists, otherwise false * @returns True if the list index exists, otherwise false + * @deprecated */ public getListIndexExists = async (): Promise => { const { esClient } = this; @@ -252,9 +255,20 @@ export class ListClient { return getBootstrapIndexExists(esClient, listIndex); }; + /** + * True if the list data stream exists, otherwise false + * @returns True if the list data stream exists, otherwise false + */ + public getListDataStreamExists = async (): Promise => { + const { esClient } = this; + const listIndex = this.getListIndex(); + return getDataStreamExists(esClient, listIndex); + }; + /** * True if the list index item exists, otherwise false * @returns True if the list item index exists, otherwise false + * @deprecated */ public getListItemIndexExists = async (): Promise => { const { esClient } = this; @@ -262,9 +276,20 @@ export class ListClient { return getBootstrapIndexExists(esClient, listItemIndex); }; + /** + * True if the list item data stream exists, otherwise false + * @returns True if the list item data stream exists, otherwise false + */ + public getListItemDataStreamExists = async (): Promise => { + const { esClient } = this; + const listItemIndex = this.getListItemIndex(); + return getDataStreamExists(esClient, listItemIndex); + }; + /** * Creates the list boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch + * @deprecated */ public createListBootStrapIndex = async (): Promise => { const { esClient } = this; @@ -272,9 +297,20 @@ export class ListClient { return createBootstrapIndex(esClient, listIndex); }; + /** + * Creates list data stream + * @returns The contents of the create data stream from Elasticsearch + */ + public createListDataStream = async (): Promise => { + const { esClient } = this; + const listIndex = this.getListIndex(); + return createDataStream(esClient, listIndex); + }; + /** * Creates the list item boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch + * @deprecated */ public createListItemBootStrapIndex = async (): Promise => { const { esClient } = this; @@ -282,6 +318,16 @@ export class ListClient { return createBootstrapIndex(esClient, listItemIndex); }; + /** + * Creates list item data stream + * @returns The contents of the create data stream from Elasticsearch + */ + public createListItemDataStream = async (): Promise => { + const { esClient } = this; + const listItemIndex = this.getListItemIndex(); + return createDataStream(esClient, listItemIndex); + }; + /** * Returns true if the list policy for ILM exists, otherwise false * @returns True if the list policy for ILM exists, otherwise false. From b80c1b456616e1e0518cd48fae5c5f2d68d7cad7 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 26 Jul 2023 11:39:05 +0100 Subject: [PATCH 04/41] update apis --- .../src/common/index.ts | 1 + .../src/common/timestamp/index.ts | 13 +++ .../src/response/list_item_schema/index.ts | 2 + .../src/response/list_schema/index.ts | 2 + .../server/routes/import_list_item_route.ts | 3 +- .../index_es_list_item_schema.ts | 2 + .../elastic_query/index_es_list_schema.ts | 2 + .../search_es_list_item_schema.ts | 2 + .../elastic_response/search_es_list_schema.ts | 2 + .../services/items/create_list_items_bulk.ts | 1 + .../items/write_lines_to_bulk_list_items.ts | 100 ++++++++++-------- .../server/services/lists/create_list.ts | 11 +- 12 files changed, 90 insertions(+), 51 deletions(-) create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts index e50500f87f61a..9c7e1a4ad31b5 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts @@ -56,6 +56,7 @@ export * from './sort_field'; export * from './sort_order'; export * from './tags'; export * from './tie_breaker_id'; +export * from './timestamp'; export * from './total'; export * from './type'; export * from './underscore_version'; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts new file mode 100644 index 0000000000000..e4b89c7590530 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as t from 'io-ts'; +import { IsoDateString } from '@kbn/securitysolution-io-ts-types'; + +export const timestamp = IsoDateString; +export const timestampOrUndefined = t.union([IsoDateString, t.undefined]); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts index ae75f3a7741bc..4a45903c82d98 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts @@ -11,6 +11,7 @@ import * as t from 'io-ts'; import { _versionOrUndefined } from '../../common/underscore_version'; import { deserializerOrUndefined } from '../../common/deserializer'; import { metaOrUndefined } from '../../common/meta'; +import { timestampOrUndefined } from '../../common/timestamp'; import { serializerOrUndefined } from '../../common/serializer'; import { created_at } from '../../common/created_at'; import { created_by } from '../../common/created_by'; @@ -25,6 +26,7 @@ import { value } from '../../common/value'; export const listItemSchema = t.exact( t.type({ _version: _versionOrUndefined, + '@timestamp': timestampOrUndefined, created_at, created_by, deserializer: deserializerOrUndefined, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts index 9db686e6de255..a9f076384fdf8 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts @@ -13,6 +13,7 @@ import { deserializerOrUndefined } from '../../common/deserializer'; import { metaOrUndefined } from '../../common/meta'; import { serializerOrUndefined } from '../../common/serializer'; import { created_at } from '../../common/created_at'; +import { timestampOrUndefined } from '../../common/timestamp'; import { created_by } from '../../common/created_by'; import { description } from '../../common/description'; import { id } from '../../common/id'; @@ -26,6 +27,7 @@ import { updated_by } from '../../common/updated_by'; export const listSchema = t.exact( t.type({ _version: _versionOrUndefined, + '@timestamp': timestampOrUndefined, created_at, created_by, description, diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts index 0af6035e40b5d..32987dec766da 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/import_list_item_route.ts @@ -45,7 +45,8 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp const stream = createStreamFromBuffer(request.body); const { deserializer, list_id: listId, serializer, type } = request.query; const lists = await getListClient(context); - const listExists = await lists.getListIndexExists(); + // TODO: migrate?? + const listExists = await lists.getListDataStreamExists(); if (!listExists) { return siemResponse.error({ body: `To import a list item, the index must exist first. Index "${lists.getListIndex()}" does not exist`, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts index b3130b95fe978..d4d94cd2a4bab 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts @@ -14,6 +14,7 @@ import { metaOrUndefined, serializerOrUndefined, tie_breaker_id, + timestamp, updated_at, updated_by, } from '@kbn/securitysolution-io-ts-list-types'; @@ -23,6 +24,7 @@ import { esDataTypeUnion } from '../common/schemas'; export const indexEsListItemSchema = t.intersection([ t.exact( t.type({ + '@timestamp': timestamp, created_at, created_by, deserializer: deserializerOrUndefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts index 85e2eb95dd7e4..f6ca3bf1ecb2f 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts @@ -16,6 +16,7 @@ import { name, serializerOrUndefined, tie_breaker_id, + timestamp, type, updated_at, updated_by, @@ -24,6 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types'; export const indexEsListSchema = t.exact( t.type({ + '@timestamp': timestamp, created_at, created_by, description, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts index 158783ce088b2..9f9a2478827cb 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts @@ -14,6 +14,7 @@ import { metaOrUndefined, serializerOrUndefined, tie_breaker_id, + timestamp, updated_at, updated_by, } from '@kbn/securitysolution-io-ts-list-types'; @@ -46,6 +47,7 @@ import { export const searchEsListItemSchema = t.exact( t.type({ + '@timestamp': timestamp, binary: binaryOrUndefined, boolean: booleanOrUndefined, byte: byteOrUndefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts index 7e2ca2d6343cb..7aec26143865e 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts @@ -16,6 +16,7 @@ import { name, serializerOrUndefined, tie_breaker_id, + timestamp, type, updated_at, updated_by, @@ -24,6 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types'; export const searchEsListSchema = t.exact( t.type({ + '@timestamp': timestamp, created_at, created_by, description, diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts index 0944e916c826a..54965751a058a 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts @@ -60,6 +60,7 @@ export const createListItemsBulk = async ({ }); if (elasticQuery != null) { const elasticBody: IndexEsListItemSchema = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, deserializer, diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts index 9e9b15284cc67..9d18b78757943 100644 --- a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts @@ -54,60 +54,68 @@ export const importListItemsToStream = ({ meta, version, }: ImportListItemsToStreamOptions): Promise => { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const readBuffer = new BufferLines({ bufferSize: config.importBufferSize, input: stream }); let fileName: string | undefined; let list: ListSchema | null = null; readBuffer.on('fileName', async (fileNameEmitted: string) => { - readBuffer.pause(); - fileName = decodeURIComponent(fileNameEmitted); - if (listId == null) { - list = await createListIfItDoesNotExist({ - description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', { - defaultMessage: 'File uploaded from file system of {fileName}', - values: { fileName }, - }), - deserializer, - esClient, - id: fileName, - immutable: false, - listIndex, - meta, - name: fileName, - serializer, - type, - user, - version, - }); + try { + readBuffer.pause(); + fileName = decodeURIComponent(fileNameEmitted); + if (listId == null) { + list = await createListIfItDoesNotExist({ + description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', { + defaultMessage: 'File uploaded from file system of {fileName}', + values: { fileName }, + }), + deserializer, + esClient, + id: fileName, + immutable: false, + listIndex, + meta, + name: fileName, + serializer, + type, + user, + version, + }); + } + readBuffer.resume(); + } catch (err) { + reject(err); } - readBuffer.resume(); }); readBuffer.on('lines', async (lines: string[]) => { - if (listId != null) { - await writeBufferToItems({ - buffer: lines, - deserializer, - esClient, - listId, - listItemIndex, - meta, - serializer, - type, - user, - }); - } else if (fileName != null) { - await writeBufferToItems({ - buffer: lines, - deserializer, - esClient, - listId: fileName, - listItemIndex, - meta, - serializer, - type, - user, - }); + try { + if (listId != null) { + await writeBufferToItems({ + buffer: lines, + deserializer, + esClient, + listId, + listItemIndex, + meta, + serializer, + type, + user, + }); + } else if (fileName != null) { + await writeBufferToItems({ + buffer: lines, + deserializer, + esClient, + listId: fileName, + listItemIndex, + meta, + serializer, + type, + user, + }); + } + } catch (err) { + reject(err); } }); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts index 3a31b7fdf6b9a..75eae4659a535 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -10,7 +10,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Description, DeserializerOrUndefined, - IdOrUndefined, + Id, Immutable, ListSchema, MetaOrUndefined, @@ -24,7 +24,7 @@ import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { IndexEsListSchema } from '../../schemas/elastic_query'; export interface CreateListOptions { - id: IdOrUndefined; + id: Id; deserializer: DeserializerOrUndefined; serializer: SerializerOrUndefined; type: Type; @@ -58,6 +58,7 @@ export const createList = async ({ }: CreateListOptions): Promise => { const createdAt = dateNow ?? new Date().toISOString(); const body: IndexEsListSchema = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, description, @@ -72,12 +73,14 @@ export const createList = async ({ updated_by: user, version, }; - const response = await esClient.index({ - body, + + const response = await esClient.create({ + document: body, id, index: listIndex, refresh: 'wait_for', }); + return { _version: encodeHitVersion(response), id: response._id, From ed9ddeeddaca4c8fd58602e50be99a11775193e9 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 26 Jul 2023 11:39:42 +0100 Subject: [PATCH 05/41] update schemas --- .../schemas/elastic_response/search_es_list_item_schema.ts | 4 ++-- .../server/schemas/elastic_response/search_es_list_schema.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts index 9f9a2478827cb..eed15628c907a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts @@ -14,7 +14,7 @@ import { metaOrUndefined, serializerOrUndefined, tie_breaker_id, - timestamp, + timestampOrUndefined, updated_at, updated_by, } from '@kbn/securitysolution-io-ts-list-types'; @@ -47,7 +47,7 @@ import { export const searchEsListItemSchema = t.exact( t.type({ - '@timestamp': timestamp, + '@timestamp': timestampOrUndefined, binary: binaryOrUndefined, boolean: booleanOrUndefined, byte: byteOrUndefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts index 7aec26143865e..742d28197e1f2 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts @@ -16,7 +16,7 @@ import { name, serializerOrUndefined, tie_breaker_id, - timestamp, + timestampOrUndefined, type, updated_at, updated_by, @@ -25,7 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types'; export const searchEsListSchema = t.exact( t.type({ - '@timestamp': timestamp, + '@timestamp': timestampOrUndefined, created_at, created_by, description, From 5b7092bb76832abeca818f6003dbb24cd463f4da Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 27 Jul 2023 14:50:48 +0100 Subject: [PATCH 06/41] add migration --- .../kbn-securitysolution-es-utils/index.ts | 2 + .../src/migrate_to_data_stream/index.ts | 24 ++++++ .../src/put_mappings/index.ts | 20 +++++ .../server/routes/create_list_index_route.ts | 84 +++++++++++-------- .../services/items/get_list_item_template.ts | 7 +- .../services/items/list_item_mappings.json | 3 + .../server/services/lists/create_list.ts | 6 +- .../services/lists/get_list_template.ts | 7 +- .../server/services/lists/list_client.ts | 38 ++++++++- .../server/services/lists/list_mappings.json | 3 + 10 files changed, 142 insertions(+), 52 deletions(-) create mode 100644 packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts index 65762c55af982..82411015758ac 100644 --- a/packages/kbn-securitysolution-es-utils/index.ts +++ b/packages/kbn-securitysolution-es-utils/index.ts @@ -23,8 +23,10 @@ export * from './src/get_data_stream_exists'; export * from './src/get_index_template_exists'; export * from './src/get_policy_exists'; export * from './src/get_template_exists'; +export * from './src/migrate_to_data_stream'; export * from './src/read_index'; export * from './src/read_privileges'; +export * from './src/put_mappings'; export * from './src/set_index_template'; export * from './src/set_policy'; export * from './src/set_template'; diff --git a/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts new file mode 100644 index 0000000000000..ed107b1ade0de --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * migrate to data stream + * @param esClient + * @param name + * @returns + */ +export const migrateToDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return esClient.indices.migrateToDataStream({ + name, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts new file mode 100644 index 0000000000000..7312813ceefdf --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient } from '../elasticsearch_client'; + +export const putMappings = async ( + esClient: ElasticsearchClient, + index: string, + mappings: Record +): Promise => { + return await esClient.indices.putMapping({ + index, + properties: mappings, + }); +}; diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts index 1315b6455c178..f05448a9cd8a4 100644 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_index_route.ts @@ -10,12 +10,31 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { acknowledgeSchema } from '@kbn/securitysolution-io-ts-list-types'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { ListClient } from '../services/lists/list_client'; import type { ListsPluginRouter } from '../types'; import { buildSiemResponse } from './utils'; import { getListClient } from '.'; +const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise => { + const legacyTemplateExists = await lists.getLegacyListTemplateExists(); + const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); + try { + // Check if the old legacy lists and items template exists and remove it + if (legacyTemplateExists) { + await lists.deleteLegacyListTemplate(); + } + if (legacyTemplateListItemsExists) { + await lists.deleteLegacyListItemTemplate(); + } + } catch (err) { + if (err.statusCode !== 404) { + throw err; + } + } +}; + export const createListIndexRoute = (router: ListsPluginRouter): void => { router.post( { @@ -33,20 +52,18 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { const listDataStreamExists = await lists.getListDataStreamExists(); const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - const policyExists = await lists.getListPolicyExists(); - const policyListItemExists = await lists.getListItemPolicyExists(); + // const policyExists = await lists.getListPolicyExists(); + // const policyListItemExists = await lists.getListItemPolicyExists(); - if (!policyExists) { - await lists.setListPolicy(); - } - if (!policyListItemExists) { - await lists.setListItemPolicy(); - } + // if (!policyExists) { + // await lists.setListPolicy(); + // } + // if (!policyListItemExists) { + // await lists.setListItemPolicy(); + // } const templateListExists = await lists.getListTemplateExists(); const templateListItemsExists = await lists.getListItemTemplateExists(); - const legacyTemplateExists = await lists.getLegacyListTemplateExists(); - const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); if (!templateListExists || !listDataStreamExists) { await lists.setListTemplate(); @@ -56,39 +73,34 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { await lists.setListItemTemplate(); } - try { - // Check if the old legacy lists and items template exists and remove it - if (legacyTemplateExists) { - await lists.deleteLegacyListTemplate(); - } - if (legacyTemplateListItemsExists) { - await lists.deleteLegacyListItemTemplate(); - } - } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - } + await removeLegacyTemplatesIfExist(lists); if (listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ body: `data stream: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, statusCode: 409, }); + } + + if (!listDataStreamExists) { + const listIndexExists = await lists.getListIndexExists(); + await (listIndexExists + ? lists.migrateListIndexToDataStream() + : lists.createListDataStream()); + } + + if (!listItemDataStreamExists) { + const listItemIndexExists = await lists.getListItemIndexExists(); + await (listItemIndexExists + ? lists.migrateListItemIndexToDataStream() + : lists.createListItemDataStream()); + } + + const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); } else { - if (!listDataStreamExists) { - await lists.createListDataStream(); - } - if (!listItemDataStreamExists) { - await lists.createListItemDataStream(); - } - - const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); - if (errors != null) { - return siemResponse.error({ body: errors, statusCode: 500 }); - } else { - return response.ok({ body: validated ?? {} }); - } + return response.ok({ body: validated ?? {} }); } } catch (err) { const error = transformError(err); diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts index eb69563af41bf..69d20f465944d 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts @@ -12,14 +12,9 @@ export const getListItemTemplate = (index: string): Record => { data_stream: {}, index_patterns: [index], template: { + lifecycle: {}, mappings: listsItemsMappings, settings: { - index: { - lifecycle: { - name: index, - rollover_alias: index, - }, - }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/items/list_item_mappings.json b/x-pack/plugins/lists/server/services/items/list_item_mappings.json index 1381402c2f2f4..67a8d784d3a73 100644 --- a/x-pack/plugins/lists/server/services/items/list_item_mappings.json +++ b/x-pack/plugins/lists/server/services/items/list_item_mappings.json @@ -1,6 +1,9 @@ { "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "tie_breaker_id": { "type": "keyword" }, diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts index 75eae4659a535..8ba231fbe9590 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -10,7 +10,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Description, DeserializerOrUndefined, - Id, + IdOrUndefined, Immutable, ListSchema, MetaOrUndefined, @@ -24,7 +24,7 @@ import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { IndexEsListSchema } from '../../schemas/elastic_query'; export interface CreateListOptions { - id: Id; + id: IdOrUndefined; deserializer: DeserializerOrUndefined; serializer: SerializerOrUndefined; type: Type; @@ -76,7 +76,7 @@ export const createList = async ({ const response = await esClient.create({ document: body, - id, + id: id ?? uuidv4(), index: listIndex, refresh: 'wait_for', }); diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.ts index 34f140b1c5002..81a66c0642af0 100644 --- a/x-pack/plugins/lists/server/services/lists/get_list_template.ts +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.ts @@ -11,14 +11,9 @@ export const getListTemplate = (index: string): Record => ({ data_stream: {}, index_patterns: [index], template: { + lifecycle: {}, mappings: listMappings, settings: { - index: { - lifecycle: { - name: index, - rollover_alias: index, - }, - }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index abdc19e58c9f6..711270edec589 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { createBootstrapIndex, @@ -18,6 +18,8 @@ import { getIndexTemplateExists, getPolicyExists, getTemplateExists, + migrateToDataStream, + putMappings, setIndexTemplate, setPolicy, } from '@kbn/securitysolution-es-utils'; @@ -49,8 +51,10 @@ import { updateListItem, } from '../items'; import listsItemsPolicy from '../items/list_item_policy.json'; +import listItemMappings from '../items/list_item_mappings.json'; import listPolicy from './list_policy.json'; +import listMappings from './list_mappings.json'; import type { ConstructorOptions, CreateListIfItDoesNotExistOptions, @@ -307,6 +311,38 @@ export class ListClient { return createDataStream(esClient, listIndex); }; + /** + * update list index mappings with @timestamp and migrates it to data stream + * @returns + */ + public migrateListIndexToDataStream = async (): Promise => { + const { esClient } = this; + const listIndex = this.getListIndex(); + // first need to update mapping of existing index to add @timestamp + await putMappings( + esClient, + listIndex, + listMappings.properties as Record + ); + return migrateToDataStream(esClient, listIndex); + }; + + /** + * update list items index mappings with @timestamp and migrates it to data stream + * @returns + */ + public migrateListItemIndexToDataStream = async (): Promise => { + const { esClient } = this; + const listItemIndex = this.getListItemIndex(); + // first need to update mapping of existing index to add @timestamp + await putMappings( + esClient, + listItemIndex, + listItemMappings.properties as Record + ); + return migrateToDataStream(esClient, listItemIndex); + }; + /** * Creates the list item boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch diff --git a/x-pack/plugins/lists/server/services/lists/list_mappings.json b/x-pack/plugins/lists/server/services/lists/list_mappings.json index d00b00b6469a3..c097408ee9527 100644 --- a/x-pack/plugins/lists/server/services/lists/list_mappings.json +++ b/x-pack/plugins/lists/server/services/lists/list_mappings.json @@ -1,6 +1,9 @@ { "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "name": { "type": "keyword" }, From a1754e3ea76ba473c20f2dbd504d387c9f23e4ea Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 11:11:50 +0100 Subject: [PATCH 07/41] add delete stream --- .../kbn-securitysolution-es-utils/index.ts | 1 + .../src/delete_data_stream/index.ts | 23 ++++ .../server/routes/create_list_index_route.ts | 30 +---- .../server/routes/delete_list_index_route.ts | 121 +++++++++++------- .../server/routes/import_list_item_route.ts | 3 +- .../lists/server/routes/utils/index.ts | 1 + .../routes/utils/remove_templates_if_exist.ts | 26 ++++ .../server/services/lists/list_client.ts | 21 +++ 8 files changed, 149 insertions(+), 77 deletions(-) create mode 100644 packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts create mode 100644 x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts index 82411015758ac..57f02580935dc 100644 --- a/packages/kbn-securitysolution-es-utils/index.ts +++ b/packages/kbn-securitysolution-es-utils/index.ts @@ -11,6 +11,7 @@ export * from './src/create_boostrap_index'; export * from './src/create_data_stream'; export * from './src/decode_version'; export * from './src/delete_all_index'; +export * from './src/delete_data_stream'; export * from './src/delete_index_template'; export * from './src/delete_policy'; export * from './src/delete_template'; diff --git a/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts new file mode 100644 index 0000000000000..198f41399ece2 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +export const deleteDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return ( + await esClient.indices.deleteDataStream( + { + name, + }, + { meta: true } + ) + ).body.acknowledged; +}; diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts index f05448a9cd8a4..23f2a274db1b9 100644 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_index_route.ts @@ -10,31 +10,12 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { acknowledgeSchema } from '@kbn/securitysolution-io-ts-list-types'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; -import { ListClient } from '../services/lists/list_client'; import type { ListsPluginRouter } from '../types'; -import { buildSiemResponse } from './utils'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from './utils'; import { getListClient } from '.'; -const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise => { - const legacyTemplateExists = await lists.getLegacyListTemplateExists(); - const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); - try { - // Check if the old legacy lists and items template exists and remove it - if (legacyTemplateExists) { - await lists.deleteLegacyListTemplate(); - } - if (legacyTemplateListItemsExists) { - await lists.deleteLegacyListItemTemplate(); - } - } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - } -}; - export const createListIndexRoute = (router: ListsPluginRouter): void => { router.post( { @@ -52,15 +33,6 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { const listDataStreamExists = await lists.getListDataStreamExists(); const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - // const policyExists = await lists.getListPolicyExists(); - // const policyListItemExists = await lists.getListItemPolicyExists(); - - // if (!policyExists) { - // await lists.setListPolicy(); - // } - // if (!policyListItemExists) { - // await lists.setListItemPolicy(); - // } const templateListExists = await lists.getListTemplateExists(); const templateListItemsExists = await lists.getListItemTemplateExists(); diff --git a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts index 2cd15f3026dbc..962360e495187 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts @@ -10,9 +10,10 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { acknowledgeSchema } from '@kbn/securitysolution-io-ts-list-types'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { ListClient } from '../services/lists/list_client'; import type { ListsPluginRouter } from '../types'; -import { buildSiemResponse } from './utils'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from './utils'; import { getListClient } from '.'; @@ -43,62 +44,35 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => { }, async (context, _, response) => { const siemResponse = buildSiemResponse(response); - + console.log('DELETE ME'); try { const lists = await getListClient(context); const listIndexExists = await lists.getListIndexExists(); const listItemIndexExists = await lists.getListItemIndexExists(); - if (!listIndexExists && !listItemIndexExists) { + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); + + // data streams exists, it means indices were migrated + if (listDataStreamExists) { + await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists); + } else if (!listIndexExists && !listItemIndexExists) { return siemResponse.error({ - body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`, + body: `index and data stream: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`, statusCode: 404, }); } else { - if (listIndexExists) { - await lists.deleteListIndex(); - } - if (listItemIndexExists) { - await lists.deleteListItemIndex(); - } - - const listsPolicyExists = await lists.getListPolicyExists(); - const listItemPolicyExists = await lists.getListItemPolicyExists(); - - if (listsPolicyExists) { - await lists.deleteListPolicy(); - } - if (listItemPolicyExists) { - await lists.deleteListItemPolicy(); - } - - const listsTemplateExists = await lists.getListTemplateExists(); - const listItemTemplateExists = await lists.getListItemTemplateExists(); - - if (listsTemplateExists) { - await lists.deleteListTemplate(); - } - if (listItemTemplateExists) { - await lists.deleteListItemTemplate(); - } - - // check if legacy template exists - const legacyTemplateExists = await lists.getLegacyListTemplateExists(); - const legacyItemTemplateExists = await lists.getLegacyListItemTemplateExists(); - if (legacyTemplateExists) { - await lists.deleteLegacyListTemplate(); - } + await deleteIndices(lists, listIndexExists, listItemIndexExists); + } - if (legacyItemTemplateExists) { - await lists.deleteLegacyListItemTemplate(); - } + await deleteIndexTemplates(lists); + await removeLegacyTemplatesIfExist(lists); - const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); - if (errors != null) { - return siemResponse.error({ body: errors, statusCode: 500 }); - } else { - return response.ok({ body: validated ?? {} }); - } + const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); } } catch (err) { const error = transformError(err); @@ -110,3 +84,58 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => { } ); }; + +/** + * Delete list/item indices + */ +const deleteIndices = async ( + lists: ListClient, + listIndexExists: boolean, + listItemIndexExists: boolean +): Promise => { + if (listIndexExists) { + await lists.deleteListIndex(); + } + if (listItemIndexExists) { + await lists.deleteListItemIndex(); + } + + const listsPolicyExists = await lists.getListPolicyExists(); + const listItemPolicyExists = await lists.getListItemPolicyExists(); + + if (listsPolicyExists) { + await lists.deleteListPolicy(); + } + if (listItemPolicyExists) { + await lists.deleteListItemPolicy(); + } +}; + +/** + * Delete list/item data streams + */ +const deleteDataStreams = async ( + lists: ListClient, + listDataStreamExists: boolean, + listItemDataStreamExists: boolean +): Promise => { + await lists.deleteListDataStream(); + if (listItemDataStreamExists) { + await lists.deleteListItemDataStream(); + } +}; + +/** + * Delete list/item index templates + */ +const deleteIndexTemplates = async (lists: ListClient): Promise => { + const listsTemplateExists = await lists.getListTemplateExists(); + const listItemTemplateExists = await lists.getListItemTemplateExists(); + + if (listsTemplateExists) { + await lists.deleteListTemplate(); + } + if (listItemTemplateExists) { + await lists.deleteListItemTemplate(); + } +}; diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts index 32987dec766da..cf4dc546681e8 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/import_list_item_route.ts @@ -45,11 +45,10 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp const stream = createStreamFromBuffer(request.body); const { deserializer, list_id: listId, serializer, type } = request.query; const lists = await getListClient(context); - // TODO: migrate?? const listExists = await lists.getListDataStreamExists(); if (!listExists) { return siemResponse.error({ - body: `To import a list item, the index must exist first. Index "${lists.getListIndex()}" does not exist`, + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListIndex()}" does not exist`, statusCode: 400, }); } diff --git a/x-pack/plugins/lists/server/routes/utils/index.ts b/x-pack/plugins/lists/server/routes/utils/index.ts index f035ae5dbfe9b..dd349bc5ec6e9 100644 --- a/x-pack/plugins/lists/server/routes/utils/index.ts +++ b/x-pack/plugins/lists/server/routes/utils/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +export * from './remove_templates_if_exist'; export * from './get_error_message_exception_list_item'; export * from './get_error_message_exception_list'; export * from './get_list_client'; diff --git a/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts b/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts new file mode 100644 index 0000000000000..1394c628eba7b --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ListClient } from '../../services/lists/list_client'; + +export const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise => { + const legacyTemplateExists = await lists.getLegacyListTemplateExists(); + const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); + try { + // Check if the old legacy lists and items template exists and remove it + if (legacyTemplateExists) { + await lists.deleteLegacyListTemplate(); + } + if (legacyTemplateListItemsExists) { + await lists.deleteLegacyListItemTemplate(); + } + } catch (err) { + if (err.statusCode !== 404) { + throw err; + } + } +}; diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 711270edec589..69ec265177fba 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -10,6 +10,7 @@ import { createBootstrapIndex, createDataStream, deleteAllIndex, + deleteDataStream, deleteIndexTemplate, deletePolicy, deleteTemplate, @@ -504,6 +505,26 @@ export class ListClient { return deleteAllIndex(esClient, `${listItemIndex}-*`); }; + /** + * Deletes the list data stream + * @returns True if the list index was deleted, otherwise false + */ + public deleteListDataStream = async (): Promise => { + const { esClient } = this; + const listIndex = this.getListIndex(); + return deleteDataStream(esClient, listIndex); + }; + + /** + * Deletes the list item data stream + * @returns True if the list index was deleted, otherwise false + */ + public deleteListItemDataStream = async (): Promise => { + const { esClient } = this; + const listItemIndex = this.getListItemIndex(); + return deleteDataStream(esClient, listItemIndex); + }; + /** * Deletes the list policy * @returns The contents of the list policy From 7960d47a16fa913fa1d91088c4e38981ab59e76b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 11:57:24 +0100 Subject: [PATCH 08/41] move routes to separate folder --- x-pack/plugins/lists/server/routes/index.ts | 6 +++--- .../{ => list_index}/create_list_index_route.ts | 10 ++++------ .../{ => list_index}/delete_list_index_route.ts | 12 +++++------- .../routes/{ => list_index}/read_list_index_route.ts | 10 ++++------ 4 files changed, 16 insertions(+), 22 deletions(-) rename x-pack/plugins/lists/server/routes/{ => list_index}/create_list_index_route.ts (94%) rename x-pack/plugins/lists/server/routes/{ => list_index}/delete_list_index_route.ts (94%) rename x-pack/plugins/lists/server/routes/{ => list_index}/read_list_index_route.ts (92%) diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index 6347d564981bf..0cb635d6fe7bf 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -9,13 +9,13 @@ export * from './create_endpoint_list_item_route'; export * from './create_endpoint_list_route'; export * from './create_exception_list_item_route'; export * from './create_exception_list_route'; -export * from './create_list_index_route'; +export * from './list_index/create_list_index_route'; export * from './create_list_item_route'; export * from './create_list_route'; export * from './delete_endpoint_list_item_route'; export * from './delete_exception_list_route'; export * from './delete_exception_list_item_route'; -export * from './delete_list_index_route'; +export * from './list_index/delete_list_index_route'; export * from './delete_list_item_route'; export * from './delete_list_route'; export * from './duplicate_exception_list_route'; @@ -36,7 +36,7 @@ export * from './patch_list_route'; export * from './read_endpoint_list_item_route'; export * from './read_exception_list_item_route'; export * from './read_exception_list_route'; -export * from './read_list_index_route'; +export * from './list_index/read_list_index_route'; export * from './read_list_item_route'; export * from './read_list_route'; export * from './read_privileges_route'; diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/create_list_index_route.ts rename to x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts index 7f59a04f6038c..3016313b57584 100644 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; -import { createListIndexResponse } from '../../common/api'; -import type { ListsPluginRouter } from '../types'; - -import { buildSiemResponse, removeLegacyTemplatesIfExist } from './utils'; - -import { getListClient } from '.'; +import { createListIndexResponse } from '../../../common/api'; +import type { ListsPluginRouter } from '../../types'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils'; +import { getListClient } from '..'; export const createListIndexRoute = (router: ListsPluginRouter): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/delete_list_index_route.ts rename to x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts index dbee6013275bc..8a3a2356d3e3e 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -9,13 +9,11 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; -import { ListClient } from '../services/lists/list_client'; -import type { ListsPluginRouter } from '../types'; -import { deleteListIndexResponse } from '../../common/api'; - -import { buildSiemResponse, removeLegacyTemplatesIfExist } from './utils'; - -import { getListClient } from '.'; +import { ListClient } from '../../services/lists/list_client'; +import type { ListsPluginRouter } from '../../types'; +import { deleteListIndexResponse } from '../../../common/api'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils'; +import { getListClient } from '..'; /** * Deletes all of the indexes, template, ilm policies, and aliases. You can check diff --git a/x-pack/plugins/lists/server/routes/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts similarity index 92% rename from x-pack/plugins/lists/server/routes/read_list_index_route.ts rename to x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts index 032c618d4b104..909f3ed564b4c 100644 --- a/x-pack/plugins/lists/server/routes/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { readListIndexResponse } from '../../common/api'; - -import { buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { readListIndexResponse } from '../../../common/api'; +import { buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListIndexRoute = (router: ListsPluginRouter): void => { router.get( From 5719cfd49ebc9ca98fd7e577bd474a80b6c0128e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 12:32:27 +0100 Subject: [PATCH 09/41] add delete list --- x-pack/plugins/lists/server/routes/index.ts | 6 +++--- .../server/routes/{ => list}/create_list_route.ts | 14 ++++++++------ .../server/routes/{ => list}/delete_list_route.ts | 14 ++++++-------- .../{ => list_item}/delete_list_item_route.ts | 10 ++++------ .../lists/server/services/lists/delete_list.ts | 12 +++++++++--- 5 files changed, 30 insertions(+), 26 deletions(-) rename x-pack/plugins/lists/server/routes/{ => list}/create_list_route.ts (89%) rename x-pack/plugins/lists/server/routes/{ => list}/delete_list_route.ts (96%) rename x-pack/plugins/lists/server/routes/{ => list_item}/delete_list_item_route.ts (94%) diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index 0cb635d6fe7bf..bb212cb8b3a91 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -11,13 +11,13 @@ export * from './create_exception_list_item_route'; export * from './create_exception_list_route'; export * from './list_index/create_list_index_route'; export * from './create_list_item_route'; -export * from './create_list_route'; +export * from './list/create_list_route'; export * from './delete_endpoint_list_item_route'; export * from './delete_exception_list_route'; export * from './delete_exception_list_item_route'; export * from './list_index/delete_list_index_route'; -export * from './delete_list_item_route'; -export * from './delete_list_route'; +export * from './list_item/delete_list_item_route'; +export * from './list/delete_list_route'; export * from './duplicate_exception_list_route'; export * from './export_exception_list_route'; export * from './export_list_item_route'; diff --git a/x-pack/plugins/lists/server/routes/create_list_route.ts b/x-pack/plugins/lists/server/routes/list/create_list_route.ts similarity index 89% rename from x-pack/plugins/lists/server/routes/create_list_route.ts rename to x-pack/plugins/lists/server/routes/list/create_list_route.ts index 484d045f23aae..9e2b66516f121 100644 --- a/x-pack/plugins/lists/server/routes/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/create_list_route.ts @@ -9,12 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { CreateListRequestDecoded, createListRequest, createListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { + CreateListRequestDecoded, + createListRequest, + createListResponse, +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const createListRoute = (router: ListsPluginRouter): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/list/delete_list_route.ts similarity index 96% rename from x-pack/plugins/lists/server/routes/delete_list_route.ts rename to x-pack/plugins/lists/server/routes/list/delete_list_route.ts index a8bb0fe01840d..ba14a2240d7e2 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/delete_list_route.ts @@ -18,14 +18,12 @@ import { import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import type { ExceptionListClient } from '../services/exception_lists/exception_list_client'; -import { escapeQuotes } from '../services/utils/escape_query'; -import { deleteListRequestQuery, deleteListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getExceptionListClient, getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import type { ExceptionListClient } from '../../services/exception_lists/exception_list_client'; +import { escapeQuotes } from '../../services/utils/escape_query'; +import { deleteListRequestQuery, deleteListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getExceptionListClient, getListClient } from '..'; export const deleteListRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/delete_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts index c580238455fff..a334d7012b461 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts @@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; +import type { ListsPluginRouter } from '../../types'; import { deleteListItemArrayResponse, deleteListItemRequestQuery, deleteListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const deleteListItemRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.ts b/x-pack/plugins/lists/server/services/lists/delete_list.ts index f82845e339ed7..661a5ae999644 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.ts @@ -39,10 +39,16 @@ export const deleteList = async ({ refresh: false, }); - await esClient.delete({ - id, + await esClient.deleteByQuery({ + body: { + query: { + ids: { + values: [id], + }, + }, + }, index: listIndex, - refresh: 'wait_for', + refresh: false, }); return list; } From 4938846b4c251e16c3590a15998048dd39dd2bc5 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 12:41:25 +0100 Subject: [PATCH 10/41] routes files cleanup --- .../kbn-securitysolution-list-constants/index.ts | 4 ++-- x-pack/plugins/lists/server/routes/index.ts | 6 +++--- .../create_exception_filter_route.ts} | 13 ++++++------- .../{ => internal}/find_lists_by_size_route.ts | 13 ++++++------- .../server/routes/{ => list}/read_list_route.ts | 10 ++++------ 5 files changed, 21 insertions(+), 25 deletions(-) rename x-pack/plugins/lists/server/routes/{get_exception_filter_route.ts => internal/create_exception_filter_route.ts} (88%) rename x-pack/plugins/lists/server/routes/{ => internal}/find_lists_by_size_route.ts (95%) rename x-pack/plugins/lists/server/routes/{ => list}/read_list_route.ts (87%) diff --git a/packages/kbn-securitysolution-list-constants/index.ts b/packages/kbn-securitysolution-list-constants/index.ts index 7bc44b534caf2..42249d5f305e9 100644 --- a/packages/kbn-securitysolution-list-constants/index.ts +++ b/packages/kbn-securitysolution-list-constants/index.ts @@ -20,8 +20,8 @@ export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`; * Internal value list routes */ export const INTERNAL_LIST_URL = '/internal/lists'; -export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const; -export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const; +export const INTERNAL_FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const; +export const INTERNAL_EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const; /** * Exception list routes diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index bb212cb8b3a91..74468a64121fb 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -26,8 +26,8 @@ export * from './find_exception_list_item_route'; export * from './find_exception_list_route'; export * from './find_list_item_route'; export * from './find_list_route'; -export * from './find_lists_by_size_route'; -export * from './get_exception_filter_route'; +export * from './internal/find_lists_by_size_route'; +export * from './internal/create_exception_filter_route'; export * from './import_exceptions_route'; export * from './import_list_item_route'; export * from './init_routes'; @@ -38,7 +38,7 @@ export * from './read_exception_list_item_route'; export * from './read_exception_list_route'; export * from './list_index/read_list_index_route'; export * from './read_list_item_route'; -export * from './read_list_route'; +export * from './list/read_list_route'; export * from './read_privileges_route'; export * from './summary_exception_list_route'; export * from './update_endpoint_list_item_route'; diff --git a/x-pack/plugins/lists/server/routes/get_exception_filter_route.ts b/x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts similarity index 88% rename from x-pack/plugins/lists/server/routes/get_exception_filter_route.ts rename to x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts index 1577040c9beb6..4fc6d2dfd402e 100644 --- a/x-pack/plugins/lists/server/routes/get_exception_filter_route.ts +++ b/x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts @@ -11,13 +11,12 @@ import { ExceptionListItemSchema, FoundExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants'; +import { INTERNAL_EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants'; -import { buildExceptionFilter } from '../services/exception_lists/build_exception_filter'; -import { ListsPluginRouter } from '../types'; -import { getExceptionFilterRequest } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; +import { buildExceptionFilter } from '../../services/exception_lists/build_exception_filter'; +import { ListsPluginRouter } from '../../types'; +import { getExceptionFilterRequest } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; export const getExceptionFilterRoute = (router: ListsPluginRouter): void => { router.post( @@ -25,7 +24,7 @@ export const getExceptionFilterRoute = (router: ListsPluginRouter): void => { options: { tags: ['access:securitySolution'], }, - path: `${EXCEPTION_FILTER}`, + path: INTERNAL_EXCEPTION_FILTER, validate: { body: buildRouteValidation(getExceptionFilterRequest), }, diff --git a/x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts b/x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts similarity index 95% rename from x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts rename to x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts index 1a6af1710ea61..6f65894d9d559 100644 --- a/x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts +++ b/x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts @@ -8,17 +8,16 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { - FIND_LISTS_BY_SIZE, + INTERNAL_FIND_LISTS_BY_SIZE, MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE, MAXIMUM_SMALL_VALUE_LIST_SIZE, } from '@kbn/securitysolution-list-constants'; import { chunk } from 'lodash'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; -import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; +import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListsBySizeRoute = (router: ListsPluginRouter): void => { router.get( @@ -26,7 +25,7 @@ export const findListsBySizeRoute = (router: ListsPluginRouter): void => { options: { tags: ['access:lists-read'], }, - path: `${FIND_LISTS_BY_SIZE}`, + path: INTERNAL_FIND_LISTS_BY_SIZE, validate: { query: buildRouteValidation(findListsBySizeRequestQuery), }, diff --git a/x-pack/plugins/lists/server/routes/read_list_route.ts b/x-pack/plugins/lists/server/routes/list/read_list_route.ts similarity index 87% rename from x-pack/plugins/lists/server/routes/read_list_route.ts rename to x-pack/plugins/lists/server/routes/list/read_list_route.ts index 583db5c065b62..d7a12a80ec7ef 100644 --- a/x-pack/plugins/lists/server/routes/read_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/read_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { readListRequestQuery, readListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { readListRequestQuery, readListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListRoute = (router: ListsPluginRouter): void => { router.get( From 599dc378f72353a2ce28c199b67c2a0ebbd4e2b2 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 15:37:12 +0100 Subject: [PATCH 11/41] create list API --- x-pack/plugins/lists/server/routes/index.ts | 4 ++-- .../lists/server/routes/list/create_list_route.ts | 6 +++--- .../routes/{ => list}/import_list_item_route.ts | 14 ++++++-------- .../{ => list_index}/export_list_item_route.ts | 10 ++++------ 4 files changed, 15 insertions(+), 19 deletions(-) rename x-pack/plugins/lists/server/routes/{ => list}/import_list_item_route.ts (92%) rename x-pack/plugins/lists/server/routes/{ => list_index}/export_list_item_route.ts (89%) diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index 74468a64121fb..67de214d58299 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -20,7 +20,7 @@ export * from './list_item/delete_list_item_route'; export * from './list/delete_list_route'; export * from './duplicate_exception_list_route'; export * from './export_exception_list_route'; -export * from './export_list_item_route'; +export * from './list_index/export_list_item_route'; export * from './find_endpoint_list_item_route'; export * from './find_exception_list_item_route'; export * from './find_exception_list_route'; @@ -29,7 +29,7 @@ export * from './find_list_route'; export * from './internal/find_lists_by_size_route'; export * from './internal/create_exception_filter_route'; export * from './import_exceptions_route'; -export * from './import_list_item_route'; +export * from './list/import_list_item_route'; export * from './init_routes'; export * from './patch_list_item_route'; export * from './patch_list_route'; diff --git a/x-pack/plugins/lists/server/routes/list/create_list_route.ts b/x-pack/plugins/lists/server/routes/list/create_list_route.ts index 9e2b66516f121..7e793b002495d 100644 --- a/x-pack/plugins/lists/server/routes/list/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/create_list_route.ts @@ -37,10 +37,10 @@ export const createListRoute = (router: ListsPluginRouter): void => { const { name, description, deserializer, id, serializer, type, meta, version } = request.body; const lists = await getListClient(context); - const listExists = await lists.getListIndexExists(); - if (!listExists) { + const dataStreamExists = await lists.getListDataStreamExists(); + if (!dataStreamExists) { return siemResponse.error({ - body: `To create a list, the index must exist first. Index "${lists.getListIndex()}" does not exist`, + body: `To create a list, the data stream must exist first. Data stream "${lists.getListIndex()}" does not exist`, statusCode: 400, }); } else { diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts similarity index 92% rename from x-pack/plugins/lists/server/routes/import_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list/import_list_item_route.ts index 120badebb7e03..ea5dbca235654 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts @@ -10,14 +10,12 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { ConfigType } from '../config'; -import { importListItemRequestQuery, importListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; -import { createStreamFromBuffer } from './utils/create_stream_from_buffer'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { ConfigType } from '../../config'; +import { importListItemRequestQuery, importListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { createStreamFromBuffer } from '../utils/create_stream_from_buffer'; +import { getListClient } from '..'; export const importListItemRoute = (router: ListsPluginRouter, config: ConfigType): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/export_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts similarity index 89% rename from x-pack/plugins/lists/server/routes/export_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts index 5ab4dcaea6f8f..079a25ab24b02 100644 --- a/x-pack/plugins/lists/server/routes/export_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts @@ -10,12 +10,10 @@ import { Stream } from 'stream'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { exportListItemRequestQuery } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { exportListItemRequestQuery } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const exportListItemRoute = (router: ListsPluginRouter): void => { router.post( From c5f82f020254804756da227301dd1d1f4ad97d0d Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 16:03:35 +0100 Subject: [PATCH 12/41] update most of APIs, apart from patch/delete --- x-pack/plugins/lists/server/routes/index.ts | 10 +++++----- .../server/routes/{ => list_index}/find_list_route.ts | 9 ++++----- .../routes/{ => list_item}/create_list_item_route.ts | 10 ++++------ .../routes/{ => list_item}/find_list_item_route.ts | 9 ++++----- .../routes/{ => list_item}/read_list_item_route.ts | 10 ++++------ .../read_list_privileges_route.mock.ts} | 0 .../read_list_privileges_route.ts} | 5 ++--- .../lists/server/services/items/create_list_item.ts | 5 +++-- .../lists/server/services/items/delete_list_item.ts | 10 +++++++--- 9 files changed, 33 insertions(+), 35 deletions(-) rename x-pack/plugins/lists/server/routes/{ => list_index}/find_list_route.ts (92%) rename x-pack/plugins/lists/server/routes/{ => list_item}/create_list_item_route.ts (93%) rename x-pack/plugins/lists/server/routes/{ => list_item}/find_list_item_route.ts (95%) rename x-pack/plugins/lists/server/routes/{ => list_item}/read_list_item_route.ts (94%) rename x-pack/plugins/lists/server/routes/{read_privileges_route.mock.ts => list_privileges/read_list_privileges_route.mock.ts} (100%) rename x-pack/plugins/lists/server/routes/{read_privileges_route.ts => list_privileges/read_list_privileges_route.ts} (93%) diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index 67de214d58299..dff1a6a6b23c4 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -10,7 +10,7 @@ export * from './create_endpoint_list_route'; export * from './create_exception_list_item_route'; export * from './create_exception_list_route'; export * from './list_index/create_list_index_route'; -export * from './create_list_item_route'; +export * from './list_item/create_list_item_route'; export * from './list/create_list_route'; export * from './delete_endpoint_list_item_route'; export * from './delete_exception_list_route'; @@ -24,8 +24,8 @@ export * from './list_index/export_list_item_route'; export * from './find_endpoint_list_item_route'; export * from './find_exception_list_item_route'; export * from './find_exception_list_route'; -export * from './find_list_item_route'; -export * from './find_list_route'; +export * from './list_item/find_list_item_route'; +export * from './list_index/find_list_route'; export * from './internal/find_lists_by_size_route'; export * from './internal/create_exception_filter_route'; export * from './import_exceptions_route'; @@ -37,9 +37,9 @@ export * from './read_endpoint_list_item_route'; export * from './read_exception_list_item_route'; export * from './read_exception_list_route'; export * from './list_index/read_list_index_route'; -export * from './read_list_item_route'; +export * from './list_item/read_list_item_route'; export * from './list/read_list_route'; -export * from './read_privileges_route'; +export * from './list_privileges/read_list_privileges_route'; export * from './summary_exception_list_route'; export * from './update_endpoint_list_item_route'; export * from './update_exception_list_item_route'; diff --git a/x-pack/plugins/lists/server/routes/find_list_route.ts b/x-pack/plugins/lists/server/routes/list_index/find_list_route.ts similarity index 92% rename from x-pack/plugins/lists/server/routes/find_list_route.ts rename to x-pack/plugins/lists/server/routes/list_index/find_list_route.ts index 411fc4e17c760..ab717a5f05ffb 100644 --- a/x-pack/plugins/lists/server/routes/find_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/find_list_route.ts @@ -9,11 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; -import { findListRequestQuery, findListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; +import { findListRequestQuery, findListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/create_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts similarity index 93% rename from x-pack/plugins/lists/server/routes/create_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts index c5642139ae3ad..9a06a9dc2c4e0 100644 --- a/x-pack/plugins/lists/server/routes/create_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import { createListItemRequest, createListItemResponse } from '../../common/api'; -import type { ListsPluginRouter } from '../types'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import { createListItemRequest, createListItemResponse } from '../../../common/api'; +import type { ListsPluginRouter } from '../../types'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const createListItemRoute = (router: ListsPluginRouter): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/find_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts similarity index 95% rename from x-pack/plugins/lists/server/routes/find_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts index 077dd90b7323a..acd1bbd0f834f 100644 --- a/x-pack/plugins/lists/server/routes/find_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts @@ -9,15 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; import { FindListItemRequestQueryDecoded, findListItemRequestQuery, findListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/read_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts index cb16d0d5f2df4..796be1ae2983d 100644 --- a/x-pack/plugins/lists/server/routes/read_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts @@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; +import type { ListsPluginRouter } from '../../types'; import { readListItemArrayResponse, readListItemRequestQuery, readListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_privileges_route.mock.ts b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.mock.ts similarity index 100% rename from x-pack/plugins/lists/server/routes/read_privileges_route.mock.ts rename to x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.mock.ts diff --git a/x-pack/plugins/lists/server/routes/read_privileges_route.ts b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts similarity index 93% rename from x-pack/plugins/lists/server/routes/read_privileges_route.ts rename to x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts index 51ffaa9713b62..1b4e254bf5aad 100644 --- a/x-pack/plugins/lists/server/routes/read_privileges_route.ts +++ b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts @@ -9,9 +9,8 @@ import { readPrivileges, transformError } from '@kbn/securitysolution-es-utils'; import { merge } from 'lodash/fp'; import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; - -import { buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { buildSiemResponse, getListClient } from '../utils'; export const readPrivilegesRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.ts b/x-pack/plugins/lists/server/services/items/create_list_item.ts index 9e80d8e4f4019..c51d3e3b944ea 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.ts @@ -52,6 +52,7 @@ export const createListItem = async ({ const createdAt = dateNow ?? new Date().toISOString(); const tieBreakerId = tieBreaker ?? uuidv4(); const baseBody = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, deserializer, @@ -68,9 +69,9 @@ export const createListItem = async ({ ...baseBody, ...elasticQuery, }; - const response = await esClient.index({ + const response = await esClient.create({ body, - id, + id: id ?? uuidv4(), index: listItemIndex, refresh: 'wait_for', }); diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.ts index 8fbdae3420acf..dffc12091d0fb 100644 --- a/x-pack/plugins/lists/server/services/items/delete_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.ts @@ -25,10 +25,14 @@ export const deleteListItem = async ({ if (listItem == null) { return null; } else { - await esClient.delete({ - id, + await esClient.deleteByQuery({ index: listItemIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, }); } return listItem; From 382342c24271972eba5b287c05408f578d64313b Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 16:48:05 +0100 Subject: [PATCH 13/41] fix types --- .../src/list_schema/index.mock.ts | 1 + .../src/response/list_item_schema/index.mock.ts | 1 + .../src/response/list_schema/index.mock.ts | 1 + packages/kbn-securitysolution-list-api/src/api/index.ts | 6 +++--- .../kbn-securitysolution-list-api/src/list_api/index.ts | 4 ++-- .../src/list_api/mocks/response/list_schema.mock.ts | 1 + .../src/mocks/response/list_schema.mock.ts | 1 + .../lists/common/schemas/response/list_item_schema.mock.ts | 1 + .../lists/common/schemas/response/list_schema.mock.ts | 1 + .../schemas/elastic_query/index_es_list_item_schema.mock.ts | 1 + .../schemas/elastic_query/index_es_list_schema.mock.ts | 1 + .../elastic_response/search_es_list_item_schema.mock.ts | 1 + .../schemas/elastic_response/search_es_list_schema.mock.ts | 1 + .../lists/server/services/items/get_list_item.test.ts | 1 + .../services/items/search_list_item_by_values.test.ts | 1 + .../plugins/lists/server/services/items/update_list_item.ts | 1 + x-pack/plugins/lists/server/services/lists/update_list.ts | 1 + .../server/services/utils/transform_elastic_to_list_item.ts | 1 + .../security_and_spaces/tests/get_exception_filter.ts | 6 +++--- 19 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts b/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts index 9dca48e2d8b38..9875db2fab1f1 100644 --- a/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts +++ b/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts @@ -42,6 +42,7 @@ export const NAME = 'some name'; // TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715 // import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts index 33d3b49cf5492..3b85f03e2facb 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts @@ -21,6 +21,7 @@ import { export const getListItemResponseMock = (): ListItemSchema => ({ _version: undefined, + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, deserializer: undefined, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts index 3c78c0cdbc976..6ca277659c078 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts @@ -23,6 +23,7 @@ import { export const getListResponseMock = (): ListSchema => ({ _version: undefined, + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/packages/kbn-securitysolution-list-api/src/api/index.ts b/packages/kbn-securitysolution-list-api/src/api/index.ts index 07d1b677a76bf..01a74756ce9be 100644 --- a/packages/kbn-securitysolution-list-api/src/api/index.ts +++ b/packages/kbn-securitysolution-list-api/src/api/index.ts @@ -37,7 +37,7 @@ import { import { ENDPOINT_LIST_URL, - EXCEPTION_FILTER, + INTERNAL_EXCEPTION_FILTER, EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL, } from '@kbn/securitysolution-list-constants'; @@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({ http, signal, }: GetExceptionFilterFromExceptionListIdsProps): Promise => - http.fetch(EXCEPTION_FILTER, { + http.fetch(INTERNAL_EXCEPTION_FILTER, { method: 'POST', body: JSON.stringify({ exception_list_ids: exceptionListIds, @@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({ chunkSize, signal, }: GetExceptionFilterFromExceptionsProps): Promise => - http.fetch(EXCEPTION_FILTER, { + http.fetch(INTERNAL_EXCEPTION_FILTER, { method: 'POST', body: JSON.stringify({ exceptions, diff --git a/packages/kbn-securitysolution-list-api/src/list_api/index.ts b/packages/kbn-securitysolution-list-api/src/list_api/index.ts index a7ca546ea2eef..81f4e9c94b3f0 100644 --- a/packages/kbn-securitysolution-list-api/src/list_api/index.ts +++ b/packages/kbn-securitysolution-list-api/src/list_api/index.ts @@ -37,7 +37,7 @@ import { LIST_ITEM_URL, LIST_PRIVILEGES_URL, LIST_URL, - FIND_LISTS_BY_SIZE, + INTERNAL_FIND_LISTS_BY_SIZE, } from '@kbn/securitysolution-list-constants'; import { toError, toPromise } from '../fp_utils'; @@ -115,7 +115,7 @@ const findListsBySize = async ({ per_page, signal, }: ApiParams & FindListSchemaEncoded): Promise => { - return http.fetch(`${FIND_LISTS_BY_SIZE}`, { + return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, { method: 'GET', query: { cursor, diff --git a/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts index 5421b4557c617..348c0506c9b26 100644 --- a/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts +++ b/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts @@ -23,6 +23,7 @@ import { } from '../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts index 5421b4557c617..348c0506c9b26 100644 --- a/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts +++ b/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts @@ -23,6 +23,7 @@ import { } from '../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts index 38bb32169beb5..cce497f87335c 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts @@ -20,6 +20,7 @@ import { } from '../../constants.mock'; export const getListItemResponseMock = (): ListItemSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts index adfe72d5e9125..abc52634e0232 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts @@ -22,6 +22,7 @@ import { } from '../../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts index 9a41946c6677c..a2b9217c0ad7a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts @@ -10,6 +10,7 @@ import { DATE_NOW, LIST_ID, META, TIE_BREAKER, USER, VALUE } from '../../../comm import { IndexEsListItemSchema } from './index_es_list_item_schema'; export const getIndexESListItemMock = (ip = VALUE): IndexEsListItemSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, deserializer: undefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts index 9e72c83223bab..b3cb37857cc9a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts @@ -20,6 +20,7 @@ import { import { IndexEsListSchema } from './index_es_list_schema'; export const getIndexESListMock = (): IndexEsListSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts index 40427f0293488..b8369c8ed06a1 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts @@ -22,6 +22,7 @@ import { getShardMock } from '../common/get_shard.mock'; import { SearchEsListItemSchema } from './search_es_list_item_schema'; export const getSearchEsListItemsAsAllUndefinedMock = (): SearchEsListItemSchema => ({ + '@timestamp': DATE_NOW, binary: undefined, boolean: undefined, byte: undefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts index 4e0cfef7c1352..1d4b282d5352a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts @@ -25,6 +25,7 @@ import { getShardMock } from '../common/get_shard.mock'; import { SearchEsListSchema } from './search_es_list_schema'; export const getSearchEsListMock = (): SearchEsListSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts index 2c2b703246a5d..55dfcb8062176 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts @@ -50,6 +50,7 @@ describe('get_list_item', () => { test('it returns null if all the values underneath the source type is undefined', async () => { const data = getSearchListItemMock(); data.hits.hits[0]._source = { + '@timestamp': DATE_NOW, binary: undefined, boolean: undefined, byte: undefined, diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts index 7673347d488dd..e52654a7a696e 100644 --- a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts @@ -74,6 +74,7 @@ describe('search_list_item_by_values', () => { { items: [ { + '@timestamp': '2020-04-20T15:25:31.830Z', _version: undefined, created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 71f6f9eba4290..2624c29880c87 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -70,6 +70,7 @@ export const updateListItem = async ({ refresh: 'wait_for', }); return { + '@timestamp': listItem['@timestamp'], _version: encodeHitVersion(response), created_at: listItem.created_at, created_by: listItem.created_by, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 300e4a20946c2..f63c5887552f3 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -67,6 +67,7 @@ export const updateList = async ({ refresh: 'wait_for', }); return { + '@timestamp': list['@timestamp'], _version: encodeHitVersion(response), created_at: list.created_at, created_by: list.created_by, diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts index 3edbab94a0cfd..b10aaebf4092e 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -56,6 +56,7 @@ export const transformElasticHitsToListItem = ({ throw new ErrorWithStatusCode(`Was expected ${type} to not be null/undefined`, 400); } else { return { + '@timestamp': _source?.['@timestamp'], _version: encodeHitVersion(hit), created_at, created_by, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts index abe119b066f88..a98f704d8a3d3 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import { - EXCEPTION_FILTER, + INTERNAL_EXCEPTION_FILTER, EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL, } from '@kbn/securitysolution-list-constants'; @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return an exception filter if correctly passed exception items', async () => { const { body } = await supertest - .post(`${EXCEPTION_FILTER}`) + .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') .send(getExceptionFilterFromExceptionItemsSchemaMock()) .expect(200); @@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext): void => { }) .expect(200); const { body } = await supertest - .post(`${EXCEPTION_FILTER}`) + .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') .send(getExceptionFilterFromExceptionIdsSchemaMock()) .expect(200); From 9f281a28f9738f0f2b8c66158cf427acb90258e5 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 28 Jul 2023 17:04:07 +0100 Subject: [PATCH 14/41] fix list privileges test --- .../security_and_spaces/tests/read_list_privileges.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts index c84bc80bb0231..9af6143b2152f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants'; -import { getReadPrivilegeMock } from '@kbn/lists-plugin/server/routes/read_privileges_route.mock'; +import { getReadPrivilegeMock } from '@kbn/lists-plugin/server/routes/list_privileges/read_list_privileges_route.mock'; import { FtrProviderContext } from '../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export From 28236434bae4042bfbf26b5d92d10eec76461a78 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 11:08:09 +0100 Subject: [PATCH 15/41] fix types --- .../security_and_spaces/tests/find_lists_by_size.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts index 49c1fc71ad27b..9452d95e7f972 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; -import { LIST_URL, FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, INTERNAL_FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return an empty found body correctly if no lists are loaded', async () => { const { body } = await supertest - .get(`${FIND_LISTS_BY_SIZE}`) + .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') .send() .expect(200); @@ -63,7 +63,7 @@ export default ({ getService }: FtrProviderContext): void => { // query the single list from _find_by_size const { body } = await supertest - .get(`${FIND_LISTS_BY_SIZE}`) + .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') .send() .expect(200); From 85ae8b66f635dd85ff04d97b97d118cc8320d703 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 11:09:39 +0100 Subject: [PATCH 16/41] fix some tests --- x-pack/test/lists_api_integration/utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 13ea80e20b0fd..95cc3b38d7a1f 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -98,8 +98,8 @@ export const removeListServerGeneratedProperties = ( list: Partial ): Partial => { /* eslint-disable-next-line @typescript-eslint/naming-convention */ - const { created_at, updated_at, id, tie_breaker_id, _version, ...removedProperties } = list; - return removedProperties; + const { created_at, updated_at, id, tie_breaker_id, _version, '@timestamp': _t, ...props } = list; + return props; }; /** From 316753d03a97323c9748a121c3bd5197c8052717 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 13:38:17 +0100 Subject: [PATCH 17/41] add more tests/fixes --- .../src/common/meta/index.ts | 3 +- .../elastic_query/update_es_list_schema.ts | 2 + .../server/services/lists/update_list.ts | 35 ++++++--- .../utils/transform_elastic_to_list.ts | 3 + .../tests/create_lists_index.ts | 2 +- .../tests/import_list_items.ts | 2 +- .../security_and_spaces/tests/update_lists.ts | 74 +++++++++++++++++++ x-pack/test/lists_api_integration/utils.ts | 4 +- 8 files changed, 111 insertions(+), 14 deletions(-) diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts index 93c38facd8f4e..99649b06ccaff 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts @@ -10,5 +10,6 @@ import * as t from 'io-ts'; export const meta = t.object; export type Meta = t.TypeOf; -export const metaOrUndefined = t.union([meta, t.undefined]); +// TODO: add null in name? +export const metaOrUndefined = t.union([meta, t.undefined, t.null]); export type MetaOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts index fe73d0fb9207f..15caf840d7dbd 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts @@ -6,6 +6,7 @@ */ import * as t from 'io-ts'; +import { version } from '@kbn/securitysolution-io-ts-types'; import { descriptionOrUndefined, metaOrUndefined, @@ -21,6 +22,7 @@ export const updateEsListSchema = t.exact( name: nameOrUndefined, updated_at, updated_by, + version, }) ); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index f63c5887552f3..c89d7fc63915e 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -15,7 +15,7 @@ import type { _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils'; +import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { UpdateEsListSchema } from '../../schemas/elastic_query'; @@ -35,7 +35,6 @@ export interface UpdateListOptions { } export const updateList = async ({ - _version, id, name, description, @@ -52,19 +51,37 @@ export const updateList = async ({ return null; } else { const calculatedVersion = version == null ? list.version + 1 : version; - const doc: UpdateEsListSchema = { + + const params: UpdateEsListSchema = { description, meta, name, updated_at: updatedAt, updated_by: user, + version: calculatedVersion, }; - const response = await esClient.update({ - ...decodeVersion(_version), - body: { doc }, - id, + + const response = await esClient.updateByQuery({ + conflicts: 'proceed', index: listIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, + script: { + lang: 'painless', + params, + source: ` + ctx._source.description = params.description; + ctx._source.meta = params.meta; + ctx._source.name = params.name; + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; + ctx._source.version = params.version; + `, + }, }); return { '@timestamp': list['@timestamp'], @@ -73,7 +90,7 @@ export const updateList = async ({ created_by: list.created_by, description: description ?? list.description, deserializer: list.deserializer, - id: response._id, + id, immutable: list.immutable, meta, name: name ?? list.name, diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts index 3242742c1cfd6..e5e917a147ac9 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts @@ -24,6 +24,9 @@ export const transformElasticToList = ({ _version: encodeHitVersion(hit), id: hit._id, ...hit._source, + // meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null + // return it as undefined to keep it consistent with payload + meta: hit._source?.meta ?? undefined, }; }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts index 8ae35e0fb0426..569610c91115c 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts @@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext) => { .expect(404); expect(fetchedIndices).to.eql({ - message: 'index .lists-default and index .items-default does not exist', + message: 'data stream .lists-default and data stream .items-default does not exist', status_code: 404, }); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts index 514ad3eb9f501..1d2be1c93d7f1 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(body).to.eql({ status_code: 400, message: - 'To import a list item, the index must exist first. Index ".lists-default" does not exist', + 'To import a list item, the data steam must exist first. Data stream ".lists-default" does not exist', }); }); }); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts index 7861c9fa2aa91..b98688f209d00 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts @@ -24,6 +24,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); describe('update_lists', () => { describe('update lists', () => { @@ -92,6 +93,79 @@ export default ({ getService }: FtrProviderContext) => { }; const bodyToCompare = removeListServerGeneratedProperties(body); expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.version).to.be(2); + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should remove unspecified meta field', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...listNoId, meta: { test: true } }); + + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList); + + expect(updatedListBody.meta).to.eql(undefined); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.meta).to.eql(undefined); + return true; + }); + }); + + it('should update meta field', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...listNoId, meta: { test: true } }); + + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + meta: { foo: 'some random value' }, + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList); + + expect(updatedListBody.meta).to.eql({ foo: 'some random value' }); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.meta).to.eql({ foo: 'some random value' }); + return true; + }); }); it('should change the version of a list when it updates a property', async () => { diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 95cc3b38d7a1f..52b07c179a90b 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -110,8 +110,8 @@ export const removeListItemServerGeneratedProperties = ( list: Partial ): Partial => { /* eslint-disable-next-line @typescript-eslint/naming-convention */ - const { created_at, updated_at, id, tie_breaker_id, _version, ...removedProperties } = list; - return removedProperties; + const { created_at, updated_at, id, tie_breaker_id, _version, '@timestamp': _t, ...props } = list; + return props; }; /** From 3e024012203ad2d9135e1629090c7bf39076abc7 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 15:38:00 +0100 Subject: [PATCH 18/41] patch list API --- .../lists/server/routes/patch_list_route.ts | 2 +- .../lists/server/services/lists/index.ts | 1 + .../server/services/lists/list_client.ts | 32 +++ .../lists/server/services/lists/patch_list.ts | 111 +++++++++ .../security_and_spaces/tests/index.ts | 1 + .../security_and_spaces/tests/patch_lists.ts | 214 ++++++++++++++++++ 6 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/lists/server/services/lists/patch_list.ts create mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/patch_list_route.ts index 41f9e1c815e54..ed15546f82744 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_route.ts @@ -32,7 +32,7 @@ export const patchListRoute = (router: ListsPluginRouter): void => { try { const { name, description, id, meta, _version, version } = request.body; const lists = await getListClient(context); - const list = await lists.updateList({ _version, description, id, meta, name, version }); + const list = await lists.patchList({ _version, description, id, meta, name, version }); if (list == null) { return siemResponse.error({ body: `list id: "${id}" not found`, diff --git a/x-pack/plugins/lists/server/services/lists/index.ts b/x-pack/plugins/lists/server/services/lists/index.ts index c12868816d91c..dcfc66055824b 100644 --- a/x-pack/plugins/lists/server/services/lists/index.ts +++ b/x-pack/plugins/lists/server/services/lists/index.ts @@ -14,3 +14,4 @@ export * from './get_list'; export * from './list_client'; export * from './types'; export * from './update_list'; +export * from './patch_list'; diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 69ec265177fba..b6aae6a30a426 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -86,6 +86,7 @@ import { getList, getListIndex, getListTemplate, + patchList, updateList, } from '.'; @@ -828,6 +829,37 @@ export class ListClient { }); }; + /** + * Patches a list container's value given the id of the list. + * @param options + * @param options._version This is the version, useful for optimistic concurrency control. + * @param options.id id of the list to replace the list container data with. + * @param options.name The new name, or "undefined" if this should not be updated. + * @param options.description The new description, or "undefined" if this should not be updated. + * @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. + * @param options.version Updates the version of the list. + */ + public patchList = async ({ + id, + name, + description, + meta, + version, + }: UpdateListOptions): Promise => { + const { esClient, user } = this; + const listIndex = this.getListIndex(); + return patchList({ + description, + esClient, + id, + listIndex, + meta, + name, + user, + version, + }); + }; + /** * Given a list item id, this returns the list item if it exists, otherwise "null". * @param options diff --git a/x-pack/plugins/lists/server/services/lists/patch_list.ts b/x-pack/plugins/lists/server/services/lists/patch_list.ts new file mode 100644 index 0000000000000..7b1f17ca109ad --- /dev/null +++ b/x-pack/plugins/lists/server/services/lists/patch_list.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import type { + DescriptionOrUndefined, + Id, + ListSchema, + MetaOrUndefined, + NameOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; +import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; + +import { UpdateEsListSchema } from '../../schemas/elastic_query'; + +import { getList } from '.'; + +export interface PatchListOptions { + id: Id; + esClient: ElasticsearchClient; + listIndex: string; + user: string; + name: NameOrUndefined; + description: DescriptionOrUndefined; + meta: MetaOrUndefined; + dateNow?: string; + version: VersionOrUndefined; +} + +export const patchList = async ({ + id, + name, + description, + esClient, + listIndex, + user, + meta, + dateNow, + version, +}: PatchListOptions): Promise => { + const updatedAt = dateNow ?? new Date().toISOString(); + const list = await getList({ esClient, id, listIndex }); + if (list == null) { + return null; + } else { + const calculatedVersion = version == null ? list.version + 1 : version; + + const params: UpdateEsListSchema = { + description, + meta, + name, + updated_at: updatedAt, + updated_by: user, + version: calculatedVersion, + }; + + const response = await esClient.updateByQuery({ + conflicts: 'proceed', + index: listIndex, + query: { + ids: { + values: [id], + }, + }, + refresh: false, + script: { + lang: 'painless', + params, + source: ` + if (params.containsKey('description')) { + ctx._source.description = params.description; + } + if (params.containsKey('meta')) { + ctx._source.meta = params.meta; + } + if (params.containsKey('name')) { + ctx._source.name = params.name; + } + if (params.containsKey('version')) { + ctx._source.version = params.version; + } + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; + `, + }, + }); + return { + '@timestamp': list['@timestamp'], + _version: encodeHitVersion(response), + created_at: list.created_at, + created_by: list.created_by, + description: description ?? list.description, + deserializer: list.deserializer, + id, + immutable: list.immutable, + meta: meta ?? list.meta, + name: name ?? list.name, + serializer: list.serializer, + tie_breaker_id: list.tie_breaker_id, + type: list.type, + updated_at: updatedAt, + updated_by: user, + version: calculatedVersion, + }; + } +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts index 02b63c732d229..c66fb4ec8b0d2 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts @@ -13,6 +13,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_lists')); loadTestFile(require.resolve('./create_lists_index')); loadTestFile(require.resolve('./create_list_items')); + loadTestFile(require.resolve('./patch_lists')); loadTestFile(require.resolve('./read_lists')); loadTestFile(require.resolve('./read_list_items')); loadTestFile(require.resolve('./update_lists')); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts new file mode 100644 index 0000000000000..97d1b8d007f99 --- /dev/null +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { PatchListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL } from '@kbn/securitysolution-list-constants'; + +import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; +import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; +import { getUpdateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_schema.mock'; +import { + createListsIndex, + deleteListsIndex, + removeListServerGeneratedProperties, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const retry = getService('retry'); + + describe('patch_lists', () => { + describe('patch lists', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should patch a single list property of name using an id', async () => { + const listId = getCreateMinimalListSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: listId }) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should patch a single list property of name using an auto-generated id', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: createListBody.id, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should not remove unspecified fields in payload', async () => { + const listId = getCreateMinimalListSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: getUpdateMinimalListSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + const persistentBodyToCompare = removeListServerGeneratedProperties(list); + + expect(persistentBodyToCompare).to.eql(outputList); + return true; + }); + }); + + it('should change the version of a list when it patches a property', async () => { + // create a simple list + const { body: createdList } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list property of name and description + const patchPayload: PatchListSchema = { + id: createdList.id, + name: 'some other name', + description: 'some other description', + }; + + const { body: patchedList } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload); + + expect(createdList.version).to.be(1); + expect(patchedList.version).to.be(2); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: patchedList.id }) + .set('kbn-xsrf', 'true'); + + expect(list.version).to.be(2); + return true; + }); + }); + + it('should give a 404 if it is given a fake id', async () => { + const simpleList: PatchListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: '5096dec6-b6b9-4d8d-8f93-6c2602079d9d', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(simpleList) + .expect(404); + + expect(body).to.eql({ + status_code: 404, + message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', + }); + }); + }); + }); +}; From d49abe63280cf46c0d0e0a8fa843be6a6ba7531f Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 16:05:32 +0100 Subject: [PATCH 19/41] fix eslint tests --- .../services/items/create_list_item.test.ts | 8 ++++---- .../services/items/delete_list_item.test.ts | 14 ++++++++----- .../services/items/find_list_item.test.ts | 1 + .../items/get_list_item_by_values.test.ts | 1 + .../items/get_list_item_template.test.ts | 5 +++-- .../server/services/lists/create_list.test.ts | 10 +++++----- .../server/services/lists/create_list.ts | 2 +- .../server/services/lists/delete_list.test.ts | 20 ++++++++++++------- .../services/lists/get_list_template.test.ts | 5 +++-- .../server/services/lists/update_list.test.ts | 16 +-------------- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts index ec8ba2876126c..700d85d935d3f 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts @@ -14,7 +14,7 @@ import { getIndexESListItemMock } from '../../schemas/elastic_query/index_es_lis import { CreateListItemOptions, createListItem } from './create_list_item'; import { getCreateListItemOptionsMock } from './create_list_item.mock'; -describe('crete_list_item', () => { +describe('create_list_item', () => { beforeEach(() => { jest.clearAllMocks(); }); @@ -26,7 +26,7 @@ describe('crete_list_item', () => { test('it returns a list item as expected with the id changed out for the elastic id', async () => { const options = getCreateListItemOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -46,14 +46,14 @@ describe('crete_list_item', () => { index: LIST_ITEM_INDEX, refresh: 'wait_for', }; - expect(options.esClient.index).toBeCalledWith(expected); + expect(options.esClient.create).toBeCalledWith(expected); }); test('It returns an auto-generated id if id is sent in undefined', async () => { const options = getCreateListItemOptionsMock(); options.id = undefined; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts index d2c424804297d..a468eae0b8811 100644 --- a/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts @@ -40,16 +40,20 @@ describe('delete_list_item', () => { expect(deletedListItem).toEqual(listItem); }); - test('Delete calls "delete" if a list item is returned from "getListItem"', async () => { + test('Delete calls "deleteByQuery" if a list item is returned from "getListItem"', async () => { const listItem = getListItemResponseMock(); (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); const options = getDeleteListItemOptionsMock(); await deleteListItem(options); - const deleteQuery = { - id: LIST_ITEM_ID, + const deleteByQuery = { index: LIST_ITEM_INDEX, - refresh: 'wait_for', + query: { + ids: { + values: [LIST_ITEM_ID], + }, + }, + refresh: false, }; - expect(options.esClient.delete).toBeCalledWith(deleteQuery); + expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery); }); }); diff --git a/x-pack/plugins/lists/server/services/items/find_list_item.test.ts b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts index 8b1457f0ce53d..93ed864c05f00 100644 --- a/x-pack/plugins/lists/server/services/items/find_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts @@ -31,6 +31,7 @@ describe('find_list_item', () => { { _id: 'some-list-item-id', _source: { + '@timestamp': '2020-04-20T15:25:31.830Z', _version: 'undefined', created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts index 9f0b183b97b5c..d48a59fb48668 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts @@ -62,6 +62,7 @@ describe('get_list_item_by_values', () => { expect(listItem).toEqual([ { + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, id: LIST_ITEM_ID, diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts index 10d1a0be11efb..dd2547cdaca13 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts @@ -23,11 +23,12 @@ describe('get_list_item_template', () => { test('it returns a list template with the string filled in', async () => { const template = getListItemTemplate('some_index'); expect(template).toEqual({ - index_patterns: ['some_index-*'], + data_stream: {}, + index_patterns: ['some_index'], template: { + lifecycle: {}, mappings: { listMappings: {} }, settings: { - index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/lists/create_list.test.ts b/x-pack/plugins/lists/server/services/lists/create_list.test.ts index 96744d9f55bae..86c0155c43ec4 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.test.ts @@ -15,7 +15,7 @@ import { getIndexESListMock } from '../../schemas/elastic_query/index_es_list_sc import { CreateListOptions, createList } from './create_list'; import { getCreateListOptionsMock } from './create_list.mock'; -describe('crete_list', () => { +describe('create_list', () => { beforeEach(() => { jest.clearAllMocks(); }); @@ -27,7 +27,7 @@ describe('crete_list', () => { test('it returns a list as expected with the id changed out for the elastic id', async () => { const options = getCreateListOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -43,7 +43,7 @@ describe('crete_list', () => { serializer: '(?)', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -67,14 +67,14 @@ describe('crete_list', () => { index: LIST_INDEX, refresh: 'wait_for', }; - expect(options.esClient.index).toBeCalledWith(expected); + expect(options.esClient.create).toBeCalledWith(expected); }); test('It returns an auto-generated id if id is sent in undefined', async () => { const options = getCreateListOptionsMock(); options.id = undefined; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts index 8ba231fbe9590..cd3497b6e891a 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -75,7 +75,7 @@ export const createList = async ({ }; const response = await esClient.create({ - document: body, + body, id: id ?? uuidv4(), index: listIndex, refresh: 'wait_for', diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts index 9fb0c5c8d6469..c00815625323a 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts @@ -40,7 +40,7 @@ describe('delete_list', () => { expect(deletedList).toEqual(list); }); - test('Delete calls "deleteByQuery" and "delete" if a list is returned from getList', async () => { + test('Delete calls "deleteByQuery" for list items if a list is returned from getList', async () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); @@ -50,20 +50,26 @@ describe('delete_list', () => { index: LIST_ITEM_INDEX, refresh: false, }; - expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery); + expect(options.esClient.deleteByQuery).toHaveBeenNthCalledWith(1, deleteByQuery); }); - test('Delete calls "delete" second if a list is returned from getList', async () => { + test('Delete calls "deleteByQuery" for list if a list is returned from getList', async () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); await deleteList(options); - const deleteQuery = { - id: LIST_ID, + const deleteByQuery = { + body: { + query: { + ids: { + values: [LIST_ID], + }, + }, + }, index: LIST_INDEX, - refresh: 'wait_for', + refresh: false, }; - expect(options.esClient.delete).toHaveBeenNthCalledWith(1, deleteQuery); + expect(options.esClient.deleteByQuery).toHaveBeenCalledWith(deleteByQuery); }); test('Delete does not call data client if the list returns null', async () => { diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts index 57d67d6d383ca..d77eb8917c5ef 100644 --- a/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts @@ -24,11 +24,12 @@ describe('get_list_template', () => { test('it returns a list template with the string filled in', async () => { const template = getListTemplate('some_index'); expect(template).toEqual({ - index_patterns: ['some_index-*'], + data_stream: {}, + index_patterns: ['some_index'], template: { + lifecycle: {}, mappings: { dynamic: 'strict', properties: {} }, settings: { - index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } }, mapping: { total_fields: { limit: 10000 } }, }, }, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts index 8e68acd358861..338f3eab476d0 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -27,20 +27,6 @@ describe('update_list', () => { jest.clearAllMocks(); }); - test('it returns a list as expected with the id changed out for the elastic id when there is a list to update', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getUpdateListOptionsMock(); - const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); - const updatedList = await updateList({ ...options, esClient }); - const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; - expect(updatedList).toEqual(expected); - }); - test('it returns a list with serializer and deserializer', async () => { const list: ListSchema = { ...getListResponseMock(), @@ -58,7 +44,7 @@ describe('update_list', () => { const expected: ListSchema = { ...getListResponseMock(), deserializer: '{{value}}', - id: 'elastic-id-123', + id: list.id, serializer: '(?)', }; expect(updatedList).toEqual(expected); From 7eaea0a53b742450e79351cef2be364ee48d8212 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Mon, 31 Jul 2023 16:12:30 +0100 Subject: [PATCH 20/41] fix typings --- .../src/common/meta/index.ts | 6 ++++-- .../schemas/elastic_response/search_es_list_schema.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts index 99649b06ccaff..93376f3112530 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts @@ -10,6 +10,8 @@ import * as t from 'io-ts'; export const meta = t.object; export type Meta = t.TypeOf; -// TODO: add null in name? -export const metaOrUndefined = t.union([meta, t.undefined, t.null]); +export const metaOrUndefined = t.union([meta, t.undefined]); export type MetaOrUndefined = t.TypeOf; + +export const nullableMetaOrUndefined = t.union([metaOrUndefined, t.null]); +export type NullableMetaOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts index 742d28197e1f2..17d2e64fb7542 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts @@ -12,8 +12,8 @@ import { description, deserializerOrUndefined, immutable, - metaOrUndefined, name, + nullableMetaOrUndefined, serializerOrUndefined, tie_breaker_id, timestampOrUndefined, @@ -31,7 +31,7 @@ export const searchEsListSchema = t.exact( description, deserializer: deserializerOrUndefined, immutable, - meta: metaOrUndefined, + meta: nullableMetaOrUndefined, name, serializer: serializerOrUndefined, tie_breaker_id, From 3185503afff7514cfbb907762049bc892abed67e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 10:43:53 +0100 Subject: [PATCH 21/41] fix unit tests --- .../lists/server/services/items/create_list_items_bulk.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts index ea2ff697c7d37..a2499a94d619e 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts @@ -56,6 +56,7 @@ describe('crete_list_item_bulk', () => { body: [ { create: { _index: LIST_ITEM_INDEX } }, { + '@timestamp': '2020-04-20T15:25:31.830Z', created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', deserializer: undefined, From 2f729aade05d9e289cdab8ac36d35a760df639aa Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 11:35:35 +0100 Subject: [PATCH 22/41] add update list item --- .../search_es_list_item_schema.ts | 4 +- .../server/services/items/update_list_item.ts | 42 ++++++++---- .../utils/transform_elastic_to_list_item.ts | 4 +- .../tests/update_list_items.ts | 66 ++++++++++++++++++- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts index eed15628c907a..8b0fa71f2222f 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts @@ -11,7 +11,7 @@ import { created_by, deserializerOrUndefined, list_id, - metaOrUndefined, + nullableMetaOrUndefined, serializerOrUndefined, tie_breaker_id, timestampOrUndefined, @@ -72,7 +72,7 @@ export const searchEsListItemSchema = t.exact( list_id, long: longOrUndefined, long_range: longRangeOrUndefined, - meta: metaOrUndefined, + meta: nullableMetaOrUndefined, serializer: serializerOrUndefined, shape: shapeOrUndefined, short: shortOrUndefined, diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 2624c29880c87..06fa4ccc7af5b 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -15,7 +15,6 @@ import type { import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { transformListItemToElasticQuery } from '../utils'; -import { UpdateEsListItemSchema } from '../../schemas/elastic_query'; import { getListItem } from './get_list_item'; @@ -53,21 +52,40 @@ export const updateListItem = async ({ if (elasticQuery == null) { return null; } else { - const doc: UpdateEsListItemSchema = { + const keyValues = Object.entries(elasticQuery).map(([key, keyValue]) => ({ + key, + value: keyValue, + })); + + const params = { + keyValues, meta, updated_at: updatedAt, updated_by: user, - ...elasticQuery, }; - const response = await esClient.update({ - ...decodeVersion(_version), - body: { - doc, - }, - id: listItem.id, + const response = await esClient.updateByQuery({ + conflicts: 'proceed', index: listItemIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, + script: { + lang: 'painless', + params, + source: ` + for (int i; i < params.keyValues.size(); i++) { + def entry = params.keyValues[i]; + ctx._source[entry.key] = entry.value; + } + ctx._source.meta = params.meta; + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; + `, + }, }); return { '@timestamp': listItem['@timestamp'], @@ -75,9 +93,9 @@ export const updateListItem = async ({ created_at: listItem.created_at, created_by: listItem.created_by, deserializer: listItem.deserializer, - id: response._id, + id, list_id: listItem.list_id, - meta: meta ?? listItem.meta, + meta, serializer: listItem.serializer, tie_breaker_id: listItem.tie_breaker_id, type: listItem.type, diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts index b10aaebf4092e..46ac86a5c8aae 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -63,7 +63,9 @@ export const transformElasticHitsToListItem = ({ deserializer, id: _id, list_id, - meta, + // meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null + // return it as undefined to keep it consistent with payload + meta: meta ?? undefined, serializer, tie_breaker_id, type, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts index e34ce8ba35465..ebceaf76ba486 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts @@ -29,6 +29,7 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); describe('update_list_items', () => { describe('update list items', () => { @@ -64,8 +65,7 @@ export default ({ getService }: FtrProviderContext) => { const { body } = await supertest .put(LIST_ITEM_URL) .set('kbn-xsrf', 'true') - .send(updatedListItem) - .expect(200); + .send(updatedListItem); const outputListItem: Partial = { ...getListItemResponseMockWithoutAutoGeneratedValues(), @@ -73,6 +73,16 @@ export default ({ getService }: FtrProviderContext) => { }; const bodyToCompare = removeListItemServerGeneratedProperties(body); expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(removeListItemServerGeneratedProperties(listItemBody)).to.eql(outputListItem); + return true; + }); }); it('should update a single list item of value using an auto-generated id of both list and list item', async () => { @@ -116,6 +126,58 @@ export default ({ getService }: FtrProviderContext) => { list_id: outputListItem.list_id, }; expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: createListItemBody.id }) + .set('kbn-xsrf', 'true'); + const listItemBodyToCompare = { + ...removeListItemServerGeneratedProperties(listItemBody), + list_id: outputListItem.list_id, + }; + expect(listItemBodyToCompare).to.eql(outputListItem); + return true; + }); + }); + + it('should remove unspecified in update payload meta property', async () => { + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateMinimalListItemSchemaMock(), meta: { test: true } }) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + value: '192.168.0.2', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem); + + expect(body.meta).to.eql(undefined); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(listItemBody.meta).to.eql(undefined); + return true; + }); }); it('should give a 404 if it is given a fake id', async () => { From 0ef449b1c442649a7abf358a7902bec320fb5f64 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 11:39:34 +0100 Subject: [PATCH 23/41] fix unit test --- .../src/field_value_lists/index.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx b/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx index 6cd92fe1a1b35..acd1a315d2c48 100644 --- a/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx +++ b/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx @@ -211,6 +211,7 @@ describe('AutocompleteFieldListsComponent', () => { await waitFor(() => { expect(mockOnChange).toHaveBeenCalledWith({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: 'some user', From cb9c6e02f58d3e95f1c7121b5c0aa3700d043fb3 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 12:25:43 +0100 Subject: [PATCH 24/41] add [atch and tests --- .../server/routes/patch_list_item_route.ts | 2 +- .../server/services/items/update_list_item.ts | 11 +- .../lists/server/services/lists/index.ts | 1 - .../server/services/lists/list_client.ts | 38 ++- .../lists/server/services/lists/patch_list.ts | 111 --------- .../server/services/lists/update_list.ts | 23 +- .../security_and_spaces/tests/index.ts | 1 + .../tests/patch_list_items.ts | 220 ++++++++++++++++++ 8 files changed, 286 insertions(+), 121 deletions(-) delete mode 100644 x-pack/plugins/lists/server/services/lists/patch_list.ts create mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts index c4496197727e4..ad18c8ef670a4 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts @@ -32,7 +32,7 @@ export const patchListItemRoute = (router: ListsPluginRouter): void => { try { const { value, id, meta, _version } = request.body; const lists = await getListClient(context); - const listItem = await lists.updateListItem({ + const listItem = await lists.patchListItem({ _version, id, meta, diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 06fa4ccc7af5b..bd9c2242da505 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -27,6 +27,7 @@ export interface UpdateListItemOptions { user: string; meta: MetaOrUndefined; dateNow?: string; + isPatch?: boolean; } export const updateListItem = async ({ @@ -38,6 +39,7 @@ export const updateListItem = async ({ user, meta, dateNow, + isPatch = false, }: UpdateListItemOptions): Promise => { const updatedAt = dateNow ?? new Date().toISOString(); const listItem = await getListItem({ esClient, id, listItemIndex }); @@ -58,6 +60,9 @@ export const updateListItem = async ({ })); const params = { + // when assigning undefined in painless, it will remover property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !isPatch, keyValues, meta, updated_at: updatedAt, @@ -81,7 +86,9 @@ export const updateListItem = async ({ def entry = params.keyValues[i]; ctx._source[entry.key] = entry.value; } - ctx._source.meta = params.meta; + if (params.assignEmpty == true || params.containsKey('meta')) { + ctx._source.meta = params.meta; + } ctx._source.updated_at = params.updated_at; ctx._source.updated_by = params.updated_by; `, @@ -95,7 +102,7 @@ export const updateListItem = async ({ deserializer: listItem.deserializer, id, list_id: listItem.list_id, - meta, + meta: isPatch ? meta ?? listItem.meta : meta, serializer: listItem.serializer, tie_breaker_id: listItem.tie_breaker_id, type: listItem.type, diff --git a/x-pack/plugins/lists/server/services/lists/index.ts b/x-pack/plugins/lists/server/services/lists/index.ts index dcfc66055824b..c12868816d91c 100644 --- a/x-pack/plugins/lists/server/services/lists/index.ts +++ b/x-pack/plugins/lists/server/services/lists/index.ts @@ -14,4 +14,3 @@ export * from './get_list'; export * from './list_client'; export * from './types'; export * from './update_list'; -export * from './patch_list'; diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index b6aae6a30a426..ec7231712cf29 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -86,7 +86,6 @@ import { getList, getListIndex, getListTemplate, - patchList, updateList, } from '.'; @@ -787,6 +786,37 @@ export class ListClient { _version, esClient, id, + isPatch: false, + listItemIndex, + meta, + user, + value, + }); + }; + + /** + * Patches a list item's value given the id of the list item. + * See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html} + * for more information around optimistic concurrency control. + * @param options + * @param options._version This is the version, useful for optimistic concurrency control. + * @param options.id id of the list to replace the list item with. + * @param options.value The value of the list item to replace. + * @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. + */ + public patchListItem = async ({ + _version, + id, + value, + meta, + }: UpdateListItemOptions): Promise => { + const { esClient, user } = this; + const listItemIndex = this.getListItemIndex(); + return updateListItem({ + _version, + esClient, + id, + isPatch: true, listItemIndex, meta, user, @@ -821,6 +851,7 @@ export class ListClient { description, esClient, id, + isPatch: false, listIndex, meta, name, @@ -840,6 +871,7 @@ export class ListClient { * @param options.version Updates the version of the list. */ public patchList = async ({ + _version, id, name, description, @@ -848,10 +880,12 @@ export class ListClient { }: UpdateListOptions): Promise => { const { esClient, user } = this; const listIndex = this.getListIndex(); - return patchList({ + return updateList({ + _version, description, esClient, id, + isPatch: true, listIndex, meta, name, diff --git a/x-pack/plugins/lists/server/services/lists/patch_list.ts b/x-pack/plugins/lists/server/services/lists/patch_list.ts deleted file mode 100644 index 7b1f17ca109ad..0000000000000 --- a/x-pack/plugins/lists/server/services/lists/patch_list.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from '@kbn/core/server'; -import type { - DescriptionOrUndefined, - Id, - ListSchema, - MetaOrUndefined, - NameOrUndefined, -} from '@kbn/securitysolution-io-ts-list-types'; -import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; - -import { UpdateEsListSchema } from '../../schemas/elastic_query'; - -import { getList } from '.'; - -export interface PatchListOptions { - id: Id; - esClient: ElasticsearchClient; - listIndex: string; - user: string; - name: NameOrUndefined; - description: DescriptionOrUndefined; - meta: MetaOrUndefined; - dateNow?: string; - version: VersionOrUndefined; -} - -export const patchList = async ({ - id, - name, - description, - esClient, - listIndex, - user, - meta, - dateNow, - version, -}: PatchListOptions): Promise => { - const updatedAt = dateNow ?? new Date().toISOString(); - const list = await getList({ esClient, id, listIndex }); - if (list == null) { - return null; - } else { - const calculatedVersion = version == null ? list.version + 1 : version; - - const params: UpdateEsListSchema = { - description, - meta, - name, - updated_at: updatedAt, - updated_by: user, - version: calculatedVersion, - }; - - const response = await esClient.updateByQuery({ - conflicts: 'proceed', - index: listIndex, - query: { - ids: { - values: [id], - }, - }, - refresh: false, - script: { - lang: 'painless', - params, - source: ` - if (params.containsKey('description')) { - ctx._source.description = params.description; - } - if (params.containsKey('meta')) { - ctx._source.meta = params.meta; - } - if (params.containsKey('name')) { - ctx._source.name = params.name; - } - if (params.containsKey('version')) { - ctx._source.version = params.version; - } - ctx._source.updated_at = params.updated_at; - ctx._source.updated_by = params.updated_by; - `, - }, - }); - return { - '@timestamp': list['@timestamp'], - _version: encodeHitVersion(response), - created_at: list.created_at, - created_by: list.created_by, - description: description ?? list.description, - deserializer: list.deserializer, - id, - immutable: list.immutable, - meta: meta ?? list.meta, - name: name ?? list.name, - serializer: list.serializer, - tie_breaker_id: list.tie_breaker_id, - type: list.type, - updated_at: updatedAt, - updated_by: user, - version: calculatedVersion, - }; - } -}; diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index c89d7fc63915e..7ad16448f07c5 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -32,6 +32,7 @@ export interface UpdateListOptions { meta: MetaOrUndefined; dateNow?: string; version: VersionOrUndefined; + isPatch?: boolean; } export const updateList = async ({ @@ -44,6 +45,7 @@ export const updateList = async ({ meta, dateNow, version, + isPatch = false, }: UpdateListOptions): Promise => { const updatedAt = dateNow ?? new Date().toISOString(); const list = await getList({ esClient, id, listIndex }); @@ -72,14 +74,27 @@ export const updateList = async ({ refresh: false, script: { lang: 'painless', - params, + params: { + ...params, + // when assigning undefined in painless, it will remover property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !isPatch, + }, source: ` + if (params.assignEmpty == true || params.containsKey('description')) { ctx._source.description = params.description; + } + if (params.assignEmpty == true || params.containsKey('meta')) { ctx._source.meta = params.meta; + } + if (params.assignEmpty == true || params.containsKey('name')) { ctx._source.name = params.name; - ctx._source.updated_at = params.updated_at; - ctx._source.updated_by = params.updated_by; + } + if (params.assignEmpty == true || params.containsKey('version')) { ctx._source.version = params.version; + } + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; `, }, }); @@ -92,7 +107,7 @@ export const updateList = async ({ deserializer: list.deserializer, id, immutable: list.immutable, - meta, + meta: isPatch ? meta ?? list.meta : meta, name: name ?? list.name, serializer: list.serializer, tie_breaker_id: list.tie_breaker_id, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts index c66fb4ec8b0d2..79217043b36bb 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts @@ -14,6 +14,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_lists_index')); loadTestFile(require.resolve('./create_list_items')); loadTestFile(require.resolve('./patch_lists')); + loadTestFile(require.resolve('./patch_list_items')); loadTestFile(require.resolve('./read_lists')); loadTestFile(require.resolve('./read_list_items')); loadTestFile(require.resolve('./update_lists')); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts new file mode 100644 index 0000000000000..adb60e0b49a75 --- /dev/null +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { + PatchListItemSchema, + CreateListItemSchema, + ListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; +import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; +import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; +import { getUpdateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_item_schema.mock'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +import { + createListsIndex, + deleteListsIndex, + removeListItemServerGeneratedProperties, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const retry = getService('retry'); + + describe.only('patch_list_items', () => { + describe('patch list items', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should patch a single list item property of value using an id', async () => { + const listItemId = getCreateMinimalListItemSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListItemSchemaMock()) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: listItemId, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload); + + const outputListItem: Partial = { + ...getListItemResponseMockWithoutAutoGeneratedValues(), + value: '192.168.0.2', + }; + const bodyToCompare = removeListItemServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(removeListItemServerGeneratedProperties(listItemBody)).to.eql(outputListItem); + return true; + }); + }); + + it('should patch a single list item of value using an auto-generated id of both list and list item', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: createListItemBody.id, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload) + .expect(200); + + const outputListItem: Partial = { + ...getListItemResponseMockWithoutAutoGeneratedValues(), + value: '192.168.0.2', + }; + const bodyToCompare = { + ...removeListItemServerGeneratedProperties(body), + list_id: outputListItem.list_id, + }; + expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: createListItemBody.id }) + .set('kbn-xsrf', 'true'); + const listItemBodyToCompare = { + ...removeListItemServerGeneratedProperties(listItemBody), + list_id: outputListItem.list_id, + }; + expect(listItemBodyToCompare).to.eql(outputListItem); + return true; + }); + }); + + it('should not remove unspecified in patch payload meta property', async () => { + const listItemId = getCreateMinimalListItemSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateMinimalListItemSchemaMock(), meta: { test: true } }) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: listItemId, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload); + + expect(body.meta).to.eql({ test: true }); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(listItemBody.meta).to.eql({ test: true }); + return true; + }); + }); + + it('should give a 404 if it is given a fake id', async () => { + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListItemSchemaMock()) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: 'some-other-id', + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload) + .expect(404); + + expect(body).to.eql({ + status_code: 404, + message: 'list item id: "some-other-id" not found', + }); + }); + }); + }); +}; From ca026c3aca0d7f476bd527a7725df7c78b5591a2 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 12:28:25 +0100 Subject: [PATCH 25/41] remove only --- .../security_and_spaces/tests/patch_list_items.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts index adb60e0b49a75..d2c7f14b24289 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const retry = getService('retry'); - describe.only('patch_list_items', () => { + describe('patch_list_items', () => { describe('patch list items', () => { beforeEach(async () => { await createListsIndex(supertest, log); From 761b5f917f342fd2e0a14f16b0ce9df4ff378fa4 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 13:06:11 +0100 Subject: [PATCH 26/41] fix index create route --- .../lists/server/routes/list/create_list_route.ts | 8 +++++++- .../tests/create_lists_index.ts | 14 +++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lists/server/routes/list/create_list_route.ts b/x-pack/plugins/lists/server/routes/list/create_list_route.ts index 7e793b002495d..df170170fe6db 100644 --- a/x-pack/plugins/lists/server/routes/list/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/create_list_route.ts @@ -38,12 +38,18 @@ export const createListRoute = (router: ListsPluginRouter): void => { request.body; const lists = await getListClient(context); const dataStreamExists = await lists.getListDataStreamExists(); - if (!dataStreamExists) { + const indexExists = await lists.getListIndexExists(); + + if (!dataStreamExists && !indexExists) { return siemResponse.error({ body: `To create a list, the data stream must exist first. Data stream "${lists.getListIndex()}" does not exist`, statusCode: 400, }); } else { + // needs to be migrated to data stream + if (!dataStreamExists && indexExists) { + await lists.migrateListIndexToDataStream(); + } if (id != null) { const list = await lists.getList({ id }); if (list != null) { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts index 569610c91115c..ab549f27c4d2d 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts @@ -28,7 +28,7 @@ export default ({ getService }: FtrProviderContext) => { await deleteListsIndex(supertest, log); }); - it('should create lists indices', async () => { + it('should create lists data streams', async () => { const { body: fetchedIndices } = await supertest .get(LIST_INDEX) .set('kbn-xsrf', 'true') @@ -46,14 +46,15 @@ export default ({ getService }: FtrProviderContext) => { expect(body).to.eql({ list_index: true, list_item_index: true }); }); - it('should update lists indices if old legacy templates exists', async () => { + it('should migrate lists indices to data streams and remove old legacy templates', async () => { // create legacy indices await createLegacyListsIndices(es); - const { body: listsIndex } = await supertest + await supertest .get(LIST_INDEX) .set('kbn-xsrf', 'true') - .expect(200); + // data stream does not exist + .expect(404); // confirm that legacy templates are in use const legacyListsTemplateExists = await getTemplateExists(es, '.lists-default'); @@ -65,10 +66,9 @@ export default ({ getService }: FtrProviderContext) => { expect(legacyItemsTemplateExists).to.equal(true); expect(nonLegacyListsTemplateExists).to.equal(false); expect(nonLegacyItemsTemplateExists).to.equal(false); - expect(listsIndex).to.eql({ list_index: true, list_item_index: true }); - // Expected 409 as index exists already, but now the templates should have been updated - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(409); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); From 7ddf5c05f94edfc7906ab7bd626f06c03ab7b437 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:49:50 +0000 Subject: [PATCH 27/41] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- x-pack/plugins/lists/server/services/items/update_list_item.ts | 2 +- x-pack/plugins/lists/server/services/lists/update_list.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index bd9c2242da505..90b731a06a12e 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -12,7 +12,7 @@ import type { MetaOrUndefined, _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils'; +import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { transformListItemToElasticQuery } from '../utils'; diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 7ad16448f07c5..071c500dc9de4 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -77,7 +77,7 @@ export const updateList = async ({ params: { ...params, // when assigning undefined in painless, it will remover property and wil set it to null - // for patch we don't want to remove unspecified value in payload + // for patch we don't want to remove unspecified value in payload assignEmpty: !isPatch, }, source: ` From 6ccedad1d622ee6acc5e12b8969bba7859143fa9 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 15:47:21 +0100 Subject: [PATCH 28/41] fix typings --- x-pack/plugins/lists/server/services/utils/find_source_value.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lists/server/services/utils/find_source_value.ts b/x-pack/plugins/lists/server/services/utils/find_source_value.ts index ed39b51dac6c6..5d03c8f6707b3 100644 --- a/x-pack/plugins/lists/server/services/utils/find_source_value.ts +++ b/x-pack/plugins/lists/server/services/utils/find_source_value.ts @@ -23,7 +23,7 @@ export const findSourceValue = ( const foundEntry = Object.entries(listItem).find( ([key, value]) => types.includes(key) && value != null ); - if (foundEntry != null) { + if (foundEntry != null && foundEntry[1] !== null) { const [foundType, value] = foundEntry; switch (foundType) { case 'shape': From ffeb82998975e2eb4791d6d07c50e3410f0d2481 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Tue, 1 Aug 2023 18:33:14 +0100 Subject: [PATCH 29/41] OCC --- .../server/services/lists/delete_list.ts | 20 +++++- .../server/services/lists/update_list.ts | 31 ++++++++- .../lists/server/services/utils/index.ts | 1 + .../utils/wait_until_document_indexed.ts | 19 ++++++ .../security_and_spaces/tests/update_lists.ts | 65 +++++++++++++++++++ 5 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.ts b/x-pack/plugins/lists/server/services/lists/delete_list.ts index 661a5ae999644..c6b452675b7c3 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.ts @@ -8,6 +8,8 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Id, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { waitUntilDocumentIndexed } from '../utils'; + import { getList } from './get_list'; export interface DeleteListOptions { @@ -35,11 +37,12 @@ export const deleteList = async ({ }, }, }, + conflicts: 'proceed', index: listItemIndex, refresh: false, }); - await esClient.deleteByQuery({ + const response = await esClient.deleteByQuery({ body: { query: { ids: { @@ -47,9 +50,24 @@ export const deleteList = async ({ }, }, }, + conflicts: 'proceed', index: listIndex, refresh: false, }); + + if (response.deleted) { + const checkIfListDeleted = async (): Promise => { + const deletedList = await getList({ esClient, id, listIndex }); + if (deletedList !== null) { + throw Error('List has not been re-indexed in time'); + } + }; + + await waitUntilDocumentIndexed(checkIfListDeleted); + } else { + throw Error('No list has been deleted'); + } + return list; } }; diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 071c500dc9de4..ed37b80684f0a 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import Boom from '@hapi/boom'; import { ElasticsearchClient } from '@kbn/core/server'; import type { DescriptionOrUndefined, @@ -15,9 +15,10 @@ import type { _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; +import { decodeVersion } from '@kbn/securitysolution-es-utils'; import { UpdateEsListSchema } from '../../schemas/elastic_query'; +import { waitUntilDocumentIndexed } from '../utils'; import { getList } from '.'; @@ -36,6 +37,7 @@ export interface UpdateListOptions { } export const updateList = async ({ + _version, id, name, description, @@ -52,6 +54,13 @@ export const updateList = async ({ if (list == null) { return null; } else { + if (_version && list._version && _version !== list._version) { + throw Boom.conflict( + `Conflict: versions mismatch. Provided versions:${JSON.stringify( + decodeVersion(_version) + )} does not match ${JSON.stringify(decodeVersion(list._version))}` + ); + } const calculatedVersion = version == null ? list.version + 1 : version; const params: UpdateEsListSchema = { @@ -98,9 +107,25 @@ export const updateList = async ({ `, }, }); + + let updatedOCCVersion: string | undefined; + if (response.updated) { + const checkIfLisUpdated = async (): Promise => { + const updatedList = await getList({ esClient, id, listIndex }); + if (updatedList?._version === list._version) { + throw Error('Document has not been re-indexed in time'); + } + updatedOCCVersion = updatedList?._version; + }; + + await waitUntilDocumentIndexed(checkIfLisUpdated); + } else { + throw Error('No list has been updated'); + } + return { '@timestamp': list['@timestamp'], - _version: encodeHitVersion(response), + _version: updatedOCCVersion, created_at: list.created_at, created_by: list.created_by, description: description ?? list.description, diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts index 64e7c50d0e7b0..6dea49102596b 100644 --- a/x-pack/plugins/lists/server/services/utils/index.ts +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -21,3 +21,4 @@ export * from './transform_elastic_named_search_to_list_item'; export * from './transform_elastic_to_list_item'; export * from './transform_elastic_to_list'; export * from './transform_list_item_to_elastic_query'; +export * from './wait_until_document_indexed'; diff --git a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts new file mode 100644 index 0000000000000..3e5679a220859 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pRetry from 'p-retry'; + +// index.refresh_interval +// https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings +const DEFAULT_INDEX_REFRESH_TIME = 1000; + +export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise => { + await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); + await pRetry(fn, { + minTimeout: DEFAULT_INDEX_REFRESH_TIME, + retries: 3, + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts index b98688f209d00..46c9c1341a817 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts @@ -212,6 +212,71 @@ export default ({ getService }: FtrProviderContext) => { message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', }); }); + + describe('version control OCC', () => { + it('should return error if _version in payload mismatched', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + _version: createListBody._version, + }; + await supertest.put(LIST_URL).set('kbn-xsrf', 'true').send(updatedList).expect(200); + + // next update with the same _version should return 409 + const { body: errorBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(409); + + expect(errorBody.message).to.equal( + 'Conflict: versions mismatch. Provided versions:{"if_primary_term":1,"if_seq_no":0} does not match {"if_primary_term":1,"if_seq_no":1}' + ); + }); + + it('should return updated _version', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + _version: createListBody._version, + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + + // version should be different + expect(updatedListBody._version).not.to.be(createListBody._version); + + // next update with the new version should be successful + await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...updatedList, _version: updatedListBody._version }) + .expect(200); + }); + }); }); }); }; From 43f55d9844410a67a3cf8faa8f727b3312f5412f Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 2 Aug 2023 11:01:42 +0100 Subject: [PATCH 30/41] tests --- .../services/items/update_list_item.test.ts | 14 ---------- .../server/services/lists/delete_list.test.ts | 18 +++++++++++++ .../server/services/lists/update_list.test.ts | 24 ++++++++++++++--- .../server/services/lists/update_list.ts | 17 ++++-------- .../services/utils/check_version_conflict.ts | 26 +++++++++++++++++++ .../lists/server/services/utils/index.ts | 1 + 6 files changed, 70 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugins/lists/server/services/utils/check_version_conflict.ts diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 14fe97638e710..7b9e1e64b4fcd 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -27,20 +27,6 @@ describe('update_list_item', () => { jest.clearAllMocks(); }); - test('it returns a list item as expected with the id changed out for the elastic id when there is a list item to update', async () => { - const listItem = getListItemResponseMock(); - (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); - const options = getUpdateListItemOptionsMock(); - const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); - const updatedList = await updateListItem({ ...options, esClient }); - const expected: ListItemSchema = { ...getListItemResponseMock(), id: 'elastic-id-123' }; - expect(updatedList).toEqual(expected); - }); - test('it returns null when there is not a list item to update', async () => { (getListItem as unknown as jest.Mock).mockResolvedValueOnce(null); const options = getUpdateListItemOptionsMock(); diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts index c00815625323a..dd2a8639e9278 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts @@ -12,6 +12,10 @@ import { getList } from './get_list'; import { deleteList } from './delete_list'; import { getDeleteListOptionsMock } from './delete_list.mock'; +jest.mock('../utils', () => ({ + waitUntilDocumentIndexed: jest.fn(), +})); + jest.mock('./get_list', () => ({ getList: jest.fn(), })); @@ -36,6 +40,7 @@ describe('delete_list', () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); const deletedList = await deleteList(options); expect(deletedList).toEqual(list); }); @@ -44,9 +49,11 @@ describe('delete_list', () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); await deleteList(options); const deleteByQuery = { body: { query: { term: { list_id: LIST_ID } } }, + conflicts: 'proceed', index: LIST_ITEM_INDEX, refresh: false, }; @@ -57,6 +64,7 @@ describe('delete_list', () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); await deleteList(options); const deleteByQuery = { body: { @@ -66,6 +74,7 @@ describe('delete_list', () => { }, }, }, + conflicts: 'proceed', index: LIST_INDEX, refresh: false, }; @@ -78,4 +87,13 @@ describe('delete_list', () => { await deleteList(options); expect(options.esClient.delete).not.toHaveBeenCalled(); }); + + test('throw error if no list was deleted', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 0 }); + + await expect(deleteList(options)).rejects.toThrow('No list has been deleted'); + }); }); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts index 338f3eab476d0..623f594fd1deb 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -14,6 +14,11 @@ import { updateList } from './update_list'; import { getList } from './get_list'; import { getUpdateListOptionsMock } from './update_list.mock'; +jest.mock('../utils', () => ({ + checkVersionConflict: jest.fn(), + waitUntilDocumentIndexed: jest.fn(), +})); + jest.mock('./get_list', () => ({ getList: jest.fn(), })); @@ -35,11 +40,9 @@ describe('update_list', () => { }; (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getUpdateListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); + esClient.updateByQuery.mockResolvedValue({ updated: 1 }); const updatedList = await updateList({ ...options, esClient }); const expected: ListSchema = { ...getListResponseMock(), @@ -56,4 +59,17 @@ describe('update_list', () => { const updatedList = await updateList(options); expect(updatedList).toEqual(null); }); + + test('throw error if no list was updated', async () => { + const list: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getUpdateListOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResolvedValue({ updated: 0 }); + await expect(updateList({ ...options, esClient })).rejects.toThrow('No list has been updated'); + }); }); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index ed37b80684f0a..bed385ef6729e 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import Boom from '@hapi/boom'; + import { ElasticsearchClient } from '@kbn/core/server'; import type { DescriptionOrUndefined, @@ -15,10 +15,9 @@ import type { _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { decodeVersion } from '@kbn/securitysolution-es-utils'; import { UpdateEsListSchema } from '../../schemas/elastic_query'; -import { waitUntilDocumentIndexed } from '../utils'; +import { checkVersionConflict, waitUntilDocumentIndexed } from '../utils'; import { getList } from '.'; @@ -54,13 +53,7 @@ export const updateList = async ({ if (list == null) { return null; } else { - if (_version && list._version && _version !== list._version) { - throw Boom.conflict( - `Conflict: versions mismatch. Provided versions:${JSON.stringify( - decodeVersion(_version) - )} does not match ${JSON.stringify(decodeVersion(list._version))}` - ); - } + checkVersionConflict(_version, list._version); const calculatedVersion = version == null ? list.version + 1 : version; const params: UpdateEsListSchema = { @@ -110,7 +103,7 @@ export const updateList = async ({ let updatedOCCVersion: string | undefined; if (response.updated) { - const checkIfLisUpdated = async (): Promise => { + const checkIfListUpdated = async (): Promise => { const updatedList = await getList({ esClient, id, listIndex }); if (updatedList?._version === list._version) { throw Error('Document has not been re-indexed in time'); @@ -118,7 +111,7 @@ export const updateList = async ({ updatedOCCVersion = updatedList?._version; }; - await waitUntilDocumentIndexed(checkIfLisUpdated); + await waitUntilDocumentIndexed(checkIfListUpdated); } else { throw Error('No list has been updated'); } diff --git a/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts b/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts new file mode 100644 index 0000000000000..66bebdae9ec05 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Boom from '@hapi/boom'; +import { decodeVersion } from '@kbn/securitysolution-es-utils'; + +/** + * checks if encoded OCC update _version matches actual version of list/item + * @param updateVersion - version in payload + * @param existingVersion - version in exiting list/item + */ +export const checkVersionConflict = ( + updateVersion: string | undefined, + existingVersion: string | undefined +): void => { + if (updateVersion && existingVersion && updateVersion !== existingVersion) { + throw Boom.conflict( + `Conflict: versions mismatch. Provided versions:${JSON.stringify( + decodeVersion(updateVersion) + )} does not match ${JSON.stringify(decodeVersion(existingVersion))}` + ); + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts index 6dea49102596b..ecd2bbc5f469f 100644 --- a/x-pack/plugins/lists/server/services/utils/index.ts +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -8,6 +8,7 @@ export * from './calculate_scroll_math'; export * from './encode_decode_cursor'; export * from './escape_query'; +export * from './check_version_conflict'; export * from './find_source_type'; export * from './find_source_value'; export * from './get_query_filter_from_type_value'; From dfbf25027ba3705ee9b469ce89e1907b10a156ad Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 2 Aug 2023 10:08:24 +0000 Subject: [PATCH 31/41] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- .../plugins/lists/server/services/items/update_list_item.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 7b9e1e64b4fcd..fe89ec2a274e1 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import type { ListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; From c8ea9acef0a67b064d387a740a5c301d548e3197 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 2 Aug 2023 11:29:27 +0100 Subject: [PATCH 32/41] addmore tests --- .../services/items/update_list_item.test.ts | 36 +++++++- .../server/services/items/update_list_item.ts | 26 +++++- .../server/services/lists/update_list.test.ts | 1 - .../tests/update_list_items.ts | 88 +++++++++++++++++++ 4 files changed, 143 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 7b9e1e64b4fcd..46e8507166662 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -14,6 +14,12 @@ import { updateListItem } from './update_list_item'; import { getListItem } from './get_list_item'; import { getUpdateListItemOptionsMock } from './update_list_item.mock'; +jest.mock('../utils/check_version_conflict', () => ({ + checkVersionConflict: jest.fn(), +})); +jest.mock('../utils/wait_until_document_indexed', () => ({ + waitUntilDocumentIndexed: jest.fn(), +})); jest.mock('./get_list_item', () => ({ getListItem: jest.fn(), })); @@ -27,11 +33,22 @@ describe('update_list_item', () => { jest.clearAllMocks(); }); + test('it returns a list item when updated', async () => { + const listItem = getListItemResponseMock(); + (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); + const options = getUpdateListItemOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResponse({ updated: 1 }); + const updatedListItem = await updateListItem({ ...options, esClient }); + const expected: ListItemSchema = getListItemResponseMock(); + expect(updatedListItem).toEqual(expected); + }); + test('it returns null when there is not a list item to update', async () => { (getListItem as unknown as jest.Mock).mockResolvedValueOnce(null); const options = getUpdateListItemOptionsMock(); - const updatedList = await updateListItem(options); - expect(updatedList).toEqual(null); + const updatedListItem = await updateListItem(options); + expect(updatedListItem).toEqual(null); }); test('it returns null when the serializer and type such as ip_range returns nothing', async () => { @@ -43,7 +60,18 @@ describe('update_list_item', () => { }; (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); const options = getUpdateListItemOptionsMock(); - const updatedList = await updateListItem(options); - expect(updatedList).toEqual(null); + const updatedListItem = await updateListItem(options); + expect(updatedListItem).toEqual(null); + }); + + test('throw error if no list item was updated', async () => { + const listItem = getListItemResponseMock(); + (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); + const options = getUpdateListItemOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResponse({ updated: 0 }); + await expect(updateListItem({ ...options, esClient })).rejects.toThrow( + 'No list item has been updated' + ); }); }); diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 90b731a06a12e..1f52a47680325 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -12,9 +12,12 @@ import type { MetaOrUndefined, _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; -import { transformListItemToElasticQuery } from '../utils'; +import { + checkVersionConflict, + transformListItemToElasticQuery, + waitUntilDocumentIndexed, +} from '../utils'; import { getListItem } from './get_list_item'; @@ -54,6 +57,7 @@ export const updateListItem = async ({ if (elasticQuery == null) { return null; } else { + checkVersionConflict(_version, listItem._version); const keyValues = Object.entries(elasticQuery).map(([key, keyValue]) => ({ key, value: keyValue, @@ -94,9 +98,25 @@ export const updateListItem = async ({ `, }, }); + + let updatedOCCVersion: string | undefined; + if (response.updated) { + const checkIfListUpdated = async (): Promise => { + const updatedListItem = await getListItem({ esClient, id, listItemIndex }); + if (updatedListItem?._version === listItem._version) { + throw Error('List item has not been re-indexed in time'); + } + updatedOCCVersion = updatedListItem?._version; + }; + + await waitUntilDocumentIndexed(checkIfListUpdated); + } else { + throw Error('No list item has been updated'); + } + return { '@timestamp': listItem['@timestamp'], - _version: encodeHitVersion(response), + _version: updatedOCCVersion, created_at: listItem.created_at, created_by: listItem.created_by, deserializer: listItem.deserializer, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts index 623f594fd1deb..4a088e01910de 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -40,7 +40,6 @@ describe('update_list', () => { }; (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getUpdateListOptionsMock(); - options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; esClient.updateByQuery.mockResolvedValue({ updated: 1 }); const updatedList = await updateList({ ...options, esClient }); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts index ebceaf76ba486..7a1bfb3f12698 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts @@ -213,6 +213,94 @@ export default ({ getService }: FtrProviderContext) => { message: 'list item id: "some-other-id" not found', }); }); + + describe('version control OCC', () => { + it('should return error if _version in payload mismatched', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: createListItemBody.id, + value: '192.168.0.2', + _version: createListItemBody._version, + }; + await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + // next update with the same _version should return 409 + const { body: errorBody } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(409); + + expect(errorBody.message).to.equal( + 'Conflict: versions mismatch. Provided versions:{"if_primary_term":1,"if_seq_no":0} does not match {"if_primary_term":1,"if_seq_no":1}' + ); + }); + + it('should return updated _version', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: createListItemBody.id, + value: '192.168.0.2', + _version: createListItemBody._version, + }; + const { body: updatedListItemBody } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + // next update with the new version should be successful + await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...updatedListItem, _version: updatedListItemBody._version }) + .expect(200); + }); + }); }); }); }; From 204881065302797b8fe6e63d77ecedb59934d552 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 2 Aug 2023 12:07:33 +0100 Subject: [PATCH 33/41] fix unit test --- .../lists/server/services/items/update_list_item.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 3d380eb8dcbd7..d7bff1c11c5df 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import type { ListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock'; From 307bfc21598d46409004ab4abe33a12cebc118b9 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 2 Aug 2023 13:24:22 +0100 Subject: [PATCH 34/41] renaming --- .../server/routes/list/create_list_route.ts | 2 +- .../routes/list/import_list_item_route.ts | 2 +- .../list_index/create_list_index_route.ts | 2 +- .../list_index/delete_list_index_route.ts | 2 +- .../list_index/read_list_index_route.ts | 6 +- .../read_list_privileges_route.ts | 4 +- .../server/services/lists/list_client.mock.ts | 4 +- .../server/services/lists/list_client.test.ts | 4 +- .../server/services/lists/list_client.ts | 244 +++++++++--------- .../threat_mapping/get_threat_list.ts | 2 +- 10 files changed, 135 insertions(+), 137 deletions(-) diff --git a/x-pack/plugins/lists/server/routes/list/create_list_route.ts b/x-pack/plugins/lists/server/routes/list/create_list_route.ts index df170170fe6db..7a1eef80aa16c 100644 --- a/x-pack/plugins/lists/server/routes/list/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/create_list_route.ts @@ -42,7 +42,7 @@ export const createListRoute = (router: ListsPluginRouter): void => { if (!dataStreamExists && !indexExists) { return siemResponse.error({ - body: `To create a list, the data stream must exist first. Data stream "${lists.getListIndex()}" does not exist`, + body: `To create a list, the data stream must exist first. Data stream "${lists.getListName()}" does not exist`, statusCode: 400, }); } else { diff --git a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts index ea5dbca235654..bb79fcaa40059 100644 --- a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts @@ -46,7 +46,7 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp const listExists = await lists.getListDataStreamExists(); if (!listExists) { return siemResponse.error({ - body: `To import a list item, the data steam must exist first. Data stream "${lists.getListIndex()}" does not exist`, + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListName()}" does not exist`, statusCode: 400, }); } diff --git a/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts index 3016313b57584..9116e5d338e5e 100644 --- a/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts @@ -47,7 +47,7 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => { if (listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ - body: `data stream: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, + body: `data stream: "${lists.getListName()}" and "${lists.getListItemName()}" already exists`, statusCode: 409, }); } diff --git a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts index 8a3a2356d3e3e..ff83a3451217c 100644 --- a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -55,7 +55,7 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => { await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists); } else if (!listIndexExists && !listItemIndexExists) { return siemResponse.error({ - body: `index and data stream: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`, + body: `index and data stream: "${lists.getListName()}" and "${lists.getListItemName()}" does not exist`, statusCode: 404, }); } else { diff --git a/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts index 909f3ed564b4c..4530f9e68febb 100644 --- a/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts @@ -43,17 +43,17 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { } } else if (!listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ - body: `data stream ${lists.getListIndex()} does not exist`, + body: `data stream ${lists.getListName()} does not exist`, statusCode: 404, }); } else if (!listItemDataStreamExists && listDataStreamExists) { return siemResponse.error({ - body: `data stream ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListItemName()} does not exist`, statusCode: 404, }); } else { return siemResponse.error({ - body: `data stream ${lists.getListIndex()} and data stream ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListName()} and data stream ${lists.getListItemName()} does not exist`, statusCode: 404, }); } diff --git a/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts index 1b4e254bf5aad..8a6c404220bbb 100644 --- a/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts +++ b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts @@ -26,8 +26,8 @@ export const readPrivilegesRoute = (router: ListsPluginRouter): void => { try { const esClient = (await context.core).elasticsearch.client.asCurrentUser; const lists = await getListClient(context); - const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListIndex()); - const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemIndex()); + const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListName()); + const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemName()); const privileges = merge( { listItems: clusterPrivilegesListItems, diff --git a/x-pack/plugins/lists/server/services/lists/list_client.mock.ts b/x-pack/plugins/lists/server/services/lists/list_client.mock.ts index a2f11e2ac1202..cd361526d2813 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.mock.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.mock.ts @@ -24,8 +24,8 @@ import { import { ListClient } from './list_client'; export class ListClientMock extends ListClient { - public getListIndex = jest.fn().mockReturnValue(LIST_INDEX); - public getListItemIndex = jest.fn().mockReturnValue(LIST_ITEM_INDEX); + public getListName = jest.fn().mockReturnValue(LIST_INDEX); + public getListItemName = jest.fn().mockReturnValue(LIST_ITEM_INDEX); public getList = jest.fn().mockResolvedValue(getListResponseMock()); public createList = jest.fn().mockResolvedValue(getListResponseMock()); public createListIfItDoesNotExist = jest.fn().mockResolvedValue(getListResponseMock()); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.test.ts b/x-pack/plugins/lists/server/services/lists/list_client.test.ts index 7259f56b9a91b..e6d6c069ccb3a 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.test.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.test.ts @@ -14,12 +14,12 @@ describe('list_client', () => { describe('Mock client checks (not exhaustive tests against it)', () => { test('it returns the get list index as expected', () => { const mock = getListClientMock(); - expect(mock.getListIndex()).toEqual(LIST_INDEX); + expect(mock.getListName()).toEqual(LIST_INDEX); }); test('it returns the get list item index as expected', () => { const mock = getListClientMock(); - expect(mock.getListItemIndex()).toEqual(LIST_ITEM_INDEX); + expect(mock.getListItemName()).toEqual(LIST_ITEM_INDEX); }); test('it returns a mock list item', async () => { diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index ec7231712cf29..62c74ab121dbf 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -122,10 +122,10 @@ export class ListClient { } /** - * Returns the list index name - * @returns The list index name + * Returns the list data stream or index name + * @returns The list data stream/index name */ - public getListIndex = (): string => { + public getListName = (): string => { const { spaceId, config: { listIndex: listsIndexName }, @@ -134,10 +134,10 @@ export class ListClient { }; /** - * Returns the list item index name - * @returns The list item index name + * Returns the list item data stream or index name + * @returns The list item data stream/index name */ - public getListItemIndex = (): string => { + public getListItemName = (): string => { const { spaceId, config: { listItemIndex: listsItemsIndexName }, @@ -153,8 +153,8 @@ export class ListClient { */ public getList = async ({ id }: GetListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getList({ esClient, id, listIndex }); + const listName = this.getListName(); + return getList({ esClient, id, listIndex: listName }); }; /** @@ -185,14 +185,14 @@ export class ListClient { version, }: CreateListOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return createList({ description, deserializer, esClient, id, immutable, - listIndex, + listIndex: listName, meta, name, serializer, @@ -232,14 +232,14 @@ export class ListClient { version, }: CreateListIfItDoesNotExistOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return createListIfItDoesNotExist({ description, deserializer, esClient, id, immutable, - listIndex, + listIndex: listName, meta, name, serializer, @@ -252,12 +252,11 @@ export class ListClient { /** * True if the list index exists, otherwise false * @returns True if the list index exists, otherwise false - * @deprecated */ public getListIndexExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getBootstrapIndexExists(esClient, listIndex); + const listName = this.getListName(); + return getBootstrapIndexExists(esClient, listName); }; /** @@ -266,19 +265,18 @@ export class ListClient { */ public getListDataStreamExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getDataStreamExists(esClient, listIndex); + const listName = this.getListName(); + return getDataStreamExists(esClient, listName); }; /** * True if the list index item exists, otherwise false * @returns True if the list item index exists, otherwise false - * @deprecated */ public getListItemIndexExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getBootstrapIndexExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getBootstrapIndexExists(esClient, listItemName); }; /** @@ -287,19 +285,19 @@ export class ListClient { */ public getListItemDataStreamExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getDataStreamExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getDataStreamExists(esClient, listItemName); }; /** * Creates the list boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch - * @deprecated + * @deprecated after moving to data streams there should not be need to use it */ public createListBootStrapIndex = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return createBootstrapIndex(esClient, listIndex); + const listName = this.getListName(); + return createBootstrapIndex(esClient, listName); }; /** @@ -308,8 +306,8 @@ export class ListClient { */ public createListDataStream = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return createDataStream(esClient, listIndex); + const listName = this.getListName(); + return createDataStream(esClient, listName); }; /** @@ -318,14 +316,14 @@ export class ListClient { */ public migrateListIndexToDataStream = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); // first need to update mapping of existing index to add @timestamp await putMappings( esClient, - listIndex, + listName, listMappings.properties as Record ); - return migrateToDataStream(esClient, listIndex); + return migrateToDataStream(esClient, listName); }; /** @@ -334,25 +332,25 @@ export class ListClient { */ public migrateListItemIndexToDataStream = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); // first need to update mapping of existing index to add @timestamp await putMappings( esClient, - listItemIndex, + listItemName, listItemMappings.properties as Record ); - return migrateToDataStream(esClient, listItemIndex); + return migrateToDataStream(esClient, listItemName); }; /** * Creates the list item boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch - * @deprecated + * @deprecated after moving to data streams there should not be need to use it */ public createListItemBootStrapIndex = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return createBootstrapIndex(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return createBootstrapIndex(esClient, listItemName); }; /** @@ -361,8 +359,8 @@ export class ListClient { */ public createListItemDataStream = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return createDataStream(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return createDataStream(esClient, listItemName); }; /** @@ -371,8 +369,8 @@ export class ListClient { */ public getListPolicyExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getPolicyExists(esClient, listIndex); + const listName = this.getListName(); + return getPolicyExists(esClient, listName); }; /** @@ -381,7 +379,7 @@ export class ListClient { */ public getListItemPolicyExists = async (): Promise => { const { esClient } = this; - const listsItemIndex = this.getListItemIndex(); + const listsItemIndex = this.getListItemName(); return getPolicyExists(esClient, listsItemIndex); }; @@ -391,8 +389,8 @@ export class ListClient { */ public getListTemplateExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getIndexTemplateExists(esClient, listIndex); + const listName = this.getListName(); + return getIndexTemplateExists(esClient, listName); }; /** @@ -401,8 +399,8 @@ export class ListClient { */ public getListItemTemplateExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getIndexTemplateExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getIndexTemplateExists(esClient, listItemName); }; /** @@ -411,8 +409,8 @@ export class ListClient { */ public getLegacyListTemplateExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getTemplateExists(esClient, listIndex); + const listName = this.getListName(); + return getTemplateExists(esClient, listName); }; /** @@ -421,8 +419,8 @@ export class ListClient { */ public getLegacyListItemTemplateExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getTemplateExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getTemplateExists(esClient, listItemName); }; /** @@ -430,8 +428,8 @@ export class ListClient { * @returns The contents of the list template for ILM. */ public getListTemplate = (): Record => { - const listIndex = this.getListIndex(); - return getListTemplate(listIndex); + const listName = this.getListName(); + return getListTemplate(listName); }; /** @@ -439,8 +437,8 @@ export class ListClient { * @returns The contents of the list item template for ILM. */ public getListItemTemplate = (): Record => { - const listItemIndex = this.getListItemIndex(); - return getListItemTemplate(listItemIndex); + const listItemName = this.getListItemName(); + return getListItemTemplate(listItemName); }; /** @@ -450,8 +448,8 @@ export class ListClient { public setListTemplate = async (): Promise => { const { esClient } = this; const template = this.getListTemplate(); - const listIndex = this.getListIndex(); - return setIndexTemplate(esClient, listIndex, template); + const listName = this.getListName(); + return setIndexTemplate(esClient, listName, template); }; /** @@ -461,8 +459,8 @@ export class ListClient { public setListItemTemplate = async (): Promise => { const { esClient } = this; const template = this.getListItemTemplate(); - const listItemIndex = this.getListItemIndex(); - return setIndexTemplate(esClient, listItemIndex, template); + const listItemName = this.getListItemName(); + return setIndexTemplate(esClient, listItemName, template); }; /** @@ -471,8 +469,8 @@ export class ListClient { */ public setListPolicy = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return setPolicy(esClient, listIndex, listPolicy); + const listName = this.getListName(); + return setPolicy(esClient, listName, listPolicy); }; /** @@ -481,8 +479,8 @@ export class ListClient { */ public setListItemPolicy = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return setPolicy(esClient, listItemIndex, listsItemsPolicy); + const listItemName = this.getListItemName(); + return setPolicy(esClient, listItemName, listsItemsPolicy); }; /** @@ -491,8 +489,8 @@ export class ListClient { */ public deleteListIndex = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteAllIndex(esClient, `${listIndex}-*`); + const listName = this.getListName(); + return deleteAllIndex(esClient, `${listName}-*`); }; /** @@ -501,8 +499,8 @@ export class ListClient { */ public deleteListItemIndex = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteAllIndex(esClient, `${listItemIndex}-*`); + const listItemName = this.getListItemName(); + return deleteAllIndex(esClient, `${listItemName}-*`); }; /** @@ -511,8 +509,8 @@ export class ListClient { */ public deleteListDataStream = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteDataStream(esClient, listIndex); + const listName = this.getListName(); + return deleteDataStream(esClient, listName); }; /** @@ -521,8 +519,8 @@ export class ListClient { */ public deleteListItemDataStream = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteDataStream(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deleteDataStream(esClient, listItemName); }; /** @@ -531,8 +529,8 @@ export class ListClient { */ public deleteListPolicy = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deletePolicy(esClient, listIndex); + const listName = this.getListName(); + return deletePolicy(esClient, listName); }; /** @@ -541,8 +539,8 @@ export class ListClient { */ public deleteListItemPolicy = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deletePolicy(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deletePolicy(esClient, listItemName); }; /** @@ -551,8 +549,8 @@ export class ListClient { */ public deleteListTemplate = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteIndexTemplate(esClient, listIndex); + const listName = this.getListName(); + return deleteIndexTemplate(esClient, listName); }; /** @@ -561,8 +559,8 @@ export class ListClient { */ public deleteListItemTemplate = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteIndexTemplate(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deleteIndexTemplate(esClient, listItemName); }; /** @@ -571,8 +569,8 @@ export class ListClient { */ public deleteLegacyListTemplate = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteTemplate(esClient, listIndex); + const listName = this.getListName(); + return deleteTemplate(esClient, listName); }; /** @@ -581,8 +579,8 @@ export class ListClient { */ public deleteLegacyListItemTemplate = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteTemplate(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deleteTemplate(esClient, listItemName); }; /** @@ -591,8 +589,8 @@ export class ListClient { */ public deleteListItem = async ({ id }: DeleteListItemOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteListItem({ esClient, id, listItemIndex }); + const listItemName = this.getListItemName(); + return deleteListItem({ esClient, id, listItemIndex: listItemName }); }; /** @@ -609,11 +607,11 @@ export class ListClient { type, }: DeleteListItemByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return deleteListItemByValue({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -627,13 +625,13 @@ export class ListClient { */ public deleteList = async ({ id }: DeleteListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return deleteList({ esClient, id, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, }); }; @@ -650,11 +648,11 @@ export class ListClient { stream, }: ExportListItemsToStreamOptions): void => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); exportListItemsToStream({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, stream, stringToAppend, }); @@ -683,15 +681,15 @@ export class ListClient { version, }: ImportListItemsToStreamOptions): Promise => { const { esClient, user, config } = this; - const listItemIndex = this.getListItemIndex(); - const listIndex = this.getListIndex(); + const listItemName = this.getListItemName(); + const listName = this.getListName(); return importListItemsToStream({ config, deserializer, esClient, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, meta, serializer, stream, @@ -715,11 +713,11 @@ export class ListClient { type, }: GetListItemByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItemByValue({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -749,13 +747,13 @@ export class ListClient { meta, }: CreateListItemOptions): Promise => { const { esClient, user } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return createListItem({ deserializer, esClient, id, listId, - listItemIndex, + listItemIndex: listItemName, meta, serializer, type, @@ -781,13 +779,13 @@ export class ListClient { meta, }: UpdateListItemOptions): Promise => { const { esClient, user } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return updateListItem({ _version, esClient, id, isPatch: false, - listItemIndex, + listItemIndex: listItemName, meta, user, value, @@ -811,13 +809,13 @@ export class ListClient { meta, }: UpdateListItemOptions): Promise => { const { esClient, user } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return updateListItem({ _version, esClient, id, isPatch: true, - listItemIndex, + listItemIndex: listItemName, meta, user, value, @@ -845,14 +843,14 @@ export class ListClient { version, }: UpdateListOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return updateList({ _version, description, esClient, id, isPatch: false, - listIndex, + listIndex: listName, meta, name, user, @@ -879,14 +877,14 @@ export class ListClient { version, }: UpdateListOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return updateList({ _version, description, esClient, id, isPatch: true, - listIndex, + listIndex: listName, meta, name, user, @@ -902,11 +900,11 @@ export class ListClient { */ public getListItem = async ({ id }: GetListItemOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItem({ esClient, id, - listItemIndex, + listItemIndex: listItemName, }); }; @@ -924,11 +922,11 @@ export class ListClient { value, }: GetListItemsByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItemByValues({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -948,11 +946,11 @@ export class ListClient { value, }: SearchListItemByValuesOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return searchListItemByValues({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -982,12 +980,12 @@ export class ListClient { runtimeMappings, }: FindListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return findList({ currentIndexPosition, esClient, filter, - listIndex, + listIndex: listName, page, perPage, runtimeMappings, @@ -1024,15 +1022,15 @@ export class ListClient { searchAfter, }: FindListItemOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return findListItem({ currentIndexPosition, esClient, filter, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, page, perPage, runtimeMappings, @@ -1049,14 +1047,14 @@ export class ListClient { sortOrder, }: FindAllListItemsOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return findAllListItems({ esClient, filter, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, sortField, sortOrder, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts index 73301b2c4dfb5..9009d20c379df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts @@ -64,7 +64,7 @@ export const getThreatList = async ({ runtime_mappings: runtimeMappings, sort: getSortForThreatList({ index, - listItemIndex: listClient.getListItemIndex(), + listItemIndex: listClient.getListItemName(), }), }, track_total_hits: false, From c142a1066cb8222c37a83a5cbe58010e537fe706 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Wed, 2 Aug 2023 16:40:58 +0100 Subject: [PATCH 35/41] add comments --- .../src/create_data_stream/index.ts | 1 - .../src/delete_data_stream/index.ts | 5 +++++ .../src/get_data_stream_exists/index.ts | 3 +-- .../src/migrate_to_data_stream/index.ts | 1 - .../kbn-securitysolution-es-utils/src/put_mappings/index.ts | 6 ++++++ .../plugins/lists/server/services/items/update_list_item.ts | 2 +- x-pack/plugins/lists/server/services/lists/update_list.ts | 2 +- .../server/services/utils/wait_until_document_indexed.ts | 6 ++++++ 8 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts index 28b65f5882f94..95427cc7aec56 100644 --- a/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts @@ -12,7 +12,6 @@ import type { ElasticsearchClient } from '../elasticsearch_client'; * creates data stream * @param esClient * @param name - * @returns */ export const createDataStream = async ( esClient: ElasticsearchClient, diff --git a/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts index 198f41399ece2..23009be11165e 100644 --- a/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts @@ -8,6 +8,11 @@ import type { ElasticsearchClient } from '../elasticsearch_client'; +/** + * deletes data stream + * @param esClient + * @param name + */ export const deleteDataStream = async ( esClient: ElasticsearchClient, name: string diff --git a/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts index 2496aa285d4a2..8d5419405147a 100644 --- a/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts @@ -9,10 +9,9 @@ import type { ElasticsearchClient } from '../elasticsearch_client'; /** - * creates data stream + * checks if data stream exists * @param esClient * @param name - * @returns */ export const getDataStreamExists = async ( esClient: ElasticsearchClient, diff --git a/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts index ed107b1ade0de..1ec831846e38f 100644 --- a/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts @@ -12,7 +12,6 @@ import type { ElasticsearchClient } from '../elasticsearch_client'; * migrate to data stream * @param esClient * @param name - * @returns */ export const migrateToDataStream = async ( esClient: ElasticsearchClient, diff --git a/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts index 7312813ceefdf..8a7c6c2b51a90 100644 --- a/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts +++ b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts @@ -8,6 +8,12 @@ import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '../elasticsearch_client'; +/** + * update mappings of index + * @param esClient + * @param index + * @param mappings + */ export const putMappings = async ( esClient: ElasticsearchClient, index: string, diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 1f52a47680325..ad3f9db5422e0 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -64,7 +64,7 @@ export const updateListItem = async ({ })); const params = { - // when assigning undefined in painless, it will remover property and wil set it to null + // when assigning undefined in painless, it will remove property and wil set it to null // for patch we don't want to remove unspecified value in payload assignEmpty: !isPatch, keyValues, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index bed385ef6729e..9dc2497bb70c2 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -78,7 +78,7 @@ export const updateList = async ({ lang: 'painless', params: { ...params, - // when assigning undefined in painless, it will remover property and wil set it to null + // when assigning undefined in painless, it will remove property and wil set it to null // for patch we don't want to remove unspecified value in payload assignEmpty: !isPatch, }, diff --git a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts index 3e5679a220859..7f5a3521c654a 100644 --- a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts +++ b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts @@ -10,6 +10,12 @@ import pRetry from 'p-retry'; // https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings const DEFAULT_INDEX_REFRESH_TIME = 1000; +/** + * retries until list/list item has been re-indexed + * After migration to data stream and using update_by_query, delete_by_query which do support only refresh=true/false, + * this utility needed response back when updates/delete applied + * @param fn execution function to retry + */ export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise => { await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); await pRetry(fn, { From aef8f75eb97c946b3816b78df3d646b9c82be497 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 3 Aug 2023 15:31:02 +0100 Subject: [PATCH 36/41] ILNM policy cleanup --- .../kbn-securitysolution-es-utils/index.ts | 1 + .../src/remove_policy_from_index/index.ts | 16 ++++++++++++++++ .../list_index/delete_list_index_route.ts | 4 +++- .../server/services/lists/list_client.ts | 19 +++++++++++++++---- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts index 57f02580935dc..049d5a7de492c 100644 --- a/packages/kbn-securitysolution-es-utils/index.ts +++ b/packages/kbn-securitysolution-es-utils/index.ts @@ -28,6 +28,7 @@ export * from './src/migrate_to_data_stream'; export * from './src/read_index'; export * from './src/read_privileges'; export * from './src/put_mappings'; +export * from './src/remove_policy_from_index'; export * from './src/set_index_template'; export * from './src/set_policy'; export * from './src/set_template'; diff --git a/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts b/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts new file mode 100644 index 0000000000000..0b77c112a833f --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +export const removePolicyFromIndex = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + return (await esClient.ilm.removePolicy({ index }, { meta: true })).body; +}; diff --git a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts index ff83a3451217c..94d0306c831a9 100644 --- a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -116,7 +116,9 @@ const deleteDataStreams = async ( listDataStreamExists: boolean, listItemDataStreamExists: boolean ): Promise => { - await lists.deleteListDataStream(); + if (listDataStreamExists) { + await lists.deleteListDataStream(); + } if (listItemDataStreamExists) { await lists.deleteListItemDataStream(); } diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 62c74ab121dbf..adee23b11ee9c 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -21,6 +21,7 @@ import { getTemplateExists, migrateToDataStream, putMappings, + removePolicyFromIndex, setIndexTemplate, setPolicy, } from '@kbn/securitysolution-es-utils'; @@ -314,7 +315,7 @@ export class ListClient { * update list index mappings with @timestamp and migrates it to data stream * @returns */ - public migrateListIndexToDataStream = async (): Promise => { + public migrateListIndexToDataStream = async (): Promise => { const { esClient } = this; const listName = this.getListName(); // first need to update mapping of existing index to add @timestamp @@ -323,14 +324,18 @@ export class ListClient { listName, listMappings.properties as Record ); - return migrateToDataStream(esClient, listName); + await migrateToDataStream(esClient, listName); + await removePolicyFromIndex(esClient, listName); + if (await this.getListPolicyExists()) { + await this.deleteListPolicy(); + } }; /** * update list items index mappings with @timestamp and migrates it to data stream * @returns */ - public migrateListItemIndexToDataStream = async (): Promise => { + public migrateListItemIndexToDataStream = async (): Promise => { const { esClient } = this; const listItemName = this.getListItemName(); // first need to update mapping of existing index to add @timestamp @@ -339,7 +344,11 @@ export class ListClient { listItemName, listItemMappings.properties as Record ); - return migrateToDataStream(esClient, listItemName); + await migrateToDataStream(esClient, listItemName); + await removePolicyFromIndex(esClient, listItemName); + if (await this.getListItemPolicyExists()) { + await this.deleteListItemPolicy(); + } }; /** @@ -466,6 +475,7 @@ export class ListClient { /** * Sets the list policy * @returns The contents of the list policy set + * @deprecated after moving to data streams there should not be need to use it */ public setListPolicy = async (): Promise => { const { esClient } = this; @@ -476,6 +486,7 @@ export class ListClient { /** * Sets the list item policy * @returns The contents of the list policy set + * @deprecated after moving to data streams there should not be need to use it */ public setListItemPolicy = async (): Promise => { const { esClient } = this; From ae4b47bac62cd4814828881b5713060f4224d747 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 4 Aug 2023 17:19:15 +0100 Subject: [PATCH 37/41] update APIs --- x-pack/plugins/lists/server/routes/index.ts | 8 +- .../routes/{ => list}/patch_list_route.ts | 20 +- .../routes/{ => list}/update_list_route.ts | 20 +- .../{ => list_item}/patch_list_item_route.ts | 20 +- .../{ => list_item}/update_list_item_route.ts | 20 +- .../server/services/items/update_list_item.ts | 4 + .../server/services/lists/list_client.ts | 4 + .../server/services/lists/update_list.ts | 4 + .../utils/wait_until_document_indexed.ts | 2 +- .../tests/patch_list_items.ts | 63 +++- .../security_and_spaces/tests/patch_lists.ts | 59 ++- .../tests/update_list_items.ts | 63 +++- .../security_and_spaces/tests/update_lists.ts | 60 ++- x-pack/test/lists_api_integration/utils.ts | 346 ++++++++++++------ 14 files changed, 550 insertions(+), 143 deletions(-) rename x-pack/plugins/lists/server/routes/{ => list}/patch_list_route.ts (74%) rename x-pack/plugins/lists/server/routes/{ => list}/update_list_route.ts (74%) rename x-pack/plugins/lists/server/routes/{ => list_item}/patch_list_item_route.ts (77%) rename x-pack/plugins/lists/server/routes/{ => list_item}/update_list_item_route.ts (77%) diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index dff1a6a6b23c4..b344977699d58 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -31,8 +31,8 @@ export * from './internal/create_exception_filter_route'; export * from './import_exceptions_route'; export * from './list/import_list_item_route'; export * from './init_routes'; -export * from './patch_list_item_route'; -export * from './patch_list_route'; +export * from './list_item/patch_list_item_route'; +export * from './list/patch_list_route'; export * from './read_endpoint_list_item_route'; export * from './read_exception_list_item_route'; export * from './read_exception_list_route'; @@ -44,8 +44,8 @@ export * from './summary_exception_list_route'; export * from './update_endpoint_list_item_route'; export * from './update_exception_list_item_route'; export * from './update_exception_list_route'; -export * from './update_list_item_route'; -export * from './update_list_route'; +export * from './list_item/update_list_item_route'; +export * from './list/update_list_route'; export * from './utils'; // internal diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/list/patch_list_route.ts similarity index 74% rename from x-pack/plugins/lists/server/routes/patch_list_route.ts rename to x-pack/plugins/lists/server/routes/list/patch_list_route.ts index ed15546f82744..02edd41e4f074 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/patch_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { patchListRequest, patchListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { patchListRequest, patchListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const patchListRoute = (router: ListsPluginRouter): void => { router.patch( @@ -32,6 +30,16 @@ export const patchListRoute = (router: ListsPluginRouter): void => { try { const { name, description, id, meta, _version, version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListIndexExists(); + if (indexExists) { + await lists.migrateListIndexToDataStream(); + } + } + const list = await lists.patchList({ _version, description, id, meta, name, version }); if (list == null) { return siemResponse.error({ diff --git a/x-pack/plugins/lists/server/routes/update_list_route.ts b/x-pack/plugins/lists/server/routes/list/update_list_route.ts similarity index 74% rename from x-pack/plugins/lists/server/routes/update_list_route.ts rename to x-pack/plugins/lists/server/routes/list/update_list_route.ts index b2fe4f49950ad..a134341acd658 100644 --- a/x-pack/plugins/lists/server/routes/update_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/update_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { updateListRequest, updateListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { updateListRequest, updateListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const updateListRoute = (router: ListsPluginRouter): void => { router.put( @@ -32,6 +30,16 @@ export const updateListRoute = (router: ListsPluginRouter): void => { try { const { name, description, id, meta, _version, version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListIndexExists(); + if (indexExists) { + await lists.migrateListIndexToDataStream(); + } + } + const list = await lists.updateList({ _version, description, id, meta, name, version }); if (list == null) { return siemResponse.error({ diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts similarity index 77% rename from x-pack/plugins/lists/server/routes/patch_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts index ad18c8ef670a4..b71c949242546 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { patchListItemRequest, patchListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { patchListItemRequest, patchListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const patchListItemRoute = (router: ListsPluginRouter): void => { router.patch( @@ -32,6 +30,16 @@ export const patchListItemRoute = (router: ListsPluginRouter): void => { try { const { value, id, meta, _version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListItemDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListItemIndexExists(); + if (indexExists) { + await lists.migrateListItemIndexToDataStream(); + } + } + const listItem = await lists.patchListItem({ _version, id, diff --git a/x-pack/plugins/lists/server/routes/update_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts similarity index 77% rename from x-pack/plugins/lists/server/routes/update_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts index d05f68bf05262..f2ea42c538f61 100644 --- a/x-pack/plugins/lists/server/routes/update_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { updateListItemRequest, updateListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { updateListItemRequest, updateListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const updateListItemRoute = (router: ListsPluginRouter): void => { router.put( @@ -32,6 +30,16 @@ export const updateListItemRoute = (router: ListsPluginRouter): void => { try { const { value, id, meta, _version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListItemDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListItemIndexExists(); + if (indexExists) { + await lists.migrateListItemIndexToDataStream(); + } + } + const listItem = await lists.updateListItem({ _version, id, diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index ad3f9db5422e0..13132c525e9cc 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -95,6 +95,10 @@ export const updateListItem = async ({ } ctx._source.updated_at = params.updated_at; ctx._source.updated_by = params.updated_by; + // needed for list items that were created before migration to data streams + if (ctx._source.containsKey('@timestamp') == false) { + ctx._source['@timestamp'] = ctx._source.created_at; + } `, }, }); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index adee23b11ee9c..d0b72313b7fd6 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -318,6 +318,8 @@ export class ListClient { public migrateListIndexToDataStream = async (): Promise => { const { esClient } = this; const listName = this.getListName(); + // update list index template + await this.setListTemplate(); // first need to update mapping of existing index to add @timestamp await putMappings( esClient, @@ -338,6 +340,8 @@ export class ListClient { public migrateListItemIndexToDataStream = async (): Promise => { const { esClient } = this; const listItemName = this.getListItemName(); + // update list items index template + await this.setListItemTemplate(); // first need to update mapping of existing index to add @timestamp await putMappings( esClient, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 9dc2497bb70c2..fcb976d7eff02 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -97,6 +97,10 @@ export const updateList = async ({ } ctx._source.updated_at = params.updated_at; ctx._source.updated_by = params.updated_by; + // needed for list that were created before migration to data streams + if (ctx._source.containsKey('@timestamp') == false) { + ctx._source['@timestamp'] = ctx._source.created_at; + } `, }, }); diff --git a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts index 7f5a3521c654a..6bed7083b615e 100644 --- a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts +++ b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts @@ -20,6 +20,6 @@ export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); await pRetry(fn, { minTimeout: DEFAULT_INDEX_REFRESH_TIME, - retries: 3, + retries: 5, }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts index d2c7f14b24289..0827b5813b6c4 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts @@ -12,7 +12,7 @@ import type { CreateListItemSchema, ListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; @@ -23,6 +23,9 @@ import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -30,6 +33,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); + const es = getService('es'); describe('patch_list_items', () => { describe('patch list items', () => { @@ -215,6 +219,63 @@ export default ({ getService }: FtrProviderContext) => { message: 'list item id: "some-other-id" not found', }); }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should patch list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should patch list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts index 97d1b8d007f99..87076851bd34c 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import type { PatchListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; @@ -17,6 +17,8 @@ import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -25,6 +27,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); + const es = getService('es'); describe('patch_lists', () => { describe('patch lists', () => { @@ -209,6 +212,60 @@ export default ({ getService }: FtrProviderContext) => { message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', }); }); + + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts index 7a1bfb3f12698..e2bcddeb24841 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts @@ -12,7 +12,7 @@ import type { CreateListItemSchema, ListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; @@ -23,6 +23,9 @@ import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -30,6 +33,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); + const es = getService('es'); describe('update_list_items', () => { describe('update list items', () => { @@ -301,6 +305,63 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); }); }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should update list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts index 46c9c1341a817..d9fc0bbe38bd3 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import type { UpdateListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; @@ -17,6 +17,8 @@ import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -25,6 +27,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); const retry = getService('retry'); + const es = getService('es'); describe('update_lists', () => { describe('update lists', () => { @@ -277,6 +280,61 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); }); }); + + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + expect(body.name).to.be('some other name'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 52b07c179a90b..780042a293dcc 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -6,6 +6,7 @@ */ import type SuperTest from 'supertest'; +import { v4 as uuidv4 } from 'uuid'; import type { Type, @@ -21,10 +22,16 @@ import { LIST_INDEX, LIST_ITEM_URL, } from '@kbn/securitysolution-list-constants'; -import { setPolicy, setTemplate, createBootstrapIndex } from '@kbn/securitysolution-es-utils'; +import { + setPolicy, + setTemplate, + setIndexTemplate, + createBootstrapIndex, +} from '@kbn/securitysolution-es-utils'; import { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock'; +import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { countDownTest } from '../detection_engine_api_integration/utils'; @@ -418,87 +425,118 @@ export const waitForTextListItems = async ( await Promise.all(itemValues.map((item) => waitForTextListItem(supertest, log, item, fileName))); }; +const testPolicy = { + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + }, + }, +}; + +const listsMappings = { + dynamic: 'strict', + properties: { + name: { + type: 'keyword', + }, + deserializer: { + type: 'keyword', + }, + serializer: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + tie_breaker_id: { + type: 'keyword', + }, + meta: { + enabled: 'false', + type: 'object', + }, + created_at: { + type: 'date', + }, + updated_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + immutable: { + type: 'boolean', + }, + }, +}; + +const itemsMappings = { + dynamic: 'strict', + properties: { + tie_breaker_id: { + type: 'keyword', + }, + list_id: { + type: 'keyword', + }, + deserializer: { + type: 'keyword', + }, + serializer: { + type: 'keyword', + }, + meta: { + enabled: 'false', + type: 'object', + }, + created_at: { + type: 'date', + }, + updated_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + ip: { + type: 'ip', + }, + keyword: { + type: 'keyword', + }, + }, +}; + /** * Convenience function for creating legacy index templates to * test out logic updating to new index templates * @param es es client */ export const createLegacyListsIndices = async (es: Client) => { - await setPolicy(es, '.lists-default', { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_size: '50gb', - }, - }, - }, - }, - }, - }); - await setPolicy(es, '.items-default', { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_size: '50gb', - }, - }, - }, - }, - }, - }); + await setPolicy(es, '.lists-default', testPolicy); + await setPolicy(es, '.items-default', testPolicy); await setTemplate(es, '.lists-default', { index_patterns: [`.lists-default-*`], - mappings: { - dynamic: 'strict', - properties: { - name: { - type: 'keyword', - }, - deserializer: { - type: 'keyword', - }, - serializer: { - type: 'keyword', - }, - description: { - type: 'keyword', - }, - type: { - type: 'keyword', - }, - tie_breaker_id: { - type: 'keyword', - }, - meta: { - enabled: 'false', - type: 'object', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - created_by: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - version: { - type: 'keyword', - }, - immutable: { - type: 'boolean', - }, - }, - }, + mappings: listsMappings, settings: { index: { lifecycle: { @@ -510,42 +548,7 @@ export const createLegacyListsIndices = async (es: Client) => { }); await setTemplate(es, '.items-default', { index_patterns: [`.items-default-*`], - mappings: { - dynamic: 'strict', - properties: { - tie_breaker_id: { - type: 'keyword', - }, - list_id: { - type: 'keyword', - }, - deserializer: { - type: 'keyword', - }, - serializer: { - type: 'keyword', - }, - meta: { - enabled: 'false', - type: 'object', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - created_by: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - ip: { - type: 'ip', - }, - }, - }, + mappings: itemsMappings, settings: { index: { lifecycle: { @@ -558,3 +561,126 @@ export const createLegacyListsIndices = async (es: Client) => { await createBootstrapIndex(es, '.lists-default'); await createBootstrapIndex(es, '.items-default'); }; + +/** + * Utility to create list indices, before they were migrated to data streams + * @param es ES client + */ +export const createListsIndices = async (es: Client) => { + await setPolicy(es, '.lists-default', testPolicy); + await setPolicy(es, '.items-default', testPolicy); + await setIndexTemplate(es, '.lists-default', { + index_patterns: [`.lists-default-*`], + template: { + mappings: listsMappings, + settings: { + index: { + lifecycle: { + name: '.lists-default', + rollover_alias: '.lists-default', + }, + }, + mapping: { + total_fields: { + limit: 10000, + }, + }, + }, + }, + }); + await setIndexTemplate(es, '.items-default', { + index_patterns: [`.items-default-*`], + template: { + mappings: itemsMappings, + settings: { + index: { + lifecycle: { + name: '.items-default', + rollover_alias: '.items-default', + }, + }, + mapping: { + total_fields: { + limit: 10000, + }, + }, + }, + }, + }); + await createBootstrapIndex(es, '.lists-default'); + await createBootstrapIndex(es, '.items-default'); +}; + +/** + * utility to create list directly by using ES, bypassing all checks + * useful, to create list in legacy indices + */ +export const createListBypassingChecks = async ({ es, id }: { es: Client; id: string }) => { + const createdAt = new Date().toISOString(); + const body = { + created_at: createdAt, + created_by: 'mock-user', + description: 'mock-description', + name: 'mock-name', + tie_breaker_id: uuidv4(), + type: 'keyword', + updated_at: createdAt, + updated_by: 'mock-user', + immutable: false, + version: 1, + }; + + const response = await es.create({ + body, + id, + index: '.lists-default', + refresh: 'wait_for', + }); + + return { + _version: encodeHitVersion(response), + id: response._id, + ...body, + }; +}; + +/** + * utility to create list item directly by using ES, bypassing all checks + * useful, to create list item in legacy indices + * supports keyword only + */ +export const createListItemBypassingChecks = async ({ + es, + listId, + id, + value, +}: { + es: Client; + listId: string; + id: string; + value: string; +}) => { + const createdAt = new Date().toISOString(); + const body = { + created_at: createdAt, + created_by: 'mock-user', + tie_breaker_id: uuidv4(), + updated_at: createdAt, + updated_by: 'mock-user', + list_id: listId, + keyword: value, + }; + + const response = await es.create({ + body, + id, + index: '.items-default', + refresh: 'wait_for', + }); + + return { + _version: encodeHitVersion(response), + id: response._id, + ...body, + }; +}; From d664868dcc8feb6e50f09d08003764efe87e5f5a Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Fri, 4 Aug 2023 18:11:46 +0100 Subject: [PATCH 38/41] import --- .../routes/list/import_list_item_route.ts | 32 +++++++++++++++---- .../tests/import_list_items.ts | 32 +++++++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts index bb79fcaa40059..60ecf15a07356 100644 --- a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts @@ -43,13 +43,33 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp const stream = createStreamFromBuffer(request.body); const { deserializer, list_id: listId, serializer, type } = request.query; const lists = await getListClient(context); - const listExists = await lists.getListDataStreamExists(); - if (!listExists) { - return siemResponse.error({ - body: `To import a list item, the data steam must exist first. Data stream "${lists.getListName()}" does not exist`, - statusCode: 400, - }); + + const listDataExists = await lists.getListDataStreamExists(); + if (!listDataExists) { + const listIndexExists = await lists.getListIndexExists(); + if (!listIndexExists) { + return siemResponse.error({ + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListName()}" does not exist`, + statusCode: 400, + }); + } + // otherwise migration is needed + await lists.migrateListIndexToDataStream(); } + + const listItemDataExists = await lists.getListItemDataStreamExists(); + if (!listItemDataExists) { + const listItemIndexExists = await lists.getListItemIndexExists(); + if (!listItemIndexExists) { + return siemResponse.error({ + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListItemName()}" does not exist`, + statusCode: 400, + }); + } + // otherwise migration is needed + await lists.migrateListItemIndexToDataStream(); + } + if (listId != null) { const list = await lists.getList({ id: listId }); if (list == null) { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts index 1d2be1c93d7f1..89ae216adc865 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts @@ -19,12 +19,14 @@ import { removeListServerGeneratedProperties, removeListItemServerGeneratedProperties, waitFor, + createListsIndices, } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const es = getService('es'); describe('import_list_items', () => { describe('importing list items without an index', () => { @@ -110,6 +112,36 @@ export default ({ getService }: FtrProviderContext): void => { }; expect(bodyToCompare).to.eql(outputtedList); }); + + describe('legacy index (before migration to data streams)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should import list to legacy index and migrate it', async () => { + // create legacy indices + await createListsIndices(es); + + const { body } = await supertest + .post(`${LIST_ITEM_URL}/_import?type=ip`) + .set('kbn-xsrf', 'true') + .attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.txt') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const bodyToCompare = removeListServerGeneratedProperties(body); + const outputtedList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'list_items.txt', + description: 'File uploaded from file system of list_items.txt', + }; + expect(bodyToCompare).to.eql(outputtedList); + }); + }); }); }); }; From ee4557217a4fc71f27aaa13d16d851b140e7253e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:27:10 +0100 Subject: [PATCH 39/41] CR: Update read_list_index_route.ts --- .../lists/server/routes/list_index/read_list_index_route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts index 4530f9e68febb..3f1ffa8b95f5d 100644 --- a/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts @@ -31,7 +31,7 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { const listDataStreamExists = await lists.getListDataStreamExists(); const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - if (listDataStreamExists || listItemDataStreamExists) { + if (listDataStreamExists && listItemDataStreamExists) { const [validated, errors] = validate( { list_index: listDataStreamExists, list_item_index: listItemDataStreamExists }, readListIndexResponse From d54dbaf7b1ce1c1bed957c3ad5f57b9a73f03602 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko Date: Thu, 17 Aug 2023 18:34:03 +0100 Subject: [PATCH 40/41] CR: delete logic rehaul --- .../routes/list_index/delete_list_index_route.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts index 94d0306c831a9..901da6a997014 100644 --- a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -50,18 +50,21 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => { const listDataStreamExists = await lists.getListDataStreamExists(); const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - // data streams exists, it means indices were migrated - if (listDataStreamExists) { - await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists); - } else if (!listIndexExists && !listItemIndexExists) { + + // return early if no data stream or indices exist + if (!listDataStreamExists && !listItemDataStreamExists && !listIndexExists && !listItemIndexExists) { return siemResponse.error({ body: `index and data stream: "${lists.getListName()}" and "${lists.getListItemName()}" does not exist`, statusCode: 404, }); - } else { - await deleteIndices(lists, listIndexExists, listItemIndexExists); } + // ensure data streams deleted if exist + await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists); + + // ensure indices deleted if exist and were not migrated + await deleteIndices(lists, listIndexExists, listItemIndexExists); + await deleteIndexTemplates(lists); await removeLegacyTemplatesIfExist(lists); From 65fc0672b6099b3b52c450bd833cf8b349c96d9e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 17 Aug 2023 18:15:12 +0000 Subject: [PATCH 41/41] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/routes/list_index/delete_list_index_route.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts index 901da6a997014..2c8a2fb3212ce 100644 --- a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -50,9 +50,13 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => { const listDataStreamExists = await lists.getListDataStreamExists(); const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - // return early if no data stream or indices exist - if (!listDataStreamExists && !listItemDataStreamExists && !listIndexExists && !listItemIndexExists) { + if ( + !listDataStreamExists && + !listItemDataStreamExists && + !listIndexExists && + !listItemIndexExists + ) { return siemResponse.error({ body: `index and data stream: "${lists.getListName()}" and "${lists.getListItemName()}" does not exist`, statusCode: 404,