diff --git a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts index 822a18fd13cf1..9e64fe59404c6 100644 --- a/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { extname } from 'path'; + import { schema } from '@kbn/config-schema'; import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; @@ -17,6 +19,8 @@ import { buildRouteValidation, buildSiemResponse } from '../utils'; import { createStreamFromBuffer } from '../utils/create_stream_from_buffer'; import { getListClient } from '..'; +const validFileExtensions = ['.csv', '.txt']; + export const importListItemRoute = (router: ListsPluginRouter, config: ConfigType): void => { router.versioned .post({ @@ -47,10 +51,29 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp async (context, request, response) => { const siemResponse = buildSiemResponse(response); try { - const stream = createStreamFromBuffer(request.body); const { deserializer, list_id: listId, serializer, type } = request.query; const lists = await getListClient(context); + const filename = await lists.getImportFilename({ + stream: createStreamFromBuffer(request.body), + }); + if (!filename) { + return siemResponse.error({ + body: 'To import a list item, the file name must be specified', + statusCode: 400, + }); + } + const fileExtension = extname(filename).toLowerCase(); + if (!validFileExtensions.includes(fileExtension)) { + return siemResponse.error({ + body: `Unsupported media type. File must be one of the following types: [${validFileExtensions.join( + ', ' + )}]`, + statusCode: 415, + }); + } + + const stream = createStreamFromBuffer(request.body); const listDataExists = await lists.getListDataStreamExists(); if (!listDataExists) { const listIndexExists = await lists.getListIndexExists(); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 826e4a545d80d..d1d5c585c3f85 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -37,6 +37,7 @@ import type { import type { ConfigType } from '../../config'; import { + BufferLines, createListItem, deleteListItem, deleteListItemByValue, @@ -69,6 +70,7 @@ import type { FindAllListItemsOptions, FindListItemOptions, FindListOptions, + GetImportFilename, GetListItemByValueOptions, GetListItemOptions, GetListItemsByValueOptions, @@ -715,6 +717,33 @@ export class ListClient { }); }; + /** + * Gets the filename of the imported file + * @param options + * @param options.stream The stream to pull the import from + * @returns + */ + public getImportFilename = ({ stream }: GetImportFilename): Promise => { + return new Promise((resolve, reject) => { + const { config } = this; + const readBuffer = new BufferLines({ bufferSize: config.importBufferSize, input: stream }); + let fileName: string | undefined; + readBuffer.on('fileName', async (fileNameEmitted: string) => { + try { + readBuffer.pause(); + fileName = decodeURIComponent(fileNameEmitted); + readBuffer.resume(); + } catch (err) { + reject(err); + } + }); + + readBuffer.on('close', () => { + resolve(fileName); + }); + }); + }; + /** * Imports list items to a stream. If the list already exists, this will append the list items to the existing list. * If the list does not exist, this will auto-create the list and then add the items to that list. diff --git a/x-pack/plugins/lists/server/services/lists/list_client_types.ts b/x-pack/plugins/lists/server/services/lists/list_client_types.ts index 7509eeb914241..4c64e8e940163 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client_types.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client_types.ts @@ -335,3 +335,12 @@ export interface SearchListItemByValuesOptions { /** The value to search for list items based off. */ value: unknown[]; } + +/** + * ListClient.getImportFilename + * {@link ListClient.getImportFilename} + */ +export interface GetImportFilename { + /** The stream to pull the import from */ + stream: Readable; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/role_based_rule_exceptions_workflows.ts index ef78015ef7715..e308732db3821 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/role_based_rule_exceptions_workflows.ts @@ -803,7 +803,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('generates no alerts when a value list exception is added for a query rule', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId); const rule: QueryRuleCreateProps = { name: 'Simple Rule Query', @@ -835,7 +835,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('generates no alerts when a value list exception is added for a threat match rule', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['zeek-sensor-amsterdam'], valueListId); const rule: ThreatMatchRuleCreateProps = { description: 'Detecting root and admin users', @@ -883,7 +883,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('generates no alerts when a value list exception is added for a threshold rule', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['zeek-sensor-amsterdam'], valueListId); const rule: ThresholdRuleCreateProps = { description: 'Detecting root and admin users', @@ -920,7 +920,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('generates no alerts when a value list exception is added for an EQL rule', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['zeek-sensor-amsterdam'], valueListId); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['auditbeat-*']), @@ -944,7 +944,7 @@ export default ({ getService }: FtrProviderContext) => { expect(alertsOpen.hits.hits.length).toEqual(0); }); it('should Not allow deleting value list when there are references and ignoreReferences is false', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId); const rule: QueryRuleCreateProps = { ...getSimpleRule(), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/rule_exception_synchronizations.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/rule_exception_synchronizations.ts index c584ec46f2ef1..5a68270e1220a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/rule_exception_synchronizations.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/trial_license_complete_tier/rule_exception_synchronizations.ts @@ -107,7 +107,7 @@ export default ({ getService }: FtrProviderContext) => { it('should Not allow editing an Exception with deleted ValueList', async () => { await createListsIndex(supertest, log); - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId); const rule: QueryRuleCreateProps = { ...getSimpleRule(), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts index e222f1ddd7cb4..60ad53f94937f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/machine_learning.ts @@ -229,7 +229,7 @@ export default ({ getService }: FtrProviderContext) => { }); it('generates no alerts when a value list exception is added for an ML rule', async () => { - const valueListId = 'value-list-id'; + const valueListId = 'value-list-id.txt'; await importFile(supertest, log, 'keyword', ['mothra'], valueListId); const { previewId } = await previewRuleWithExceptionEntries({ supertest, diff --git a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/items/import_list_items.ts b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/items/import_list_items.ts index 99a96ae9052b1..7fabd749bc01d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/items/import_list_items.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/items/import_list_items.ts @@ -54,6 +54,39 @@ export default ({ getService }: FtrProviderContext): void => { await deleteListsIndex(supertest, log); }); + it('should not import a list item if the imported file is not .txt or .csv', async () => { + const { body } = await supertest + .post(`${LIST_ITEM_URL}/_import?type=ip`) + .set('kbn-xsrf', 'true') + .attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.pdf') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(415); + + expect(body).to.eql({ + status_code: 415, + message: 'Unsupported media type. File must be one of the following types: [.csv, .txt]', + }); + }); + + it('should not import a list item if the imported file exceed the file size limit', async () => { + const { body } = await supertest + .post(`${LIST_ITEM_URL}/_import?type=ip`) + .set('kbn-xsrf', 'true') + .attach( + 'file', + getImportListItemAsBuffer(Array(1000000).fill('127.0.0.1')), + 'list_items.txt' + ) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(413); + + expect(body).to.eql({ + statusCode: 413, + error: 'Request Entity Too Large', + message: 'Payload content length greater than maximum allowed: 9000000', + }); + }); + it('should set the response content types to be expected when importing two items', async () => { await supertest .post(`${LIST_ITEM_URL}/_import?type=ip`)