From dfa911db9223a61bac6456e921f7600cc1d3b0f5 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 25 Aug 2021 19:38:05 -0700 Subject: [PATCH] [Reporting] Fix ability to export CSV on searched data with frozen indices (#109976) (#110163) * use include frozen setting in csv export * add api integration test * add fixes * Update x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts * test polish * update per feedback --- x-pack/plugins/reporting/common/constants.ts | 1 + .../generate_csv/generate_csv.test.ts | 10 +- .../generate_csv/generate_csv.ts | 7 +- .../generate_csv/get_export_settings.test.ts | 4 + .../generate_csv/get_export_settings.ts | 12 +- .../reporting_and_security/index.ts | 1 + .../search_frozen_indices.ts | 127 ++++++++++++++++++ 7 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts index 4ba406a14bafc..bd6836517d868 100644 --- a/x-pack/plugins/reporting/common/constants.ts +++ b/x-pack/plugins/reporting/common/constants.ts @@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-']; +export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen'; export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo'; export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator'; export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues'; diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts index 3e4e663733e2c..cb5670f6eafd9 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts @@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { message: ['this is a great message'], }, @@ -248,7 +248,7 @@ it('warns if max size was reached', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => { rawResponse: { _scroll_id: 'awesome-scroll-hero', hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => { mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({ body: { hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => { expect(mockDataClient.search).toHaveBeenCalledTimes(1); expect(mockDataClient.search).toBeCalledWith( - { params: { scroll: '30s', size: 500 } }, + { params: { ignore_throttled: true, scroll: '30s', size: 500 } }, { strategy: 'es' } ); diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index b49dbc1043135..7f01f9fcb297c 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -83,8 +83,9 @@ export class CsvGenerator { private async scan( index: IndexPattern, searchSource: ISearchSource, - scrollSettings: CsvExportSettings['scroll'] + settings: CsvExportSettings ) { + const { scroll: scrollSettings, includeFrozen } = settings; const searchBody = searchSource.getSearchRequestBody(); this.logger.debug(`executing search request`); const searchParams = { @@ -93,8 +94,10 @@ export class CsvGenerator { index: index.title, scroll: scrollSettings.duration, size: scrollSettings.size, + ignore_throttled: !includeFrozen, }, }; + const results = ( await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise() ).rawResponse as estypes.SearchResponse; @@ -326,7 +329,7 @@ export class CsvGenerator { let results: estypes.SearchResponse | undefined; if (scrollId == null) { // open a scroll cursor in Elasticsearch - results = await this.scan(index, searchSource, scrollSettings); + results = await this.scan(index, searchSource, settings); scrollId = results?._scroll_id; if (results.hits?.total != null) { totalRecords = results.hits.total as number; diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts index efdb583a89dc8..2ae3e5e712d31 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts @@ -9,6 +9,7 @@ import { UI_SETTINGS_DATEFORMAT_TZ, UI_SETTINGS_CSV_QUOTE_VALUES, UI_SETTINGS_CSV_SEPARATOR, + UI_SETTINGS_SEARCH_INCLUDE_FROZEN, } from '../../../../common/constants'; import { IUiSettingsClient } from 'kibana/server'; import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks'; @@ -36,6 +37,8 @@ describe('getExportSettings', () => { return ','; case UI_SETTINGS_DATEFORMAT_TZ: return 'Browser'; + case UI_SETTINGS_SEARCH_INCLUDE_FROZEN: + return false; } return 'helo world'; @@ -49,6 +52,7 @@ describe('getExportSettings', () => { "checkForFormulas": undefined, "escapeFormulaValues": undefined, "escapeValue": [Function], + "includeFrozen": false, "maxSizeBytes": undefined, "scroll": Object { "duration": undefined, diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts index 0b8815836367f..7d4db38ef4aae 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts @@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common' import { ReportingConfig } from '../../../'; import { CSV_BOM_CHARS, - UI_SETTINGS_DATEFORMAT_TZ, UI_SETTINGS_CSV_QUOTE_VALUES, UI_SETTINGS_CSV_SEPARATOR, + UI_SETTINGS_DATEFORMAT_TZ, + UI_SETTINGS_SEARCH_INCLUDE_FROZEN, } from '../../../../common/constants'; import { LevelLogger } from '../../../lib'; @@ -30,6 +31,7 @@ export interface CsvExportSettings { checkForFormulas: boolean; escapeFormulaValues: boolean; escapeValue: (value: string) => string; + includeFrozen: boolean; } export const getExportSettings = async ( @@ -38,9 +40,7 @@ export const getExportSettings = async ( timezone: string | undefined, logger: LevelLogger ): Promise => { - // Timezone let setTimezone: string; - // timezone in job params? if (timezone) { setTimezone = timezone; } else { @@ -59,8 +59,9 @@ export const getExportSettings = async ( } } - // Separator, QuoteValues - const [separator, quoteValues] = await Promise.all([ + // Advanced Settings that affect search export + CSV + const [includeFrozen, separator, quoteValues] = await Promise.all([ + client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN), client.get(UI_SETTINGS_CSV_SEPARATOR), client.get(UI_SETTINGS_CSV_QUOTE_VALUES), ]); @@ -76,6 +77,7 @@ export const getExportSettings = async ( duration: config.get('csv', 'scroll', 'duration'), }, bom, + includeFrozen, separator, maxSizeBytes: config.get('csv', 'maxSizeBytes'), checkForFormulas: config.get('csv', 'checkForFormulas'), diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index 64ad4a480fe7c..aea441f57244b 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./spaces')); loadTestFile(require.resolve('./usage')); loadTestFile(require.resolve('./ilm_migration_apis')); + loadTestFile(require.resolve('./search_frozen_indices')); }); } diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts new file mode 100644 index 0000000000000..daa749649e250 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService }: FtrProviderContext) { + const kibanaServer = getService('kibanaServer'); + const supertestSvc = getService('supertest'); + const esSupertest = getService('esSupertest'); + const indexPatternId = 'cool-test-index-pattern'; + + async function callExportAPI() { + const job = { + browserTimezone: 'UTC', + columns: ['@timestamp', 'ip', 'utilization'], + searchSource: { + fields: [{ field: '*', include_unmapped: 'true' }], + filter: [ + { + meta: { field: '@timestamp', index: indexPatternId, params: {} }, + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-08-24T00:00:00.000Z', + lte: '2022-08-24T21:40:48.346Z', + }, + }, + }, + ], + index: indexPatternId, + parent: { filter: [], index: indexPatternId, query: { language: 'kuery', query: '' } }, + sort: [{ '@timestamp': 'desc' }], + trackTotalHits: true, + }, + title: 'Test search', + }; + + return await supertestSvc + .post(`/api/reporting/v1/generate/immediate/csv_searchsource`) + .set('kbn-xsrf', 'xxx') + .send(job); + } + + describe('Frozen indices search', () => { + const reset = async () => { + await kibanaServer.uiSettings.replace({ 'search:includeFrozen': false }); + try { + await esSupertest.delete('/test1,test2,test3'); + await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId }); + } catch (err) { + // ignore 404 error + } + }; + + before(reset); + after(reset); + + it('Search includes frozen indices based on Advanced Setting', async () => { + await kibanaServer.uiSettings.update({ 'csv:quoteValues': true }); + + // setup: add multiple indices of test data + await Promise.all([ + esSupertest + .post('/test1/_doc') + .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }), + esSupertest + .post('/test2/_doc') + .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }), + esSupertest + .post('/test3/_doc') + .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }), + ]); + await esSupertest.post('/test*/_refresh'); + + // setup: create index pattern + const indexPatternCreateResponse = await kibanaServer.savedObjects.create({ + type: 'index-pattern', + id: indexPatternId, + overwrite: true, + attributes: { title: 'test*', timeFieldName: '@timestamp' }, + }); + expect(indexPatternCreateResponse.id).to.be(indexPatternId); + + // 1. check the initial data with a CSV export + const initialSearch = await callExportAPI(); + expectSnapshot(initialSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); + + // 2. freeze an index in the pattern + await esSupertest.post('/test3/_freeze').expect(200); + await esSupertest.post('/test*/_refresh').expect(200); + + // 3. recheck the search results + const afterFreezeSearch = await callExportAPI(); + expectSnapshot(afterFreezeSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + " + `); + + // 4. update setting to allow searching frozen data + await kibanaServer.uiSettings.update({ 'search:includeFrozen': true }); + + // 5. recheck the search results + const afterAllowSearch = await callExportAPI(); + expectSnapshot(afterAllowSearch.text).toMatchInline(` + "\\"@timestamp\\",ip,utilization + \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" + \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" + \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\" + " + `); + }); + }); +}