From 9c9df14929504c2b4a235ee868bb21204215091d Mon Sep 17 00:00:00 2001 From: OUEDRAOGO WENDZOODO FABRICE GHISLAIN Date: Fri, 21 Aug 2020 22:55:36 +0200 Subject: [PATCH] APIs endpoints for data reports. (#50) Adding API endpoints for data reports --- package.json | 4 +- server/model/index.ts | 160 ++--- server/routes/dataReport.ts | 355 +++++++++++ server/routes/dataReportMetadata.ts | 131 ++++ server/routes/index.ts | 8 +- .../routes/utils/__tests__/dataReport.test.ts | 589 ++++++++++++++++++ server/routes/utils/constants.ts | 4 + server/routes/utils/dataReportHelpers.ts | 212 +++++++ server/routes/utils/helpers.ts | 3 + 9 files changed, 1383 insertions(+), 83 deletions(-) create mode 100644 server/routes/dataReport.ts create mode 100644 server/routes/dataReportMetadata.ts create mode 100644 server/routes/utils/__tests__/dataReport.test.ts create mode 100644 server/routes/utils/dataReportHelpers.ts diff --git a/package.json b/package.json index 77f95413..587a6f15 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,8 @@ "eslint-plugin-react": "^7.12.4", "jest": "^26.0.1", "prettier": "2.0.5", - "react-test-renderer": "^16.13.1" + "react-test-renderer": "^16.13.1", + "elastic-builder": "^2.7.1", + "json-2-csv": "^3.7.6" } } diff --git a/server/model/index.ts b/server/model/index.ts index fd452e65..6dd767f1 100644 --- a/server/model/index.ts +++ b/server/model/index.ts @@ -15,104 +15,104 @@ import { schema } from '@kbn/config-schema'; -// TODO: needs update when we integrate csv feature -const dataReportSchema = schema.object({ - saved_search_id: schema.string(), - time_range: schema.string(), - report_format: schema.oneOf([schema.literal('csv'), schema.literal('xlsx')]), +export const dataReportSchema = schema.object({ + saved_search_id: schema.string(), + start: schema.string(), + end: schema.string(), + report_format: schema.oneOf([schema.literal('csv'), schema.literal('xlsx')]), }); const visualReportSchema = schema.object({ - url: schema.uri(), - window_width: schema.number({ defaultValue: 1200 }), - window_height: schema.number({ defaultValue: 800 }), - report_format: schema.oneOf([schema.literal('pdf'), schema.literal('png')]), + url: schema.uri(), + window_width: schema.number({ defaultValue: 1200 }), + window_height: schema.number({ defaultValue: 800 }), + report_format: schema.oneOf([schema.literal('pdf'), schema.literal('png')]), }); export const scheduleSchema = schema.object({ - schedule_type: schema.oneOf([ - schema.literal('Now'), - schema.literal('Future Date'), - schema.literal('Recurring'), - schema.literal('Cron Based'), - ]), - schedule: schema.any(), + schedule_type: schema.oneOf([ + schema.literal('Now'), + schema.literal('Future Date'), + schema.literal('Recurring'), + schema.literal('Cron Based'), + ]), + schedule: schema.any(), }); export const intervalSchema = schema.object({ - interval: schema.object({ - period: schema.number(), - // Refer to job scheduler SPI https://github.com/opendistro-for-elasticsearch/job-scheduler/blob/b333469c183a15ddbf496a690300cc9e34d937fb/spi/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/spi/schedule/IntervalSchedule.java - unit: schema.oneOf([ - schema.literal('MINUTES'), - schema.literal('HOURS'), - schema.literal('DAYS'), - ]), - // timeStamp - start_time: schema.number(), - }), + interval: schema.object({ + period: schema.number(), + // Refer to job scheduler SPI https://github.com/opendistro-for-elasticsearch/job-scheduler/blob/b333469c183a15ddbf496a690300cc9e34d937fb/spi/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/spi/schedule/IntervalSchedule.java + unit: schema.oneOf([ + schema.literal('MINUTES'), + schema.literal('HOURS'), + schema.literal('DAYS'), + ]), + // timeStamp + start_time: schema.number(), + }), }); export const cronSchema = schema.object({ - corn: schema.object({ - expression: schema.string(), - time_zone: schema.string(), - }), + corn: schema.object({ + expression: schema.string(), + time_zone: schema.string(), + }), }); export const emailSchema = schema.object({ - subject: schema.string(), - body: schema.string(), - has_attachment: schema.boolean({ defaultValue: true }), - recipients: schema.arrayOf(schema.string(), { minSize: 1 }), + subject: schema.string(), + body: schema.string(), + has_attachment: schema.boolean({ defaultValue: true }), + recipients: schema.arrayOf(schema.string(), { minSize: 1 }), }); export const reportSchema = schema.object({ - report_name: schema.string(), - report_source: schema.oneOf([ - schema.literal('Dashboard'), - schema.literal('Visualization'), - schema.literal('Saved search'), - ]), - report_type: schema.oneOf([ - schema.literal('Download'), - schema.literal('Alert'), - schema.literal('Schedule'), - ]), - description: schema.string(), - report_params: schema.conditional( - schema.siblingRef('report_source'), - 'Saved search', - dataReportSchema, - visualReportSchema - ), + report_name: schema.string(), + report_source: schema.oneOf([ + schema.literal('Dashboard'), + schema.literal('Visualization'), + schema.literal('Saved search'), + ]), + report_type: schema.oneOf([ + schema.literal('Download'), + schema.literal('Alert'), + schema.literal('Schedule'), + ]), + description: schema.string(), + report_params: schema.conditional( + schema.siblingRef('report_source'), + 'Saved search', + dataReportSchema, + visualReportSchema + ), - delivery: schema.maybe( - schema.object({ - channel: schema.oneOf([ - schema.literal('Email'), - schema.literal('Slack'), - schema.literal('Chime'), - schema.literal('Kibana User'), - ]), - //TODO: no validation on delivery settings for now, because @kbn/config-schema has no support for more than 2 conditions - delivery_params: schema.any(), - }) - ), + delivery: schema.maybe( + schema.object({ + channel: schema.oneOf([ + schema.literal('Email'), + schema.literal('Slack'), + schema.literal('Chime'), + schema.literal('Kibana User'), + ]), + //TODO: no validation on delivery settings for now, because @kbn/config-schema has no support for more than 2 conditions + delivery_params: schema.any(), + }) + ), - trigger: schema.maybe( - schema.object({ - trigger_type: schema.oneOf([ - schema.literal('Alert'), - schema.literal('Schedule'), - ]), - trigger_params: schema.conditional( - schema.siblingRef('trigger_type'), - 'Alert', - // TODO: add alerting schema here once we finished the design for alerting integration - schema.any(), - scheduleSchema - ), - }) - ), + trigger: schema.maybe( + schema.object({ + trigger_type: schema.oneOf([ + schema.literal('Alert'), + schema.literal('Schedule'), + ]), + trigger_params: schema.conditional( + schema.siblingRef('trigger_type'), + 'Alert', + // TODO: add alerting schema here once we finished the design for alerting integration + schema.any(), + scheduleSchema + ), + }) + ), }); diff --git a/server/routes/dataReport.ts b/server/routes/dataReport.ts new file mode 100644 index 00000000..26792ea3 --- /dev/null +++ b/server/routes/dataReport.ts @@ -0,0 +1,355 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { schema } from '@kbn/config-schema'; +import { + IRouter, + IKibanaResponse, + ResponseError, +} from '../../../../src/core/server'; +import { API_PREFIX } from '../../common'; +import { parseEsErrorResponse } from './utils/helpers'; +import { buildQuery, getEsData, convertToCSV } from './utils/dataReportHelpers'; + +export default function (router: IRouter) { + //download the data-report from meta data + router.get( + { + path: `${API_PREFIX}/data-report/generate/{reportId}`, + validate: { + params: schema.object({ + reportId: schema.string(), + }), + query: schema.object({ + nbRows: schema.maybe(schema.number({ min: 1 })), + scroll_size: schema.maybe(schema.number({ min: 1 })), + }), + }, + }, + async ( + context, + request, + response + ): Promise> => { + try { + let { nbRows, scroll_size } = request.query as { + nbRows?: number; + scroll_size?: number; + }; + + let dataset: any = []; + let arrayHits: any = []; + let esData: any = {}; + let message: string = 'success'; + let fetch_size: number = 0; + let nbScroll: number = 0; + + //get the metadata of the report from ES using reportId + const report = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'get', + { + index: 'datareport', + id: request.params.reportId, + } + ); + + //fetch ES query max size windows + const indexPattern: string = report._source.paternName; + let settings = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'indices.getSettings', + { + index: indexPattern, + includeDefaults: true, + } + ); + + const default_max_size: number = + settings[indexPattern].defaults.index.max_result_window; + + //build the ES Count query + const countReq = buildQuery(report, 1); + + //Count the Data in ES + const esCount = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'count', + { + index: indexPattern, + body: countReq.toJSON(), + } + ); + + //If No data in elasticsearch + if (esCount.count === 0) { + return response.custom({ + statusCode: 200, + body: 'No data in Elasticsearch.', + }); + } + + //build the ES query + const reqBody = buildQuery(report, 0); + + //first case: No args passed. No need to scroll + if (!nbRows && !scroll_size) { + if (esCount.count > default_max_size) { + message = `Truncated Data! The requested data has reached the limit of Elasticsearch query size of ${default_max_size}. Please increase the limit and try again !`; + } + esData = await fetchData(report, reqBody, default_max_size); + arrayHits.push(esData.hits); + } + + //Second case: 1 arg passed + + //Only Number of Rows is passed + + if (nbRows && !scroll_size) { + let rows = 0; + if (nbRows > default_max_size) { + //fetch the data + fetch_size = default_max_size; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + if (nbRows > esCount.count) { + rows = esCount.count; + nbScroll = Math.floor(esCount.count / default_max_size); + } else { + rows = nbRows; + nbScroll = Math.floor(nbRows / default_max_size); + } + // let data = { + // esData, + // rows, + // nbScroll, + // fetch_size, + // report, + // reqBody, + // }; + + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = rows % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData(report, reqBody, extra_fetch); + arrayHits.push(extra_esData.hits); + } + } else { + fetch_size = nbRows; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + } + } else if (scroll_size && !nbRows) { + //Only scroll_size is passed + if (esCount.count > default_max_size) { + fetch_size = scroll_size; + nbScroll = Math.floor(esCount.count / scroll_size); + if (scroll_size > default_max_size) { + fetch_size = default_max_size; + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + nbScroll = Math.floor(esCount.count / default_max_size); + } + //fetch the data + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = esCount.count % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData(report, reqBody, extra_fetch); + arrayHits.push(extra_esData.hits); + } + } else { + //no need to scroll + esData = await fetchData(report, reqBody, esCount.count); + arrayHits.push(esData.hits); + } + } + //Third case: 2 args passed + if (scroll_size && nbRows) { + if (nbRows > esCount.count) { + if (esCount.count > default_max_size) { + //perform the scroll + if (scroll_size > default_max_size) { + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + fetch_size = default_max_size; + nbScroll = Math.floor(esCount.count / default_max_size); + } else { + fetch_size = scroll_size; + nbScroll = Math.floor(esCount.count / scroll_size); + } + + //fetch the data then perform the scroll + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + + //perform the scroll + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = esCount.count % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData( + report, + reqBody, + extra_fetch + ); + arrayHits.push(extra_esData.hits); + } + } else { + //no need to perform the scroll just fetch the data + fetch_size = esCount.count; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + } + } else { + if (nbRows > default_max_size) { + if (scroll_size > default_max_size) { + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + fetch_size = default_max_size; + nbScroll = Math.floor(nbRows / default_max_size); + } else { + fetch_size = scroll_size; + nbScroll = Math.floor(nbRows / scroll_size); + } + //fetch the data then perform the scroll + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = nbRows % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData( + report, + reqBody, + extra_fetch + ); + arrayHits.push(extra_esData.hits); + } + } else { + //just fetch the data no need of scroll + esData = await fetchData(report, reqBody, nbRows); + arrayHits.push(esData.hits); + } + } + } + + //Get data + dataset.push(getEsData(arrayHits, report)); + + //Convert To csv + const csv = await convertToCSV(dataset); + + const data = { + default_max_size, + message, + nbScroll, + total: esCount.count, + datasetCount: dataset[0].length, + dataset, + csv, + }; + + // To do: return the data + return response.ok({ + body: data, + headers: { + 'content-type': 'application/json', + }, + }); + } catch (error) { + //@ts-ignore + context.reporting_plugin.logger.error( + `Fail to generate the report: ${error}` + ); + return response.custom({ + statusCode: error.statusCode || 500, + body: parseEsErrorResponse(error), + }); + } + + //Fecth the data from ES + async function fetchData(report, reqBody, fetch_size) { + const docvalues = []; + for (let dateType of report._source.dateFields) { + docvalues.push({ + field: dateType, + format: 'date_hour_minute', + }); + } + + const newBody = { + query: reqBody.toJSON().query, + docvalue_fields: docvalues, + }; + + const esData = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'search', + { + index: report._source.paternName, + scroll: '1m', + body: newBody, + size: fetch_size, + } + ); + return esData; + } + } + ); +} diff --git a/server/routes/dataReportMetadata.ts b/server/routes/dataReportMetadata.ts new file mode 100644 index 00000000..e3f694fc --- /dev/null +++ b/server/routes/dataReportMetadata.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { schema } from '@kbn/config-schema'; +import { + IRouter, + IKibanaResponse, + ResponseError, +} from '../../../../src/core/server'; +import { API_PREFIX } from '../../common'; +import { dataReportSchema } from '../model'; +import { parseEsErrorResponse } from './utils/helpers'; +import { metaData, getSelectedFields } from './utils/dataReportHelpers'; +const axios = require('axios'); + +export default function (router: IRouter) { + //generate report csv meta data + router.post( + { + path: `${API_PREFIX}/data-report/metadata`, + validate: { + body: schema.any(), + }, + }, + async ( + context, + request, + response + ): Promise> => { + // input validation + try { + dataReportSchema.validate(request.body); + } catch (error) { + return response.badRequest({ body: error }); + } + try { + let dataReport = request.body; + metaData.saved_search_id = dataReport.saved_search_id; + metaData.report_format = dataReport.report_format; + metaData.start = dataReport.start; + metaData.end = dataReport.end; + let resIndexPattern: any = {}; + + //get the saved search infos + const ssParams = { + index: '.kibana', + id: 'search:' + dataReport.saved_search_id, + }; + + const ssInfos = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'get', + ssParams + ); + + // get the sorting + metaData.sorting = ssInfos._source.search.sort; + + // get the saved search type + metaData.type = ssInfos._source.type; + + // get the filters + metaData.filters = + ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; + + //get the list of selected columns in the saved search.Otherwise select all the fields under the _source + await getSelectedFields(ssInfos._source.search.columns); + + //Get index name + for (let item of ssInfos._source.references) { + if (item.name === JSON.parse(metaData.filters).indexRefName) { + //Get index-pattern informations + const indexPattern = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'get', + { + index: '.kibana', + id: 'index-pattern:' + item.id, + } + ); + resIndexPattern = indexPattern._source['index-pattern']; + metaData.paternName = resIndexPattern.title; + (metaData.timeFieldName = resIndexPattern.timeFieldName), + (metaData.fields = resIndexPattern.fields); //Get all fields + //Getting fields of type Date + for (let item of JSON.parse(metaData.fields)) { + if (item.type === 'date') { + metaData.dateFields.push(item.name); + } + } + } + } + + //save the meta data to the dataReport index to be updated with the right mapping + const report = await context.core.elasticsearch.adminClient.callAsInternalUser( + 'index', + { + index: 'datareport', + body: metaData, + } + ); + + return response.ok({ + body: { report, metaData }, + headers: { + 'content-type': 'application/json', + }, + }); + } catch (error) { + //@ts-ignore + context.reporting_plugin.logger.error( + `Failed to generate the report meta data: ${error}` + ); + return response.custom({ + statusCode: error.statusCode || 500, + body: parseEsErrorResponse(error), + }); + } + } + ); +} diff --git a/server/routes/index.ts b/server/routes/index.ts index 866a92f9..e46b2a31 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -15,11 +15,15 @@ import registerReportRoute from './report'; import registerReportDefinitionRoute from './reportDefinition'; +import registerDataReport from './dataReport'; +import registerDataReportMetadata from './dataReportMetadata'; import dashboardRoute from './getDashboards'; import { IRouter } from '../../../../src/core/server'; export default function (router: IRouter) { - registerReportRoute(router); - registerReportDefinitionRoute(router); + registerReportRoute(router); + registerReportDefinitionRoute(router); + registerDataReportMetadata(router); + registerDataReport(router); dashboardRoute(router); } diff --git a/server/routes/utils/__tests__/dataReport.test.ts b/server/routes/utils/__tests__/dataReport.test.ts new file mode 100644 index 00000000..dd3d5436 --- /dev/null +++ b/server/routes/utils/__tests__/dataReport.test.ts @@ -0,0 +1,589 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import 'regenerator-runtime/runtime'; +const axios = require('axios'); + +describe('data report metadata tests suites', () => { + test('test to generate data report meta data successfully', async () => { + expect.assertions(5); + const url = '/api/reporting/data-report/metadata'; + const input = { + saved_search_id: '571aaf70-4c88-11e8-b3d7-01146121b73d', + start: '1343576635300', + end: '1596037435301', + report_format: 'csv', + }; + let response: any = {}; + const report = await axios({ + method: 'POST', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + data: input, + }).then((res) => { + response = res.data; + }); + const { + saved_search_id, + report_format, + start, + end, + paternName, + } = response.metaData; + expect(saved_search_id).toEqual(input.saved_search_id); + expect(paternName).toEqual('kibana_sample_data_flights'); + expect(start).toEqual(input.start); + expect(end).toEqual(input.end); + expect(report_format).toEqual('csv'); + }, 20000); + + test("test to generate data report meta data Report doesn't exist", async () => { + expect.assertions(1); + const url = '/api/reporting/data-report/metadata'; + const input = { + saved_search_id: '571aaf70-4c88-11e8-b3d7-01146121b73d0', + start: '1343576635300', + end: '1596037435301', + report_format: 'csv', + }; + let response: any = {}; + let message: string = ''; + const report = await axios({ + method: 'POST', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + data: input, + }) + .then((res) => { + response = res.data; + }) + .catch((error) => { + message = error.response.data.message; + }); + expect(message).toEqual("Saved Search doesn't exist !"); + }, 20000); +}); + +describe('data report data generation tests suites', () => { + describe('test for Case 1 No args ', () => { + test('esCount > default_fetch_size => default_fetch_size', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.default_max_size); + }, 20000); + + test('esCount < default_fetch_size => esCount ', async () => { + expect.assertions(1); + const input = { + reportId: 'LNf9-3MBBSuiES58df_j', + esCount: 818, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + expect(datasetCount).toEqual(input.esCount); + }, 20000); + }); + + describe('test for Case 2: arg => nbRows ', () => { + test('nbRows == esCount => esCount', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 111396, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.esCount); + }, 20000); + + test('nbRows > esCount < default_max_size => esCount', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 200000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.esCount); + }, 20000); + + test('nbRows > esCount > default_max_size => default_max_size', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 200000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.esCount); + }, 20000); + + test('nbRows > default_max_size && nbRows < esCount => nbRows ', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 30000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.nbRows); + }, 20000); + + test('nbRows < default_max_size => nbRows', async () => { + expect.assertions(1); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 5000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + }, + }).then((res) => { + response = res.data; + }); + const { datasetCount } = response; + + expect(datasetCount).toEqual(input.nbRows); + }, 20000); + }); + + describe('test for Case 2: arg => scroll_size ', () => { + test('esCount > default_max_size && scroll_size > default_max_size => esCount', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + scroll_size: 200000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + let scrolls = Math.floor(input.esCount / input.default_max_size); + const { datasetCount, nbScroll } = response; + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount > default_max_size && scroll_size < default_max_size ', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + scroll_size: 8000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + let scrolls = Math.floor(input.esCount / input.scroll_size); + const { datasetCount, nbScroll } = response; + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount < default_max_size ', async () => { + expect.assertions(2); + const input = { + reportId: 'LNf9-3MBBSuiES58df_j', + scroll_size: 200000, + esCount: 818, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + let scrolls = 0; + const { datasetCount, nbScroll } = response; + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + }); + + describe('test for Case 3: args => nbRows && scroll_size ', () => { + test('nbRows > esCount > default_max_size && scroll_size < default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 200000, + scroll_size: 200, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + let scrolls = Math.floor(input.esCount / input.scroll_size); + const { datasetCount, nbScroll } = response; + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('nbRows > esCount > default_max_size && scroll_size > default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 200000, + scroll_size: 12000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = Math.floor(input.esCount / input.default_max_size); + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('nbRows > esCount < default_max_size && scroll_size > default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'LNf9-3MBBSuiES58df_j', + nbRows: 200000, + scroll_size: 18000, + esCount: 818, + default_max_size: 10000, + }; + + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = 0; + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('nbRows > esCount < default_max_size && scroll_size < default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'LNf9-3MBBSuiES58df_j', + nbRows: 200000, + scroll_size: 8000, + esCount: 818, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = 0; + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.esCount); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount > nbRows < default_max_size && scroll_size > default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 5000, + scroll_size: 28000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = 0; + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.nbRows); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount > nbRows < default_max_size && scroll_size < default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 5000, + scroll_size: 8000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = 0; + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.nbRows); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount > nbRows > default_max_size && scroll_size > default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 20000, + scroll_size: 18000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = Math.floor(input.nbRows / input.default_max_size); + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.nbRows); + expect(nbScroll).toEqual(scrolls); + }, 20000); + + test('esCount > nbRows > default_max_size && scroll_size < default_max_size', async () => { + expect.assertions(2); + const input = { + reportId: 'ldfN-3MBBSuiES58RP5j', + nbRows: 15000, + scroll_size: 7000, + esCount: 111396, + default_max_size: 10000, + }; + let url = '/api/reporting/data-report/generate/' + input.reportId; + let response: any = {}; + const data = await axios({ + method: 'GET', + proxy: { host: '127.0.0.1', port: 5601 }, + url, + headers: { 'kbn-xsrf': 'reporting' }, + params: { + nbRows: input.nbRows, + scroll_size: input.scroll_size, + }, + }).then((res) => { + response = res.data; + }); + + let scrolls = Math.floor(input.nbRows / input.scroll_size); + const { datasetCount, nbScroll } = response; + + expect(datasetCount).toEqual(input.nbRows); + expect(nbScroll).toEqual(scrolls); + }, 20000); + }); +}); diff --git a/server/routes/utils/constants.ts b/server/routes/utils/constants.ts index f820e210..25e7d3f7 100644 --- a/server/routes/utils/constants.ts +++ b/server/routes/utils/constants.ts @@ -41,3 +41,7 @@ export enum SCHEDULE_TYPE { future = 'Future date', cron = 'Cron based', } + +export enum DATA_REPORT_CONFIG { + excelDateFormat = 'MM/DD/YYYY h:mm:ss a', +} diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts new file mode 100644 index 00000000..13816ec8 --- /dev/null +++ b/server/routes/utils/dataReportHelpers.ts @@ -0,0 +1,212 @@ +import { async } from 'rxjs/internal/scheduler/async'; +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { DATA_REPORT_CONFIG } from './constants'; + +const esb = require('elastic-builder'); +const moment = require('moment'); +const converter = require('json-2-csv'); + +export var metaData = { + saved_search_id: null, + report_format: null, + start: null, + end: null, + fields: null, + type: null, + timeFieldName: null, + sorting: null, + fields_exist: false, + selectedFields: [], + paternName: null, + filters: [], + dateFields: [], +}; + +// Get the selected columns by the user. +export const getSelectedFields = async (columns) => { + for (let column of columns) { + if (column !== '_source') { + metaData.fields_exist = true; + metaData.selectedFields.push(column); + } else { + metaData.selectedFields.push('_source'); + } + } +}; + +//Build the ES query from the meta data +// is_count is set to 1 if we building the count query but 0 if we building the fetch data query +export const buildQuery = (report, is_count) => { + let requestBody = esb.boolQuery(); + const filters = report._source.filters; + for (let item of JSON.parse(filters).filter) { + if (item.meta.disabled === false) { + switch (item.meta.negate) { + case false: + switch (item.meta.type) { + case 'phrase': + requestBody.must( + esb.matchPhraseQuery(item.meta.key, item.meta.params.query) + ); + break; + case 'exists': + requestBody.must(esb.existsQuery(item.meta.key)); + break; + case 'phrases': + if (item.meta.value.indexOf(',') > -1) { + const valueSplit = item.meta.value.split(', '); + for (const [key, incr] of valueSplit.entries()) { + requestBody.should(esb.matchPhraseQuery(item.meta.key, incr)); + } + } else { + requestBody.should( + esb.matchPhraseQuery(item.meta.key, item.meta.params.query) + ); + } + requestBody.minimumShouldMatch(1); + break; + } + break; + case true: + switch (item.meta.type) { + case 'phrase': + requestBody.mustNot( + esb.matchPhraseQuery(item.meta.key, item.meta.params.query) + ); + break; + case 'exists': + requestBody.mustNot(esb.existsQuery(item.meta.key)); + break; + case 'phrases': + if (item.meta.value.indexOf(',') > -1) { + const valueSplit = item.meta.value.split(', '); + for (const [key, incr] of valueSplit.entries()) { + requestBody.should(esb.matchPhraseQuery(item.meta.key, incr)); + } + } else { + requestBody.should( + esb.matchPhraseQuery(item.meta.key, item.meta.params.query) + ); + } + requestBody.minimumShouldMatch(1); + break; + } + break; + } + } + } + //search part + let searchQuery = JSON.parse(filters) + .query.query.replace(/ and /g, ' AND ') + .replace(/ or /g, ' OR ') + .replace(/ not /g, ' NOT '); + if (searchQuery) { + requestBody.must(esb.queryStringQuery(searchQuery)); + } + + if (report._source.timeFieldName && report._source.timeFieldName.length > 0) { + requestBody.must( + esb + .rangeQuery(report._source.timeFieldName) + .format('epoch_millis') + .gte(report._source.start) + .lte(report._source.end) + ); + } + if (is_count) { + return esb.requestBodySearch().query(requestBody); + } + + //Add the Sort to the query + let reqBody = esb.requestBodySearch().query(requestBody).version(true); + + if (report._source.sorting.length > 0) { + if (report._source.sorting.length === 1) + reqBody.sort( + esb.sort(report._source.sorting[0][0], report._source.sorting[0][1]) + ); + else + reqBody.sort( + esb.sort(report._source.sorting[0], report._source.sorting[1]) + ); + } + + //get the selected fields only + if (report._source.fields_exist) { + reqBody.source({ includes: report._source.selectedFields }); + } + return reqBody; +}; + +// Fetch the data from ES +export const getEsData = (arrayHits, report) => { + let hits: any = []; + for (let valueRes of arrayHits) { + for (let data of valueRes.hits) { + const fields = data.fields; + //get all the fields of type date and fromat them to excel format + for (let dateType of report._source.dateFields) { + if (data._source[dateType]) { + data._source[dateType] = moment(fields[dateType][0]).format( + DATA_REPORT_CONFIG.excelDateFormat + ); + } + } + delete data['fields']; + if (report._source.fields_exist === true) { + let result = traverse(data, report._source.selectedFields); + hits.push(result); + } else { + hits.push(data); + } + } + } + return hits; +}; + +//Convert the data to Csv format +export const convertToCSV = async (dataset) => { + let convertedData: any = []; + const options = { + delimiter: { field: ',' }, + emptyFieldValue: ' ', + }; + await converter.json2csvAsync(dataset, options).then((csv) => { + convertedData = csv; + }); + return convertedData; +}; + +//Return only the selected fields +function traverse(data, keys, result = {}) { + for (let k of Object.keys(data)) { + if (keys.includes(k)) { + result = Object.assign({}, result, { + [k]: data[k], + }); + continue; + } + if ( + data[k] && + typeof data[k] === 'object' && + Object.keys(data[k]).length > 0 + ) { + result = traverse(data[k], keys, result); + } + } + return result; +} diff --git a/server/routes/utils/helpers.ts b/server/routes/utils/helpers.ts index 448451f6..cc636ed6 100644 --- a/server/routes/utils/helpers.ts +++ b/server/routes/utils/helpers.ts @@ -17,6 +17,9 @@ export function parseEsErrorResponse(error: any) { if (error.response) { try { const esErrorResponse = JSON.parse(error.response); + if (!esErrorResponse.found) { + return "Saved Search doesn't exist !"; + } return esErrorResponse.reason || error.response; } catch (parsingError) { return error.response;