diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index fd38dc9d..f8617e90 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -232,6 +232,31 @@ describe('test create saved search report', () => { ); }, 20000); + test('create report for data set with nested fields', async () => { + const hits = [ + hit({ + 'geoip.country_iso_code': 'GB', + 'geoip.location': { lon: -0.1, lat: 51.5 }, + }), + hit({ + 'geoip.country_iso_code': 'US', + 'geoip.city_name': 'New York', + 'geoip.location': { lon: -74, lat: 40.8 }, + }), + ]; + const client = mockEsClient( + hits, + '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' + ); + const { dataUrl } = await createSavedSearchReport(input, client); + + expect(dataUrl).toEqual( + 'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' + + 'GB,-0.1,51.5, \n' + + 'US,-74,40.8,New York' + ); + }, 20000); + test('create report by sanitizing data set for Excel', async () => { const hits = [ hit({ category: 'c1', customer_gender: '=Male' }), @@ -281,7 +306,10 @@ describe('test create saved search report', () => { /** * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ -function mockEsClient(mockHits: Array<{ _source: any }>) { +function mockEsClient( + mockHits: Array<{ _source: any }>, + columns = '"category", "customer_gender"' +) { let call = 0; const client = jest.fn(); client.callAsInternalUser = jest @@ -292,7 +320,7 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { return { _source: params.id.startsWith('index-pattern:') ? mockIndexPattern() - : mockSavedSearch(), + : mockSavedSearch(columns), }; case 'indices.getSettings': return mockIndexSettings(); @@ -326,9 +354,9 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { } /** - * Mock a saved search for kibana_sample_data_ecommerce with 2 selected fields: category and customer_gender. + * Mock a saved search for kibana_sample_data_ecommerce with 2 default selected fields: category and customer_gender. */ -function mockSavedSearch() { +function mockSavedSearch(columns = '"category", "customer_gender"') { return JSON.parse(` { "type": "search", @@ -337,10 +365,7 @@ function mockSavedSearch() { "title": "Show category and gender", "description": "", "hits": 0, - "columns": [ - "category", - "customer_gender" - ], + "columns": [ ${columns} ], "sort": [], "version": 1, "kibanaSavedObjectMeta": { diff --git a/kibana-reports/server/routes/utils/dataReportHelpers.ts b/kibana-reports/server/routes/utils/dataReportHelpers.ts index 3f589c2d..05d675a1 100644 --- a/kibana-reports/server/routes/utils/dataReportHelpers.ts +++ b/kibana-reports/server/routes/utils/dataReportHelpers.ts @@ -18,6 +18,7 @@ import { DATA_REPORT_CONFIG } from './constants'; import esb from 'elastic-builder'; import moment from 'moment'; import converter from 'json-2-csv'; +import _ from 'lodash'; export var metaData = { saved_search_id: null, @@ -168,7 +169,7 @@ export const getEsData = (arrayHits, report, params) => { } delete data['fields']; if (report._source.fields_exist === true) { - let result = traverse(data, report._source.selectedFields); + let result = traverse(data._source, report._source.selectedFields); hits.push(params.excel ? sanitize(result) : result); } else { hits.push(params.excel ? sanitize(data) : data); @@ -196,26 +197,39 @@ export const convertToCSV = async (dataset) => { return convertedData; }; -//Return only the selected fields -function traverse(data, keys, result = {}) { - for (let k of Object.keys(data)) { - if (keys.includes(k)) { - result = Object.assign({}, result, { - [k]: data[k], - }); - continue; - } +function flattenHits(hits, result = {}, prefix = '') { + for (const [key, value] of Object.entries(hits)) { + if (!hits.hasOwnProperty(key)) continue; if ( - data[k] && - typeof data[k] === 'object' && - Object.keys(data[k]).length > 0 + value != null && + typeof value === 'object' && + !Array.isArray(value) && + Object.keys(value).length > 0 ) { - result = traverse(data[k], keys, result); + flattenHits(value, result, prefix + key + '.'); + } else { + result[prefix + key] = value; } } return result; } +//Return only the selected fields +function traverse(data, keys, result = {}) { + data = flattenHits(data); + const sourceKeys = Object.keys(data); + keys.forEach((key) => { + const value = _.get(data, key, undefined); + if (value !== undefined) result[key] = value; + else { + Object.keys(data) + .filter((sourceKey) => sourceKey.startsWith(key + '.')) + .forEach((sourceKey) => (result[sourceKey] = data[sourceKey])); + } + }); + return result; +} + /** * Escape special characters if field value prefixed with. * This is intend to avoid CSV injection in Microsoft Excel.