Adds a new config flag to encode with BOM for our CSVs (elastic#63006) (

elastic#63265)

* Adds a new config flag to encode with BOM for our CSVs

* Push out bom-chars to it's own constant

* Getting those snapshots back into shape 💪

Co-authored-by: Elastic Machine <[email protected]>

Co-authored-by: Joel Griffith <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

x-pack/legacy/plugins/reporting/common/constants.ts

@@ -19,6 +19,7 @@ export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv
 
 export const CONTENT_TYPE_CSV = 'text/csv';
 export const CSV_REPORTING_ACTION = 'downloadCsvReport';
+export const CSV_BOM_CHARS = '\ufeff';
 
 export const WHITELISTED_JOB_CONTENT_TYPES = [
   'application/json',

x-pack/legacy/plugins/reporting/config.ts

@@ -135,6 +135,7 @@ export async function config(Joi: any) {
         .default(),
     }).default(),
     csv: Joi.object({
+      useByteOrderMarkEncoding: Joi.boolean().default(false),
       checkForFormulas: Joi.boolean().default(true),
       enablePanelActionDownload: Joi.boolean().default(true),
       maxSizeBytes: Joi.number()

x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.test.js

@@ -13,6 +13,7 @@ import { createMockReportingCore } from '../../../test_helpers';
 import { LevelLogger } from '../../../server/lib/level_logger';
 import { setFieldFormats } from '../../../server/services';
 import { executeJobFactory } from './execute_job';
+import { CSV_BOM_CHARS } from '../../../common/constants';
 
 const delay = ms => new Promise(resolve => setTimeout(() => resolve(), ms));
 
@@ -374,6 +375,50 @@ describe('CSV Execute Job', function() {
     });
   });
 
+  describe('Byte order mark encoding', () => {
+    it('encodes CSVs with BOM', async () => {
+      configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(true);
+      callAsCurrentUserStub.onFirstCall().returns({
+        hits: {
+          hits: [{ _source: { one: 'one', two: 'bar' } }],
+        },
+        _scroll_id: 'scrollId',
+      });
+
+      const executeJob = await executeJobFactory(mockReportingPlugin, mockLogger);
+      const jobParams = {
+        headers: encryptedHeaders,
+        fields: ['one', 'two'],
+        conflictedTypesFields: [],
+        searchRequest: { index: null, body: null },
+      };
+      const { content } = await executeJob('job123', jobParams, cancellationToken);
+
+      expect(content).toEqual(`${CSV_BOM_CHARS}one,two\none,bar\n`);
+    });
+
+    it('encodes CSVs without BOM', async () => {
+      configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(false);
+      callAsCurrentUserStub.onFirstCall().returns({
+        hits: {
+          hits: [{ _source: { one: 'one', two: 'bar' } }],
+        },
+        _scroll_id: 'scrollId',
+      });
+
+      const executeJob = await executeJobFactory(mockReportingPlugin, mockLogger);
+      const jobParams = {
+        headers: encryptedHeaders,
+        fields: ['one', 'two'],
+        conflictedTypesFields: [],
+        searchRequest: { index: null, body: null },
+      };
+      const { content } = await executeJob('job123', jobParams, cancellationToken);
+
+      expect(content).toEqual('one,two\none,bar\n');
+    });
+  });
+
   describe('Elasticsearch call errors', function() {
     it('should reject Promise if search call errors out', async function() {
       callAsCurrentUserStub.rejects(new Error());

x-pack/legacy/plugins/reporting/export_types/csv/server/execute_job.ts

@@ -7,7 +7,7 @@
 import { i18n } from '@kbn/i18n';
 import Hapi from 'hapi';
 import { IUiSettingsClient, KibanaRequest } from '../../../../../../../src/core/server';
-import { CSV_JOB_TYPE } from '../../../common/constants';
+import { CSV_JOB_TYPE, CSV_BOM_CHARS } from '../../../common/constants';
 import { ReportingCore } from '../../../server/core';
 import { cryptoFactory } from '../../../server/lib';
 import { getFieldFormats } from '../../../server/services';
@@ -121,6 +121,8 @@ export const executeJobFactory: ExecuteJobFactory<ESQueueWorkerExecuteFn<
     ]);
 
     const generateCsv = createGenerateCsv(jobLogger);
+    const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';
+
     const { content, maxSizeReached, size, csvContainsFormulas } = await generateCsv({
       searchRequest,
       fields,
@@ -139,7 +141,7 @@ export const executeJobFactory: ExecuteJobFactory<ESQueueWorkerExecuteFn<
 
     return {
       content_type: 'text/csv',
-      content,
+      content: bom + content,
       max_size_reached: maxSizeReached,
       size,
       csv_contains_formulas: csvContainsFormulas,

x-pack/legacy/plugins/reporting/server/config/index.ts

@@ -108,6 +108,7 @@ export interface ReportingConfigType {
     enablePanelActionDownload: boolean;
     checkForFormulas: boolean;
     maxSizeBytes: number;
+    useByteOrderMarkEncoding: boolean;
   };
   encryptionKey: string;
   kibanaServer: any;