[Reporting] Formatting fixes for CSV export in Discover, CSV download from Dashboard panel

x-pack/plugins/reporting/common/types.ts

@@ -6,6 +6,8 @@
 
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 export { ReportingConfigType } from '../server/config';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+export { LayoutInstance } from '../server/export_types/common/layouts';
 
 export type JobId = string;
 export type JobStatus =

x-pack/plugins/reporting/public/plugin.tsx

@@ -26,7 +26,7 @@ import {
 import { ManagementSectionId, ManagementSetup } from '../../../../src/plugins/management/public';
 import { SharePluginSetup } from '../../../../src/plugins/share/public';
 import { LicensingPluginSetup } from '../../licensing/public';
-import { ReportingConfigType, JobId, JobStatusBuckets } from '../common/types';
+import { JobId, JobStatusBuckets, ReportingConfigType } from '../common/types';
 import { JOB_COMPLETION_NOTIFICATIONS_SESSION_KEY } from '../constants';
 import { getGeneralErrorToast } from './components';
 import { ReportListing } from './components/report_listing';
@@ -144,7 +144,7 @@ export class ReportingPublicPlugin implements Plugin<void, void> {
 
     uiActions.addTriggerAction(CONTEXT_MENU_TRIGGER, action);
 
-    share.register(csvReportingProvider({ apiClient, toasts, license$ }));
+    share.register(csvReportingProvider({ apiClient, toasts, license$, uiSettings }));
     share.register(
       reportingPDFPNGProvider({
         apiClient,

x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx

@@ -5,22 +5,29 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import moment from 'moment-timezone';
 import React from 'react';
-
-import { ToastsSetup } from 'src/core/public';
+import { IUiSettingsClient, ToastsSetup } from 'src/core/public';
+import { ShareContext } from '../../../../../src/plugins/share/public';
+import { LicensingPluginSetup } from '../../../licensing/public';
+import { JobParamsDiscoverCsv, SearchRequest } from '../../server/export_types/csv/types';
 import { ReportingPanelContent } from '../components/reporting_panel_content';
-import { ReportingAPIClient } from '../lib/reporting_api_client';
 import { checkLicense } from '../lib/license_check';
-import { LicensingPluginSetup } from '../../../licensing/public';
-import { ShareContext } from '../../../../../src/plugins/share/public';
+import { ReportingAPIClient } from '../lib/reporting_api_client';
 
 interface ReportingProvider {
   apiClient: ReportingAPIClient;
   toasts: ToastsSetup;
   license$: LicensingPluginSetup['license$'];
+  uiSettings: IUiSettingsClient;
 }
 
-export const csvReportingProvider = ({ apiClient, toasts, license$ }: ReportingProvider) => {
+export const csvReportingProvider = ({
+  apiClient,
+  toasts,
+  license$,
+  uiSettings,
+}: ReportingProvider) => {
   let toolTipContent = '';
   let disabled = true;
   let hasCSVReporting = false;
@@ -33,6 +40,11 @@ export const csvReportingProvider = ({ apiClient, toasts, license$ }: ReportingP
     disabled = !enableLinks;
   });
 
+  const browserTimezone =
+    uiSettings.get('dateFormat:tz') === 'Browser'
+      ? moment.tz.guess()
+      : uiSettings.get('dateFormat:tz');
+
   const getShareMenuItems = ({
     objectType,
     objectId,
@@ -44,13 +56,19 @@ export const csvReportingProvider = ({ apiClient, toasts, license$ }: ReportingP
       return [];
     }
 
-    const getJobParams = () => {
-      return {
-        ...sharingData,
-        type: objectType,
-      };
+    const jobParams: JobParamsDiscoverCsv = {
+      browserTimezone,
+      objectType,
+      title: sharingData.title as string,
+      indexPatternId: sharingData.indexPatternId as string,
+      searchRequest: sharingData.searchRequest as SearchRequest,
+      fields: sharingData.fields as string[],
+      metaFields: sharingData.metaFields as string[],
+      conflictedTypesFields: sharingData.conflictedTypesFields as string[],
     };
 
+    const getJobParams = () => jobParams;
+
     const shareActions = [];
 
     if (hasCSVReporting) {

x-pack/plugins/reporting/public/share_context_menu/register_pdf_png_reporting.tsx

@@ -7,12 +7,15 @@
 import { i18n } from '@kbn/i18n';
 import moment from 'moment-timezone';
 import React from 'react';
-import { ToastsSetup, IUiSettingsClient } from 'src/core/public';
-import { ReportingAPIClient } from '../lib/reporting_api_client';
-import { checkLicense } from '../lib/license_check';
-import { ScreenCapturePanelContent } from '../components/screen_capture_panel_content';
-import { LicensingPluginSetup } from '../../../licensing/public';
+import { IUiSettingsClient, ToastsSetup } from 'src/core/public';
 import { ShareContext } from '../../../../../src/plugins/share/public';
+import { LicensingPluginSetup } from '../../../licensing/public';
+import { LayoutInstance } from '../../common/types';
+import { JobParamsPNG } from '../../server/export_types/png/types';
+import { JobParamsPDF } from '../../server/export_types/printable_pdf/types';
+import { ScreenCapturePanelContent } from '../components/screen_capture_panel_content';
+import { checkLicense } from '../lib/license_check';
+import { ReportingAPIClient } from '../lib/reporting_api_client';
 
 interface ReportingPDFPNGProvider {
   apiClient: ReportingAPIClient;
@@ -39,6 +42,11 @@ export const reportingPDFPNGProvider = ({
     disabled = !enableLinks;
   });
 
+  const browserTimezone =
+    uiSettings.get('dateFormat:tz') === 'Browser'
+      ? moment.tz.guess()
+      : uiSettings.get('dateFormat:tz');
+
   const getShareMenuItems = ({
     objectType,
     objectId,
@@ -57,44 +65,36 @@ export const reportingPDFPNGProvider = ({
       return [];
     }
 
-    const getReportingJobParams = () => {
+    const getPdfJobParams = (): JobParamsPDF => {
       // Relative URL must have URL prefix (Spaces ID prefix), but not server basePath
       // Replace hashes with original RISON values.
       const relativeUrl = shareableUrl.replace(
         window.location.origin + apiClient.getServerBasePath(),
         ''
       );
 
-      const browserTimezone =
-        uiSettings.get('dateFormat:tz') === 'Browser'
-          ? moment.tz.guess()
-          : uiSettings.get('dateFormat:tz');
-
       return {
-        ...sharingData,
         objectType,
         browserTimezone,
-        relativeUrls: [relativeUrl],
+        relativeUrls: [relativeUrl], // multi URL for PDF
+        layout: sharingData.layout as LayoutInstance,
+        title: sharingData.title as string,
       };
     };
 
-    const getPngJobParams = () => {
+    const getPngJobParams = (): JobParamsPNG => {
       // Replace hashes with original RISON values.
       const relativeUrl = shareableUrl.replace(
         window.location.origin + apiClient.getServerBasePath(),
         ''
       );
 
-      const browserTimezone =
-        uiSettings.get('dateFormat:tz') === 'Browser'
-          ? moment.tz.guess()
-          : uiSettings.get('dateFormat:tz');
-
       return {
-        ...sharingData,
         objectType,
         browserTimezone,
-        relativeUrl,
+        relativeUrl, // single URL for PNG
+        layout: sharingData.layout as LayoutInstance,
+        title: sharingData.title as string,
       };
     };
 
@@ -161,7 +161,7 @@ export const reportingPDFPNGProvider = ({
               reportType="printablePdf"
               objectType={objectType}
               objectId={objectId}
-              getJobParams={getReportingJobParams}
+              getJobParams={getPdfJobParams}
               isDirty={isDirty}
               onClose={onClose}
             />

x-pack/plugins/reporting/server/export_types/csv/server/create_job.ts

@@ -13,21 +13,19 @@ export const scheduleTaskFnFactory: ScheduleTaskFnFactory<ESQueueCreateJobFn<
 >> = function createJobFactoryFn(reporting) {
   const config = reporting.getConfig();
   const crypto = cryptoFactory(config.get('encryptionKey'));
-  const setupDeps = reporting.getPluginSetupDeps();
 
   return async function scheduleTask(jobParams, context, request) {
     const serializedEncryptedHeaders = await crypto.encrypt(request.headers);
 
     const savedObjectsClient = context.core.savedObjects.client;
     const indexPatternSavedObject = await savedObjectsClient.get(
       'index-pattern',
-      jobParams.indexPatternId!
+      jobParams.indexPatternId
     );
 
     return {
       headers: serializedEncryptedHeaders,
       indexPatternSavedObject,
-      basePath: setupDeps.basePath(request),
       ...jobParams,
     };
   };

x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts

@@ -4,81 +4,27 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { i18n } from '@kbn/i18n';
-import Hapi from 'hapi';
-import { IUiSettingsClient, KibanaRequest } from '../../../../../../../src/core/server';
-import {
-  CSV_QUOTE_VALUES_SETTING,
-  CSV_SEPARATOR_SETTING,
-} from '../../../../../../../src/plugins/share/server';
-import { CSV_BOM_CHARS, CSV_JOB_TYPE } from '../../../../common/constants';
-import { getFieldFormats } from '../../../../server/services';
+import { CONTENT_TYPE_CSV, CSV_JOB_TYPE } from '../../../../common/constants';
 import { cryptoFactory } from '../../../lib';
 import { ESQueueWorkerExecuteFn, RunTaskFnFactory } from '../../../types';
 import { ScheduledTaskParamsCSV } from '../types';
-import { fieldFormatMapFactory } from './lib/field_format_map';
-import { createGenerateCsv } from './lib/generate_csv';
+import { createGenerateCsv } from './generate_csv';
+import { getRequest } from './lib/get_request';
 
 export const runTaskFnFactory: RunTaskFnFactory<ESQueueWorkerExecuteFn<
   ScheduledTaskParamsCSV
 >> = function executeJobFactoryFn(reporting, parentLogger) {
   const config = reporting.getConfig();
   const crypto = cryptoFactory(config.get('encryptionKey'));
   const logger = parentLogger.clone([CSV_JOB_TYPE, 'execute-job']);
-  const serverBasePath = config.kbnConfig.get('server', 'basePath');
 
   return async function runTask(jobId, job, cancellationToken) {
     const elasticsearch = reporting.getElasticsearchService();
     const jobLogger = logger.clone([jobId]);
+    const generateCsv = createGenerateCsv(jobLogger);
 
-    const {
-      searchRequest,
-      fields,
-      indexPatternSavedObject,
-      metaFields,
-      conflictedTypesFields,
-      headers,
-      basePath,
-    } = job;
-
-    const decryptHeaders = async () => {
-      try {
-        if (typeof headers !== 'string') {
-          throw new Error(
-            i18n.translate(
-              'xpack.reporting.exportTypes.csv.executeJob.missingJobHeadersErrorMessage',
-              {
-                defaultMessage: 'Job headers are missing',
-              }
-            )
-          );
-        }
-        return await crypto.decrypt(headers);
-      } catch (err) {
-        logger.error(err);
-        throw new Error(
-          i18n.translate(
-            'xpack.reporting.exportTypes.csv.executeJob.failedToDecryptReportJobDataErrorMessage',
-            {
-              defaultMessage: 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report. {err}',
-              values: { encryptionKey: 'xpack.reporting.encryptionKey', err: err.toString() },
-            }
-          )
-        ); // prettier-ignore
-      }
-    };
-
-    const fakeRequest = KibanaRequest.from({
-      headers: await decryptHeaders(),
-      // This is used by the spaces SavedObjectClientWrapper to determine the existing space.
-      // We use the basePath from the saved job, which we'll have post spaces being implemented;
-      // or we use the server base path, which uses the default space
-      getBasePath: () => basePath || serverBasePath,
-      path: '/',
-      route: { settings: {} },
-      url: { href: '/' },
-      raw: { req: { url: '/' } },
-    } as Hapi.Request);
+    const { headers } = job;
+    const fakeRequest = await getRequest(headers, crypto, logger);
 
     const { callAsCurrentUser } = elasticsearch.legacy.client.asScoped(fakeRequest);
     const callEndpoint = (endpoint: string, clientParams = {}, options = {}) =>
@@ -87,62 +33,18 @@ export const runTaskFnFactory: RunTaskFnFactory<ESQueueWorkerExecuteFn<
     const savedObjectsClient = await reporting.getSavedObjectsClient(fakeRequest);
     const uiSettingsClient = await reporting.getUiSettingsServiceFactory(savedObjectsClient);
 
-    const getFormatsMap = async (client: IUiSettingsClient) => {
-      const fieldFormats = await getFieldFormats().fieldFormatServiceFactory(client);
-      return fieldFormatMapFactory(indexPatternSavedObject, fieldFormats);
-    };
-    const getUiSettings = async (client: IUiSettingsClient) => {
-      const [separator, quoteValues, timezone] = await Promise.all([
-        client.get(CSV_SEPARATOR_SETTING),
-        client.get(CSV_QUOTE_VALUES_SETTING),
-        client.get('dateFormat:tz'),
-      ]);
-
-      if (timezone === 'Browser') {
-        logger.warn(
-          i18n.translate('xpack.reporting.exportTypes.csv.executeJob.dateFormateSetting', {
-            defaultMessage: 'Kibana Advanced Setting "{dateFormatTimezone}" is set to "Browser". Dates will be formatted as UTC to avoid ambiguity.',
-            values: { dateFormatTimezone: 'dateFormat:tz' }
-          })
-        ); // prettier-ignore
-      }
-
-      return {
-        separator,
-        quoteValues,
-        timezone,
-      };
-    };
-
-    const [formatsMap, uiSettings] = await Promise.all([
-      getFormatsMap(uiSettingsClient),
-      getUiSettings(uiSettingsClient),
-    ]);
-
-    const generateCsv = createGenerateCsv(jobLogger);
-    const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';
-
-    const { content, maxSizeReached, size, csvContainsFormulas, warnings } = await generateCsv({
-      searchRequest,
-      fields,
-      metaFields,
-      conflictedTypesFields,
+    const { content, maxSizeReached, size, csvContainsFormulas, warnings } = await generateCsv(
+      job,
+      config,
+      uiSettingsClient,
       callEndpoint,
-      cancellationToken,
-      formatsMap,
-      settings: {
-        ...uiSettings,
-        checkForFormulas: config.get('csv', 'checkForFormulas'),
-        maxSizeBytes: config.get('csv', 'maxSizeBytes'),
-        scroll: config.get('csv', 'scroll'),
-        escapeFormulaValues: config.get('csv', 'escapeFormulaValues'),
-      },
-    });
+      cancellationToken
+    );
 
     // @TODO: Consolidate these one-off warnings into the warnings array (max-size reached and csv contains formulas)
     return {
-      content_type: 'text/csv',
-      content: bom + content,
+      content_type: CONTENT_TYPE_CSV,
+      content,
       max_size_reached: maxSizeReached,
       size,
       csv_contains_formulas: csvContainsFormulas,