added Timestamp Feature (#238)

Signed-off-by: Marco Hald <[email protected]>
opensearch-project · Nov 1, 2023 · 407edcd · 407edcd
1 parent 3b2ffdc
commit 407edcd
Show file tree

Hide file tree

Showing 4 changed files with 107 additions and 29 deletions.
diff --git a/server/routes/lib/createReport.ts b/server/routes/lib/createReport.ts
@@ -78,7 +78,8 @@ export const createReport = async (
         csvSeparator,
         allowLeadingWildcards,
         isScheduledTask,
-        logger
+        logger,
+        timezone
       );
     } else {
       // report source can only be one of [saved search, visualization, dashboard, notebook]

diff --git a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts
@@ -44,6 +44,7 @@ const input = {
 };
 
 const mockDateFormat = 'MM/DD/YYYY h:mm:ss.SSS a';
+const mockTimezone = 'UTC';
 
 /**
  * Max result window size in OpenSearch index settings.
@@ -66,7 +67,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
     expect(fileName).toContain(`test report table order_`);
   }, 20000);
@@ -79,7 +81,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
     expect(csvReport.fileName).toContain('.csv');
 
@@ -91,7 +94,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
     expect(xlsxReport.fileName).toContain('.xlsx');
   }, 20000);
@@ -106,7 +110,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
     expect(dataUrl).toEqual('');
   }, 20000);
@@ -192,7 +197,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -370,7 +376,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -473,7 +480,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male');
@@ -633,7 +641,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -745,7 +754,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -812,7 +822,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -876,7 +887,8 @@ describe('test create saved search report', () => {
       '|',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -930,7 +942,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -1021,7 +1034,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -1118,7 +1132,8 @@ describe('test create saved search report', () => {
       ',',
       true,
       undefined,
-      mockLogger
+      mockLogger,
+      mockTimezone
     );
 
     expect(dataUrl).toEqual(
@@ -1185,7 +1200,8 @@ test('create report for data set contains null field value', async () => {
     ',',
     true,
     undefined,
-    mockLogger
+    mockLogger,
+    mockTimezone
   );
 
   expect(dataUrl).toEqual(
@@ -1282,7 +1298,8 @@ test('create report for data set with metadata fields', async () => {
     ',',
     true,
     undefined,
-    mockLogger
+    mockLogger,
+    mockTimezone
   );
 
   expect(dataUrl).toEqual(
@@ -1295,6 +1312,61 @@ test('create report for data set with metadata fields', async () => {
   );
 }, 20000);
 
+test('create report with Etc/GMT-2 Timezone', async () => {
+  const hits = [
+    hit(
+      { category: 'c1', customer_gender: 'Ma', order_date: [] },
+      { order_date: [] }
+    ),
+    hit(
+      {
+        category: 'c2',
+        customer_gender: 'le',
+        order_date: ['2021-12-16T14:04:55'],
+      },
+      { order_date: ['2021-12-16T14:04:55'] }
+    ),
+    hit(
+      {
+        category: 'c3',
+        customer_gender: 'he',
+        order_date: ['2021-12-17T14:04:55', '2021-12-18T14:04:55'],
+      },
+      { order_date: ['2021-12-17T14:04:55', '2021-12-18T14:04:55'] }
+    ),
+    hit(
+      {
+        category: 'c4',
+        customer_gender: 'te',
+        order_date: '2021-12-19T14:04:55',
+      },
+      { order_date: ['2021-12-19T14:04:55'] }
+    ),
+  ];
+  const client = mockOpenSearchClient(
+    hits,
+    '"category", "customer_gender", "order_date"'
+  );
+  const { dataUrl } = await createSavedSearchReport(
+    input,
+    client,
+    mockDateFormat,
+    ',',
+    true,
+    undefined,
+    mockLogger,
+    "Etc/GMT-2"
+  );
+
+  expect(dataUrl).toEqual(
+    'category,customer_gender,order_date\n' +
+      'c1,Ma,[]\n' +
+      'c2,le,"[""12/16/2021 4:04:55.000 pm""]"\n' +
+      'c3,he,"[""12/17/2021 4:04:55.000 pm"",""12/18/2021 4:04:55.000 pm""]"\n' +
+      'c4,te,12/19/2021 4:04:55.000 pm'
+  );
+}, 20000);
+
 test('create report with empty/one/multiple(list) date values', async () => {
   const hits = [
     hit(
@@ -1365,7 +1437,8 @@ test('create report with empty/one/multiple(list) date values', async () => {
     ',',
     true,
     undefined,
-    mockLogger
+    mockLogger,
+    mockTimezone
   );
   expect(dataUrl).toEqual(
     'category,customer_gender,order_date\n' +

diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts
@@ -6,7 +6,7 @@
 import esb, { Sort } from 'elastic-builder';
 import converter from 'json-2-csv';
 import _ from 'lodash';
-import moment from 'moment';
+import moment from 'moment-timezone';
 import { DATA_REPORT_CONFIG } from './constants';
 import {
   buildOpenSearchQuery,
@@ -122,7 +122,8 @@ export const getOpenSearchData = (
   arrayHits: any,
   report: { _source: any },
   params: { excel: any; limit: number },
-  dateFormat: string
+  dateFormat: string,
+  timezone: string
 ) => {
   let hits: any = [];
   for (let valueRes of arrayHits) {
@@ -142,13 +143,13 @@ export const getOpenSearchData = (
           if (keys.length === 1) {
             // if conditions to determine if the date field's value is an array or a string
             if (typeof dateValue === 'string') {
-              data._source[keys] = moment(dateValue).format(dateFormat);
+              data._source[keys] = moment.utc(dateValue).tz(timezone).format(dateFormat);
             } else if (
               fieldDateValue.length !== 0 &&
               fieldDateValue instanceof Array
             ) {
               fieldDateValue.forEach((element, index) => {
-                data._source[keys][index] = moment(element).format(dateFormat);
+                data._source[keys][index] = moment.utc(element).tz(timezone).format(dateFormat);
               });
             } else {
               data._source[keys] = [];
@@ -158,14 +159,14 @@ export const getOpenSearchData = (
             let keyElement = keys.shift();
             // if conditions to determine if the date field's value is an array or a string
             if (typeof fieldDateValue === 'string') {
-              keys.push(moment(fieldDateValue).format(dateFormat));
+              keys.push(moment.utc(fieldDateValue).tz(timezone).format(dateFormat));
             } else if (
               fieldDateValue.length !== 0 &&
               fieldDateValue instanceof Array
             ) {
               let tempArray: string[] = [];
               fieldDateValue.forEach((index) => {
-                tempArray.push(moment(index).format(dateFormat));
+                tempArray.push(moment.utc(index).tz(timezone).format(dateFormat));
               });
               keys.push(tempArray);
             } else {
@@ -310,4 +311,4 @@ const addDocValueFields = (report: any, requestBody: any) => {
     docvalue_fields: docValues,
   };
   return requestBody;
-};
+};
diff --git a/server/routes/utils/savedSearchReportHelper.ts b/server/routes/utils/savedSearchReportHelper.ts
@@ -32,7 +32,8 @@ export async function createSavedSearchReport(
   csvSeparator: string,
   allowLeadingWildcards: boolean,
   isScheduledTask: boolean = true,
-  logger: Logger
+  logger: Logger,
+  timezone: string
 ): Promise<CreateReportResultType> {
   const params = report.report_definition.report_params;
   const reportFormat = params.core_params.report_format;
@@ -46,7 +47,8 @@ export async function createSavedSearchReport(
     csvSeparator,
     allowLeadingWildcards,
     isScheduledTask,
-    logger
+    logger,
+    timezone
   );
 
   const curTime = new Date();
@@ -134,7 +136,8 @@ async function generateReportData(
   csvSeparator: string,
   allowLeadingWildcards: boolean,
   isScheduledTask: boolean,
-  logger: Logger
+  logger: Logger,
+  timezone: string
 ) {
   let opensearchData: any = {};
   const arrayHits: any = [];
@@ -261,7 +264,7 @@ async function generateReportData(
   // Parse OpenSearch data and convert to CSV
   async function convertOpenSearchDataToCsv() {
     const dataset: any = [];
-    dataset.push(getOpenSearchData(arrayHits, report, params, dateFormat));
+    dataset.push(getOpenSearchData(arrayHits, report, params, dateFormat, timezone));
     return await convertToCSV(dataset, csvSeparator);
   }
 }