Feature: data query storage (#3363)

* data query dialog (WIP) * data query edit panel (WIP) * data query load/update * added data query delete * data query reset * layout adjustments * code cleanup * updated arena-core and arena-server versions * bug fixes (query cleanup) * keep save button enabled * code cleanup; fixed issue updating query summary; * layout adjustments * prepare query summary validation * make data query serializable * added form validation * added form validation --------- Co-authored-by: Stefano Ricci <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
openforis · Apr 16, 2024 · 4920913 · 4920913
1 parent ca61373
commit 4920913
Show file tree

Hide file tree

Showing 39 changed files with 698 additions and 172 deletions.
diff --git a/common/model/query/query/defaults.js b/common/model/query/query/defaults.js
@@ -10,5 +10,5 @@ export const defaults = {
   [keys.entityDefUuid]: null,
   [keys.attributeDefUuids]: [],
   [keys.dimensions]: [],
-  [keys.measures]: new Map(),
+  [keys.measures]: {},
 }
diff --git a/common/model/query/query/query.js b/common/model/query/query/query.js
@@ -27,21 +27,25 @@ export const create = ({
 })
 
 // ====== READ
-export { displayTypes }
+export { displayTypes, modes }
+const getPropOrDefault = (key) => A.propOr(defaults[key], key)
 export const getMode = A.prop(keys.mode)
 export const getDisplayType = A.prop(keys.displayType)
 export const getFilter = A.prop(keys.filter)
 export const getFilterRecordUuid = A.prop(keys.filterRecordUuid)
 export const getFilterRecordUuids = A.prop(keys.filterRecordUuids)
-export const getSort = A.prop(keys.sort)
+export const getSort = getPropOrDefault(keys.sort)
 export const getEntityDefUuid = A.prop(keys.entityDefUuid)
-export const getAttributeDefUuids = A.prop(keys.attributeDefUuids)
-export const getDimensions = A.prop(keys.dimensions)
-export const getMeasures = A.prop(keys.measures)
+export const getAttributeDefUuids = getPropOrDefault(keys.attributeDefUuids)
+export const getDimensions = getPropOrDefault(keys.dimensions)
+export const getMeasures = getPropOrDefault(keys.measures)
+export const getMeasuresKeys = A.pipe(getMeasures, Object.keys)
+export const getMeasureAggregateFunctions = (nodeDefUuid) => (query) => getMeasures(query)?.[nodeDefUuid] ?? []
 
 // mode
 const isMode = (mode) => (query) => getMode(query) === mode
 export const isModeAggregate = isMode(modes.aggregate)
+export const isModeRaw = isMode(modes.raw)
 export const isModeRawEdit = isMode(modes.rawEdit)
 
 // utils
@@ -59,23 +63,24 @@ export const assocFilter = A.assoc(keys.filter)
 export const assocFilterRecordUuid = A.assoc(keys.filterRecordUuid)
 export const assocFilterRecordUuids = A.assoc(keys.filterRecordUuids)
 export const assocSort = A.assoc(keys.sort)
-export const assocMode = A.assoc(keys.mode)
 
-// mode
-export const toggleModeAggregate = (query) => ({
-  ...create({ entityDefUuid: getEntityDefUuid(query) }),
-  [keys.mode]: isModeAggregate(query) ? modes.raw : modes.aggregate,
-})
-export const toggleModeEdit = (query) => ({
-  ...query,
-  [keys.mode]: isModeRawEdit(query) ? modes.raw : modes.rawEdit,
-})
+const cleanup = (query) => {
+  if (isModeAggregate(query)) {
+    return assocAttributeDefUuids([])(query)
+  } else if (isModeRaw(query)) {
+    return A.pipe(assocDimensions(defaults[keys.dimensions]), assocMeasures(defaults[keys.measures]))(query)
+  }
+  return query
+}
+
+export const assocMode = (mode) => A.pipe(A.assoc(keys.mode, mode), cleanup)
 
 export const toggleMeasureAggregateFunction =
   ({ nodeDefUuid, aggregateFn }) =>
   (query) => {
     const measures = getMeasures(query)
-    const aggregateFns = measures.get(nodeDefUuid)
+    const aggregateFns = measures[nodeDefUuid] ?? []
     const aggregateFnsUpdated = ArrayUtils.addOrRemoveItem({ item: aggregateFn })(aggregateFns)
-    return assocMeasures(measures.set(nodeDefUuid, aggregateFnsUpdated))(query)
+    const measuresUpdated = { ...measures, [nodeDefUuid]: aggregateFnsUpdated }
+    return assocMeasures(measuresUpdated)(query)
   }
diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js
@@ -673,12 +673,21 @@ Are you sure you want to continue?`,
   },
 
   dataView: {
-    aggregateMode: 'Aggregate Mode',
+    dataQuery: {
+      deleteConfirmMessage: 'Delete the query "{{name}}"?',
+      manageQueries: 'Manage queries',
+      mode: {
+        label: 'Mode:',
+        aggregate: 'Aggregate',
+        raw: 'Raw',
+        rawEdit: 'Raw edit',
+      },
+      replaceQueryConfirmMessage: 'Replace current query with the selected one?',
+    },
     editSelectedRecord: 'Edit selected record',
-    editMode: 'Edit Mode',
     filterAttributeTypes: 'Filter attribute types',
     filterRecords: {
-      buttonTitle: 'Filter records',
+      buttonTitle: 'Filter',
       expressionEditorHeader: 'Expression to filter records',
     },
     invalidRecord: 'Invalid record',
@@ -739,7 +748,7 @@ Are you sure you want to continue?`,
       label: 'Selected attributes:',
     },
     showValidationReport: 'Show validation report',
-    sort: 'Sort records',
+    sort: 'Sort',
     dataExport: {
       source: {
         label: 'Source',

diff --git a/package.json b/package.json
@@ -106,8 +106,8 @@
     "@mui/material": "^5.15.11",
     "@mui/x-data-grid": "^6.19.5",
     "@mui/x-date-pickers": "^6.19.5",
-    "@openforis/arena-core": "^0.0.183",
-    "@openforis/arena-server": "^0.1.31",
+    "@openforis/arena-core": "^0.0.188",
+    "@openforis/arena-server": "^0.1.32",
     "@reduxjs/toolkit": "^2.2.1",
     "@sendgrid/mail": "^8.1.1",
     "@shopify/draggable": "^1.1.3",

diff --git a/server/modules/reporting/api/reportingApi.js b/server/modules/reporting/api/reportingApi.js
@@ -35,7 +35,7 @@ export const init = (app) => {
 
       const chartSpec = A.parse(chart)
 
-      let query = A.parse(queryParam)
+      let query = queryParam
 
       const limit = chartSpec.chartType === 'scatterPlot' ? 10000 : null
 
@@ -57,21 +57,20 @@ export const init = (app) => {
 
         query = Query.assocDimensions([groupByFieldUuid])(query)
 
-        // Convert measures to a Map object before passing it to Query.assocMeasures
-        const measures = new Map(
-          Object.entries({
-            [metricFieldUuid]: [aggregateFunction],
-          })
-        )
+        const measures = { [metricFieldUuid]: [aggregateFunction] }
 
         query = Query.assocMeasures(measures)(query)
 
         query = Query.assocMode(mode)(query)
       }
-      const data = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, limit })
+      if (Query.hasSelection(query)) {
+        const data = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, limit })
 
-      const chartResult = await generateChart({ chartSpec, data })
-      res.json({ chartResult })
+        const chartResult = await generateChart({ chartSpec, data })
+        res.json({ chartResult })
+      } else {
+        res.json({ chartResult: [] })
+      }
     } catch (error) {
       next(error)
     }

diff --git a/server/modules/surveyRdb/api/surveyRdbApi.js b/server/modules/surveyRdb/api/surveyRdbApi.js
@@ -1,5 +1,3 @@
-import * as A from '../../../../core/arena'
-
 import * as Request from '../../../utils/request'
 import * as Response from '../../../utils/response'
 import * as FileUtils from '../../../utils/file/fileUtils'
@@ -23,9 +21,8 @@ export const init = (app) => {
 
   app.post('/surveyRdb/:surveyId/:nodeDefUuidTable/query', requireRecordListViewPermission, async (req, res, next) => {
     try {
-      const { surveyId, cycle, query: queryParam, offset, limit } = Request.getParams(req)
+      const { surveyId, cycle, query, offset, limit } = Request.getParams(req)
       const user = Request.getUser(req)
-      const query = A.parse(queryParam)
 
       const rows = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, offset, limit })
 
@@ -40,11 +37,9 @@ export const init = (app) => {
     requireRecordListViewPermission,
     async (req, res, next) => {
       try {
-        const { surveyId, cycle, query: queryParam } = Request.getParams(req)
+        const { surveyId, cycle, query } = Request.getParams(req)
         const user = Request.getUser(req)
 
-        const query = A.parse(queryParam)
-
         const count = await SurveyRdbService.countTable({ user, surveyId, cycle, query })
 
         res.json(count)
@@ -75,9 +70,8 @@ export const init = (app) => {
     requireRecordListViewPermission,
     async (req, res, next) => {
       try {
-        const { surveyId, cycle, query: queryParam } = Request.getParams(req)
+        const { surveyId, cycle, query } = Request.getParams(req)
         const user = Request.getUser(req)
-        const query = A.parse(queryParam)
 
         const tempFileName = await SurveyRdbService.exportViewDataToTempFile({
           user,

diff --git a/server/modules/surveyRdb/manager/surveyRdbCsvExport.js b/server/modules/surveyRdb/manager/surveyRdbCsvExport.js
@@ -102,7 +102,7 @@ const getCsvExportFieldsAgg = ({ survey, query }) => {
     fields.push(new ColumnNodeDef(viewDataNodeDef, nodeDefDimension).name)
   })
   // measures
-  Array.from(Query.getMeasures(query).entries()).forEach(([nodeDefUuid, aggFunctions]) => {
+  Object.entries(Query.getMeasures(query)).forEach(([nodeDefUuid, aggFunctions]) => {
     const nodeDefMeasure = Survey.getNodeDefByUuid(nodeDefUuid)(survey)
     aggFunctions.forEach((aggregateFnOrUuid) => {
       const fieldAlias = ColumnNodeDef.getColumnNameAggregateFunction({

diff --git a/server/modules/surveyRdb/repository/dataView/readAgg.js b/server/modules/surveyRdb/repository/dataView/readAgg.js
@@ -20,8 +20,7 @@ const _getSelectQuery = ({ survey, cycle, recordOwnerUuid, query }) => {
   const queryBuilder = new SqlSelectAggBuilder({ viewDataNodeDef })
 
   // SELECT measures
-  const measures = Query.getMeasures(query)
-  Array.from(measures.entries()).forEach(([nodeDefUuid, aggFunctions], index) =>
+  Object.entries(Query.getMeasures(query)).forEach(([nodeDefUuid, aggFunctions], index) =>
     queryBuilder.selectMeasure({ aggFunctions, nodeDefUuid, index, cycle, filter: Query.getFilter(query) })
   )
 

diff --git a/webapp/components/DataQuery/ButtonBar/ButtonBar.js b/webapp/components/DataQuery/ButtonBar/ButtonBar.js
@@ -5,6 +5,10 @@ import classNames from 'classnames'
 
 import { Query } from '@common/model/query'
 
+import NodeDefLabelSwitch from '@webapp/components/survey/NodeDefLabelSwitch'
+import { ButtonGroup } from '@webapp/components/form'
+import { FormItem } from '@webapp/components/form/Input'
+
 import { useIsAppSaving } from '@webapp/store/app'
 import { useAuthCanCleanseRecords } from '@webapp/store/user'
 import { useI18n } from '@webapp/store/system'
@@ -13,7 +17,7 @@ import { useButtonBar } from './store'
 import ButtonDownload from './ButtonDownload'
 import ButtonFilter from './ButtonFilter'
 import ButtonSort from './ButtonSort'
-import NodeDefLabelSwitch from '@webapp/components/survey/NodeDefLabelSwitch'
+import ButtonShowQueries from './ButtonShowQueries'
 
 const ButtonBar = (props) => {
   const {
@@ -26,71 +30,91 @@ const ButtonBar = (props) => {
     onChangeQuery,
     onNodeDefLabelTypeChange,
     setNodeDefsSelectorVisible,
+    selectedQuerySummaryUuid,
+    setSelectedQuerySummaryUuid,
   } = props
 
   const i18n = useI18n()
   const appSaving = useIsAppSaving()
   const canEdit = useAuthCanCleanseRecords()
   const modeEdit = Query.isModeRawEdit(query)
-  const modeAggregate = Query.isModeAggregate(query)
   const hasSelection = Query.hasSelection(query)
   const { Actions, state } = useButtonBar()
 
+  const queryChangeDisabled = modeEdit || !dataLoaded || dataLoading
+
   return (
     <div className="data-query-button-bar">
-      <div>
-        <button
-          type="button"
-          title={i18n.t(nodeDefsSelectorVisible ? 'dataView.nodeDefsSelector.hide' : 'dataView.nodeDefsSelector.show')}
-          className={classNames('btn', 'btn-s', { highlight: nodeDefsSelectorVisible })}
-          onClick={() => setNodeDefsSelectorVisible(!nodeDefsSelectorVisible)}
-        >
-          <span className="icon icon-tab icon-14px" />
-        </button>
+      <button
+        type="button"
+        title={i18n.t(nodeDefsSelectorVisible ? 'dataView.nodeDefsSelector.hide' : 'dataView.nodeDefsSelector.show')}
+        className={classNames('btn', 'btn-s', { highlight: nodeDefsSelectorVisible })}
+        onClick={() => setNodeDefsSelectorVisible(!nodeDefsSelectorVisible)}
+      >
+        <span className="icon icon-tab icon-14px" />
+      </button>
 
-        <button
-          type="button"
-          title={i18n.t('dataView.aggregateMode')}
-          className={classNames('btn', 'btn-s', 'btn-edit', { highlight: Query.isModeAggregate(query) })}
-          onClick={() => onChangeQuery(Query.toggleModeAggregate(query))}
-          aria-disabled={appSaving || modeEdit || !nodeDefsSelectorVisible}
-        >
-          <span className="icon icon-sigma icon-14px" />
-        </button>
-        {canEdit && hasSelection && (
-          <button
-            type="button"
-            title={i18n.t('dataView.editMode')}
-            className={classNames('btn', 'btn-s', 'btn-edit', { highlight: modeEdit })}
-            onClick={() => onChangeQuery(Query.toggleModeEdit(query))}
-            aria-disabled={appSaving || modeAggregate || dataEmpty || !dataLoaded}
-          >
-            <span className="icon icon-pencil2 icon-14px" />
-          </button>
-        )}
-      </div>
+      <FormItem className="mode-form-item" label={i18n.t('dataView.dataQuery.mode.label')}>
+        <ButtonGroup
+          disabled={appSaving || !nodeDefsSelectorVisible}
+          groupName="queryMode"
+          selectedItemKey={Query.getMode(query)}
+          onChange={(mode) => onChangeQuery(Query.assocMode(mode)(query))}
+          items={[
+            {
+              key: Query.modes.raw,
+              iconClassName: 'icon-file-text2',
+              label: 'dataView.dataQuery.mode.raw',
+            },
+            {
+              key: Query.modes.aggregate,
+              iconClassName: 'icon-sigma',
+              label: 'dataView.dataQuery.mode.aggregate',
+            },
+            ...(canEdit && hasSelection
+              ? [
+                  {
+                    key: Query.modes.rawEdit,
+                    iconClassName: 'icon-pencil2',
+                    label: 'dataView.dataQuery.mode.rawEdit',
+                    disabled: dataEmpty,
+                  },
+                ]
+              : []),
+          ]}
+        />
+      </FormItem>
 
       {hasSelection && (
         <div>
           <ButtonFilter
             query={query}
-            disabled={modeEdit || !dataLoaded || dataLoading}
+            disabled={queryChangeDisabled}
             onChangeQuery={onChangeQuery}
             state={state}
             Actions={Actions}
           />
           <ButtonSort
             query={query}
-            disabled={modeEdit || !dataLoaded || dataLoading}
+            disabled={queryChangeDisabled}
             onChangeQuery={onChangeQuery}
             state={state}
             Actions={Actions}
           />
-          <ButtonDownload query={query} disabled={modeEdit || !dataLoaded || dataLoading} />
+          <ButtonDownload query={query} disabled={queryChangeDisabled} />
         </div>
       )}
 
       <NodeDefLabelSwitch labelType={nodeDefLabelType} onChange={onNodeDefLabelTypeChange} />
+
+      <ButtonShowQueries
+        query={query}
+        onChangeQuery={onChangeQuery}
+        state={state}
+        Actions={Actions}
+        selectedQuerySummaryUuid={selectedQuerySummaryUuid}
+        setSelectedQuerySummaryUuid={setSelectedQuerySummaryUuid}
+      />
     </div>
   )
 }

diff --git a/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js b/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js
@@ -20,7 +20,7 @@ const ButtonDownload = (props) => {
     API.downloadDataQueryExport({ surveyId, cycle, entityDefUuid, tempFileName })
   }
 
-  return <ButtonDownloadSimple disabled={disabled} title="common.csvExport" showLabel={false} onClick={onClick} />
+  return <ButtonDownloadSimple disabled={disabled} label="common.csvExport" onClick={onClick} />
 }
 
 ButtonDownload.propTypes = {

diff --git a/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js b/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js
@@ -22,13 +22,14 @@ const ButtonFilter = (props) => {
   return (
     <>
       <ButtonIconFilter
-        className={`btn btn-s btn-edit${filter ? ' highlight' : ''}`}
+        className={`btn btn-edit${filter ? ' highlight' : ''}`}
         disabled={disabled}
         onClick={Actions.togglePanelFilter}
-        title={filter ? Expression.toString(filter, Expression.modes.sql) : 'dataView.filterRecords.buttonTitle'}
+        label="dataView.filterRecords.buttonTitle"
+        title={filter ? Expression.toString(filter, Expression.modes.sql) : undefined}
       />
 
-      {State.showPanelFilter(state) && (
+      {State.isPanelFilterShown(state) && (
         <ExpressionEditorPopup
           nodeDefUuidContext={entityDefUuid}
           query={filter ? Expression.toString(filter) : ''}