From 49209132bfbae02f8c27cd0c61e0fc860b874b21 Mon Sep 17 00:00:00 2001 From: Stefano Ricci <1219739+SteRiccio@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:04:08 +0200 Subject: [PATCH] Feature: data query storage (#3363) * data query dialog (WIP) * data query edit panel (WIP) * data query load/update * added data query delete * data query reset * layout adjustments * code cleanup * updated arena-core and arena-server versions * bug fixes (query cleanup) * keep save button enabled * code cleanup; fixed issue updating query summary; * layout adjustments * prepare query summary validation * make data query serializable * added form validation * added form validation --------- Co-authored-by: Stefano Ricci Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- common/model/query/query/defaults.js | 2 +- common/model/query/query/query.js | 39 ++-- core/i18n/resources/en.js | 17 +- package.json | 4 +- server/modules/reporting/api/reportingApi.js | 19 +- server/modules/surveyRdb/api/surveyRdbApi.js | 12 +- .../surveyRdb/manager/surveyRdbCsvExport.js | 2 +- .../surveyRdb/repository/dataView/readAgg.js | 3 +- .../DataQuery/ButtonBar/ButtonBar.js | 94 +++++---- .../ButtonDownload/ButtonDownload.js | 2 +- .../ButtonBar/ButtonFilter/ButtonFilter.js | 7 +- .../ButtonShowQueries/ButtonShowQueries.js | 49 +++++ .../DataQueriesPanel/DataQueriesPanel.js | 96 +++++++++ .../DataQueriesPanel/DataQueriesPanel.scss | 55 ++++++ .../DataQueriesPanel/DataQueryEditForm.js | 51 +++++ .../DataQuerySummaryValidator.js | 14 ++ .../DataQueriesPanel/index.js | 1 + .../DataQueriesPanel/useDataQueriesPanel.js | 182 ++++++++++++++++++ .../ButtonBar/ButtonShowQueries/index.js | 1 + .../ButtonBar/ButtonSort/ButtonSort.js | 7 +- .../DataQuery/ButtonBar/buttonBar.scss | 23 +-- .../DataQuery/ButtonBar/store/state/state.js | 10 +- .../DataQuery/ButtonBar/store/useButtonBar.js | 5 +- webapp/components/DataQuery/DataQuery.js | 3 + .../DataQuerySelectedAttributes.js | 2 +- .../Table/Row/Column/store/useColumn.js | 7 +- .../Visualizer/Table/store/useTable.js | 2 +- .../DataQuery/store/actions/useFetchCount.js | 4 +- .../DataQuery/store/actions/useFetchData.js | 3 +- .../DataQuery/store/useDataQuery.js | 6 +- webapp/components/buttons/ButtonNew.js | 9 +- webapp/components/buttons/ButtonSave.js | 2 +- webapp/components/form/buttonGroup.js | 16 +- .../NodeDefsSelectorAggregate.js | 16 +- webapp/service/api/data/index.js | 3 +- webapp/service/api/dataQuery/index.js | 45 +++++ webapp/service/api/index.js | 7 + webapp/style/table.scss | 2 +- yarn.lock | 48 ++--- 39 files changed, 698 insertions(+), 172 deletions(-) create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/ButtonShowQueries.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/DataQueriesPanel.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/DataQueriesPanel.scss create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/DataQueryEditForm.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/DataQuerySummaryValidator.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/index.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/DataQueriesPanel/useDataQueriesPanel.js create mode 100644 webapp/components/DataQuery/ButtonBar/ButtonShowQueries/index.js create mode 100644 webapp/service/api/dataQuery/index.js diff --git a/common/model/query/query/defaults.js b/common/model/query/query/defaults.js index b6ad60668d..dafe6bc8d9 100644 --- a/common/model/query/query/defaults.js +++ b/common/model/query/query/defaults.js @@ -10,5 +10,5 @@ export const defaults = { [keys.entityDefUuid]: null, [keys.attributeDefUuids]: [], [keys.dimensions]: [], - [keys.measures]: new Map(), + [keys.measures]: {}, } diff --git a/common/model/query/query/query.js b/common/model/query/query/query.js index 882706d7ed..971eda5953 100644 --- a/common/model/query/query/query.js +++ b/common/model/query/query/query.js @@ -27,21 +27,25 @@ export const create = ({ }) // ====== READ -export { displayTypes } +export { displayTypes, modes } +const getPropOrDefault = (key) => A.propOr(defaults[key], key) export const getMode = A.prop(keys.mode) export const getDisplayType = A.prop(keys.displayType) export const getFilter = A.prop(keys.filter) export const getFilterRecordUuid = A.prop(keys.filterRecordUuid) export const getFilterRecordUuids = A.prop(keys.filterRecordUuids) -export const getSort = A.prop(keys.sort) +export const getSort = getPropOrDefault(keys.sort) export const getEntityDefUuid = A.prop(keys.entityDefUuid) -export const getAttributeDefUuids = A.prop(keys.attributeDefUuids) -export const getDimensions = A.prop(keys.dimensions) -export const getMeasures = A.prop(keys.measures) +export const getAttributeDefUuids = getPropOrDefault(keys.attributeDefUuids) +export const getDimensions = getPropOrDefault(keys.dimensions) +export const getMeasures = getPropOrDefault(keys.measures) +export const getMeasuresKeys = A.pipe(getMeasures, Object.keys) +export const getMeasureAggregateFunctions = (nodeDefUuid) => (query) => getMeasures(query)?.[nodeDefUuid] ?? [] // mode const isMode = (mode) => (query) => getMode(query) === mode export const isModeAggregate = isMode(modes.aggregate) +export const isModeRaw = isMode(modes.raw) export const isModeRawEdit = isMode(modes.rawEdit) // utils @@ -59,23 +63,24 @@ export const assocFilter = A.assoc(keys.filter) export const assocFilterRecordUuid = A.assoc(keys.filterRecordUuid) export const assocFilterRecordUuids = A.assoc(keys.filterRecordUuids) export const assocSort = A.assoc(keys.sort) -export const assocMode = A.assoc(keys.mode) -// mode -export const toggleModeAggregate = (query) => ({ - ...create({ entityDefUuid: getEntityDefUuid(query) }), - [keys.mode]: isModeAggregate(query) ? modes.raw : modes.aggregate, -}) -export const toggleModeEdit = (query) => ({ - ...query, - [keys.mode]: isModeRawEdit(query) ? modes.raw : modes.rawEdit, -}) +const cleanup = (query) => { + if (isModeAggregate(query)) { + return assocAttributeDefUuids([])(query) + } else if (isModeRaw(query)) { + return A.pipe(assocDimensions(defaults[keys.dimensions]), assocMeasures(defaults[keys.measures]))(query) + } + return query +} + +export const assocMode = (mode) => A.pipe(A.assoc(keys.mode, mode), cleanup) export const toggleMeasureAggregateFunction = ({ nodeDefUuid, aggregateFn }) => (query) => { const measures = getMeasures(query) - const aggregateFns = measures.get(nodeDefUuid) + const aggregateFns = measures[nodeDefUuid] ?? [] const aggregateFnsUpdated = ArrayUtils.addOrRemoveItem({ item: aggregateFn })(aggregateFns) - return assocMeasures(measures.set(nodeDefUuid, aggregateFnsUpdated))(query) + const measuresUpdated = { ...measures, [nodeDefUuid]: aggregateFnsUpdated } + return assocMeasures(measuresUpdated)(query) } diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js index f417e913a9..5ef55b9e01 100644 --- a/core/i18n/resources/en.js +++ b/core/i18n/resources/en.js @@ -673,12 +673,21 @@ Are you sure you want to continue?`, }, dataView: { - aggregateMode: 'Aggregate Mode', + dataQuery: { + deleteConfirmMessage: 'Delete the query "{{name}}"?', + manageQueries: 'Manage queries', + mode: { + label: 'Mode:', + aggregate: 'Aggregate', + raw: 'Raw', + rawEdit: 'Raw edit', + }, + replaceQueryConfirmMessage: 'Replace current query with the selected one?', + }, editSelectedRecord: 'Edit selected record', - editMode: 'Edit Mode', filterAttributeTypes: 'Filter attribute types', filterRecords: { - buttonTitle: 'Filter records', + buttonTitle: 'Filter', expressionEditorHeader: 'Expression to filter records', }, invalidRecord: 'Invalid record', @@ -739,7 +748,7 @@ Are you sure you want to continue?`, label: 'Selected attributes:', }, showValidationReport: 'Show validation report', - sort: 'Sort records', + sort: 'Sort', dataExport: { source: { label: 'Source', diff --git a/package.json b/package.json index 5ec6b5dfbd..8b184db8c2 100644 --- a/package.json +++ b/package.json @@ -106,8 +106,8 @@ "@mui/material": "^5.15.11", "@mui/x-data-grid": "^6.19.5", "@mui/x-date-pickers": "^6.19.5", - "@openforis/arena-core": "^0.0.183", - "@openforis/arena-server": "^0.1.31", + "@openforis/arena-core": "^0.0.188", + "@openforis/arena-server": "^0.1.32", "@reduxjs/toolkit": "^2.2.1", "@sendgrid/mail": "^8.1.1", "@shopify/draggable": "^1.1.3", diff --git a/server/modules/reporting/api/reportingApi.js b/server/modules/reporting/api/reportingApi.js index 24ea6c0d4e..08a13f5d7a 100644 --- a/server/modules/reporting/api/reportingApi.js +++ b/server/modules/reporting/api/reportingApi.js @@ -35,7 +35,7 @@ export const init = (app) => { const chartSpec = A.parse(chart) - let query = A.parse(queryParam) + let query = queryParam const limit = chartSpec.chartType === 'scatterPlot' ? 10000 : null @@ -57,21 +57,20 @@ export const init = (app) => { query = Query.assocDimensions([groupByFieldUuid])(query) - // Convert measures to a Map object before passing it to Query.assocMeasures - const measures = new Map( - Object.entries({ - [metricFieldUuid]: [aggregateFunction], - }) - ) + const measures = { [metricFieldUuid]: [aggregateFunction] } query = Query.assocMeasures(measures)(query) query = Query.assocMode(mode)(query) } - const data = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, limit }) + if (Query.hasSelection(query)) { + const data = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, limit }) - const chartResult = await generateChart({ chartSpec, data }) - res.json({ chartResult }) + const chartResult = await generateChart({ chartSpec, data }) + res.json({ chartResult }) + } else { + res.json({ chartResult: [] }) + } } catch (error) { next(error) } diff --git a/server/modules/surveyRdb/api/surveyRdbApi.js b/server/modules/surveyRdb/api/surveyRdbApi.js index 0c67c8d431..9eeaac8b05 100644 --- a/server/modules/surveyRdb/api/surveyRdbApi.js +++ b/server/modules/surveyRdb/api/surveyRdbApi.js @@ -1,5 +1,3 @@ -import * as A from '../../../../core/arena' - import * as Request from '../../../utils/request' import * as Response from '../../../utils/response' import * as FileUtils from '../../../utils/file/fileUtils' @@ -23,9 +21,8 @@ export const init = (app) => { app.post('/surveyRdb/:surveyId/:nodeDefUuidTable/query', requireRecordListViewPermission, async (req, res, next) => { try { - const { surveyId, cycle, query: queryParam, offset, limit } = Request.getParams(req) + const { surveyId, cycle, query, offset, limit } = Request.getParams(req) const user = Request.getUser(req) - const query = A.parse(queryParam) const rows = await SurveyRdbService.fetchViewData({ user, surveyId, cycle, query, offset, limit }) @@ -40,11 +37,9 @@ export const init = (app) => { requireRecordListViewPermission, async (req, res, next) => { try { - const { surveyId, cycle, query: queryParam } = Request.getParams(req) + const { surveyId, cycle, query } = Request.getParams(req) const user = Request.getUser(req) - const query = A.parse(queryParam) - const count = await SurveyRdbService.countTable({ user, surveyId, cycle, query }) res.json(count) @@ -75,9 +70,8 @@ export const init = (app) => { requireRecordListViewPermission, async (req, res, next) => { try { - const { surveyId, cycle, query: queryParam } = Request.getParams(req) + const { surveyId, cycle, query } = Request.getParams(req) const user = Request.getUser(req) - const query = A.parse(queryParam) const tempFileName = await SurveyRdbService.exportViewDataToTempFile({ user, diff --git a/server/modules/surveyRdb/manager/surveyRdbCsvExport.js b/server/modules/surveyRdb/manager/surveyRdbCsvExport.js index 165adf8955..85149d8d83 100644 --- a/server/modules/surveyRdb/manager/surveyRdbCsvExport.js +++ b/server/modules/surveyRdb/manager/surveyRdbCsvExport.js @@ -102,7 +102,7 @@ const getCsvExportFieldsAgg = ({ survey, query }) => { fields.push(new ColumnNodeDef(viewDataNodeDef, nodeDefDimension).name) }) // measures - Array.from(Query.getMeasures(query).entries()).forEach(([nodeDefUuid, aggFunctions]) => { + Object.entries(Query.getMeasures(query)).forEach(([nodeDefUuid, aggFunctions]) => { const nodeDefMeasure = Survey.getNodeDefByUuid(nodeDefUuid)(survey) aggFunctions.forEach((aggregateFnOrUuid) => { const fieldAlias = ColumnNodeDef.getColumnNameAggregateFunction({ diff --git a/server/modules/surveyRdb/repository/dataView/readAgg.js b/server/modules/surveyRdb/repository/dataView/readAgg.js index 8914020f88..f076314656 100644 --- a/server/modules/surveyRdb/repository/dataView/readAgg.js +++ b/server/modules/surveyRdb/repository/dataView/readAgg.js @@ -20,8 +20,7 @@ const _getSelectQuery = ({ survey, cycle, recordOwnerUuid, query }) => { const queryBuilder = new SqlSelectAggBuilder({ viewDataNodeDef }) // SELECT measures - const measures = Query.getMeasures(query) - Array.from(measures.entries()).forEach(([nodeDefUuid, aggFunctions], index) => + Object.entries(Query.getMeasures(query)).forEach(([nodeDefUuid, aggFunctions], index) => queryBuilder.selectMeasure({ aggFunctions, nodeDefUuid, index, cycle, filter: Query.getFilter(query) }) ) diff --git a/webapp/components/DataQuery/ButtonBar/ButtonBar.js b/webapp/components/DataQuery/ButtonBar/ButtonBar.js index 22b6cfb7bc..8386dec1ba 100644 --- a/webapp/components/DataQuery/ButtonBar/ButtonBar.js +++ b/webapp/components/DataQuery/ButtonBar/ButtonBar.js @@ -5,6 +5,10 @@ import classNames from 'classnames' import { Query } from '@common/model/query' +import NodeDefLabelSwitch from '@webapp/components/survey/NodeDefLabelSwitch' +import { ButtonGroup } from '@webapp/components/form' +import { FormItem } from '@webapp/components/form/Input' + import { useIsAppSaving } from '@webapp/store/app' import { useAuthCanCleanseRecords } from '@webapp/store/user' import { useI18n } from '@webapp/store/system' @@ -13,7 +17,7 @@ import { useButtonBar } from './store' import ButtonDownload from './ButtonDownload' import ButtonFilter from './ButtonFilter' import ButtonSort from './ButtonSort' -import NodeDefLabelSwitch from '@webapp/components/survey/NodeDefLabelSwitch' +import ButtonShowQueries from './ButtonShowQueries' const ButtonBar = (props) => { const { @@ -26,71 +30,91 @@ const ButtonBar = (props) => { onChangeQuery, onNodeDefLabelTypeChange, setNodeDefsSelectorVisible, + selectedQuerySummaryUuid, + setSelectedQuerySummaryUuid, } = props const i18n = useI18n() const appSaving = useIsAppSaving() const canEdit = useAuthCanCleanseRecords() const modeEdit = Query.isModeRawEdit(query) - const modeAggregate = Query.isModeAggregate(query) const hasSelection = Query.hasSelection(query) const { Actions, state } = useButtonBar() + const queryChangeDisabled = modeEdit || !dataLoaded || dataLoading + return (
-
- + - - {canEdit && hasSelection && ( - - )} -
+ + onChangeQuery(Query.assocMode(mode)(query))} + items={[ + { + key: Query.modes.raw, + iconClassName: 'icon-file-text2', + label: 'dataView.dataQuery.mode.raw', + }, + { + key: Query.modes.aggregate, + iconClassName: 'icon-sigma', + label: 'dataView.dataQuery.mode.aggregate', + }, + ...(canEdit && hasSelection + ? [ + { + key: Query.modes.rawEdit, + iconClassName: 'icon-pencil2', + label: 'dataView.dataQuery.mode.rawEdit', + disabled: dataEmpty, + }, + ] + : []), + ]} + /> + {hasSelection && (
- +
)} + +
) } diff --git a/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js b/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js index 8bfe91ed7f..0338ba89d1 100644 --- a/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js +++ b/webapp/components/DataQuery/ButtonBar/ButtonDownload/ButtonDownload.js @@ -20,7 +20,7 @@ const ButtonDownload = (props) => { API.downloadDataQueryExport({ surveyId, cycle, entityDefUuid, tempFileName }) } - return + return } ButtonDownload.propTypes = { diff --git a/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js b/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js index c376a769e2..905cbfdd06 100644 --- a/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js +++ b/webapp/components/DataQuery/ButtonBar/ButtonFilter/ButtonFilter.js @@ -22,13 +22,14 @@ const ButtonFilter = (props) => { return ( <> - {State.showPanelFilter(state) && ( + {State.isPanelFilterShown(state) && ( { + const { disabled, query, onChangeQuery, state, Actions, selectedQuerySummaryUuid, setSelectedQuerySummaryUuid } = + props + + return ( + <> +