From 749898c17eee8c56804d46b051aaabd417772b95 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 1 Sep 2021 13:37:42 +0300 Subject: [PATCH] feat(core): add support for adhoc columns --- .../src/operators/pivotOperator.ts | 9 +++- .../src/operators/timeComparePivotOperator.ts | 10 ++++- .../src/query/extractQueryFields.ts | 6 ++- .../src/query/getColumnLabel.ts | 11 +++++ .../src/query/getMetricLabel.ts | 8 ++-- packages/superset-ui-core/src/query/index.ts | 1 + .../src/query/types/Column.ts | 16 +++++++ .../src/query/types/Metric.ts | 5 +++ .../src/query/types/QueryFormData.ts | 11 +++-- .../test/query/getColumnLabel.test.ts | 42 +++++++++++++++++++ .../src/BoxPlot/buildQuery.ts | 4 +- .../src/BoxPlot/transformProps.ts | 20 +++++---- .../src/Funnel/transformProps.ts | 14 ++++--- .../plugin-chart-echarts/src/Funnel/types.ts | 2 +- .../src/Gauge/transformProps.ts | 6 ++- .../src/Pie/transformProps.ts | 6 ++- .../src/Radar/transformProps.ts | 12 +++--- .../plugin-chart-echarts/src/utils/series.ts | 3 +- yarn.lock | 2 +- 19 files changed, 148 insertions(+), 40 deletions(-) create mode 100644 packages/superset-ui-core/src/query/getColumnLabel.ts create mode 100644 packages/superset-ui-core/test/query/getColumnLabel.test.ts diff --git a/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts b/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts index 45e5a1d173..81bdb4b18a 100644 --- a/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts +++ b/packages/superset-ui-chart-controls/src/operators/pivotOperator.ts @@ -16,7 +16,12 @@ * specific language governing permissions and limitationsxw * under the License. */ -import { ensureIsArray, getMetricLabel, PostProcessingPivot } from '@superset-ui/core'; +import { + ensureIsArray, + getColumnLabel, + getMetricLabel, + PostProcessingPivot, +} from '@superset-ui/core'; import { PostProcessingFactory } from './types'; import { TIME_COLUMN, isValidTimeCompare } from './utils'; import { timeComparePivotOperator } from './timeComparePivotOperator'; @@ -35,7 +40,7 @@ export const pivotOperator: PostProcessingFactory [metric, { operator: 'mean' }])), diff --git a/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts b/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts index 74c9c90207..41f2c2a3e7 100644 --- a/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts +++ b/packages/superset-ui-chart-controls/src/operators/timeComparePivotOperator.ts @@ -17,7 +17,13 @@ * specific language governing permissions and limitationsxw * under the License. */ -import { ComparisionType, PostProcessingPivot, NumpyFunction } from '@superset-ui/core'; +import { + ComparisionType, + PostProcessingPivot, + NumpyFunction, + ensureIsArray, + getColumnLabel, +} from '@superset-ui/core'; import { getMetricOffsetsMap, isValidTimeCompare, TIME_COMPARISON_SEPARATOR } from './utils'; import { PostProcessingFactory } from './types'; @@ -47,7 +53,7 @@ export const timeComparePivotOperator: PostProcessingFactory typeof x === 'string' && x)), + columns: removeDuplicates( + columns.filter(col => col !== ''), + getColumnLabel, + ), metrics: queryMode === QueryMode.raw ? undefined : removeDuplicates(metrics, getMetricLabel), orderby: orderby.length > 0 diff --git a/packages/superset-ui-core/src/query/getColumnLabel.ts b/packages/superset-ui-core/src/query/getColumnLabel.ts new file mode 100644 index 0000000000..3364251a11 --- /dev/null +++ b/packages/superset-ui-core/src/query/getColumnLabel.ts @@ -0,0 +1,11 @@ +import { isPhysicalColumn, QueryFormColumn } from './types'; + +export default function getColumnLabel(column: QueryFormColumn): string { + if (isPhysicalColumn(column)) { + return column; + } + if (column.label) { + return column.label; + } + return column.sqlExpression; +} diff --git a/packages/superset-ui-core/src/query/getMetricLabel.ts b/packages/superset-ui-core/src/query/getMetricLabel.ts index 23adbd0aa7..06b09b7ba6 100644 --- a/packages/superset-ui-core/src/query/getMetricLabel.ts +++ b/packages/superset-ui-core/src/query/getMetricLabel.ts @@ -1,13 +1,13 @@ -import { QueryFormMetric } from './types/QueryFormData'; +import { isAdhocMetricSimple, isSavedMetric, QueryFormMetric } from './types'; -export default function getMetricLabel(metric: QueryFormMetric) { - if (typeof metric === 'string') { +export default function getMetricLabel(metric: QueryFormMetric): string { + if (isSavedMetric(metric)) { return metric; } if (metric.label) { return metric.label; } - if (metric.expressionType === 'SIMPLE') { + if (isAdhocMetricSimple(metric)) { return `${metric.aggregate}(${metric.column.columnName || metric.column.column_name})`; } return metric.sqlExpression; diff --git a/packages/superset-ui-core/src/query/index.ts b/packages/superset-ui-core/src/query/index.ts index 2e476b2620..9bbfbc59fb 100644 --- a/packages/superset-ui-core/src/query/index.ts +++ b/packages/superset-ui-core/src/query/index.ts @@ -24,6 +24,7 @@ export { default as buildQueryContext } from './buildQueryContext'; export { default as buildQueryObject } from './buildQueryObject'; export { default as convertFilter } from './convertFilter'; export { default as extractTimegrain } from './extractTimegrain'; +export { default as getColumnLabel } from './getColumnLabel'; export { default as getMetricLabel } from './getMetricLabel'; export { default as DatasourceKey } from './DatasourceKey'; export { default as normalizeOrderBy } from './normalizeOrderBy'; diff --git a/packages/superset-ui-core/src/query/types/Column.ts b/packages/superset-ui-core/src/query/types/Column.ts index 039a33f4f1..89cdd76f33 100644 --- a/packages/superset-ui-core/src/query/types/Column.ts +++ b/packages/superset-ui-core/src/query/types/Column.ts @@ -20,6 +20,18 @@ import { GenericDataType } from './QueryResponse'; +export interface AdhocColumn { + hasCustomLabel?: boolean; + label?: string; + optionName?: string; + sqlExpression: string; +} + +/** + * A column that is physically defined in datasource. + */ +export type PhysicalColumn = string; + /** * Column information defined in datasource. */ @@ -39,3 +51,7 @@ export interface Column { } export default {}; + +export function isPhysicalColumn(column: AdhocColumn | PhysicalColumn): column is PhysicalColumn { + return typeof column === 'string'; +} diff --git a/packages/superset-ui-core/src/query/types/Metric.ts b/packages/superset-ui-core/src/query/types/Metric.ts index 953ea02acb..257b1d1d15 100644 --- a/packages/superset-ui-core/src/query/types/Metric.ts +++ b/packages/superset-ui-core/src/query/types/Metric.ts @@ -23,6 +23,7 @@ import { Column } from './Column'; export type Aggregate = 'AVG' | 'COUNT' | 'COUNT_DISTINCT' | 'MAX' | 'MIN' | 'SUM'; export interface AdhocMetricBase { + hasCustomLabel?: boolean; label?: string; optionName?: string; } @@ -66,3 +67,7 @@ export interface Metric { } export default {}; + +export function isAdhocMetricSimple(metric: AdhocMetric): metric is AdhocMetricSimple { + return metric.expressionType === 'SIMPLE'; +} diff --git a/packages/superset-ui-core/src/query/types/QueryFormData.ts b/packages/superset-ui-core/src/query/types/QueryFormData.ts index f7ba4f85dc..e99b62ebb5 100644 --- a/packages/superset-ui-core/src/query/types/QueryFormData.ts +++ b/packages/superset-ui-core/src/query/types/QueryFormData.ts @@ -29,6 +29,7 @@ import { QueryObject, QueryObjectExtras, QueryObjectFilterClause } from './Query import { TimeRange, TimeRangeEndpoints } from './Time'; import { TimeGranularity } from '../../time-format'; import { JsonObject } from '../../connection'; +import { AdhocColumn, PhysicalColumn } from './Column'; /** * Metric definition/reference in query object. @@ -37,9 +38,9 @@ export type QueryFormMetric = SavedMetric | AdhocMetric; /** * Column selects in query object (used as dimensions in both groupby or raw - * query mode). Only support referring existing columns. + * query mode). Can be either reference to physical column or expresion. */ -export type QueryFormColumn = string; +export type QueryFormColumn = PhysicalColumn | AdhocColumn; /** * Order query results by columns. @@ -167,7 +168,7 @@ export interface BaseFormData extends TimeRange, FormDataResidual { /** row offset for server side pagination */ row_offset?: string | number | null; /** The metric used to order timeseries for limiting */ - timeseries_limit_metric?: QueryFormColumn; + timeseries_limit_metric?: QueryFormMetric; /** Force refresh */ force?: boolean; result_format?: string; @@ -209,4 +210,8 @@ export function isDruidFormData(formData: QueryFormData): formData is DruidFormD return 'granularity' in formData; } +export function isSavedMetric(metric: QueryFormMetric): metric is SavedMetric { + return typeof metric === 'string'; +} + export default {}; diff --git a/packages/superset-ui-core/test/query/getColumnLabel.test.ts b/packages/superset-ui-core/test/query/getColumnLabel.test.ts new file mode 100644 index 0000000000..865463481c --- /dev/null +++ b/packages/superset-ui-core/test/query/getColumnLabel.test.ts @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { getColumnLabel } from '@superset-ui/core/src/query'; + +describe('getColumnLabel', () => { + it('should handle physical column', () => { + expect(getColumnLabel('gender')).toEqual('gender'); + }); + + it('should handle adhoc columns with label', () => { + expect( + getColumnLabel({ + sqlExpression: "case when 1 then 'a' else 'b' end", + label: 'my col', + }), + ).toEqual('my col'); + }); + + it('should handle adhoc columns without label', () => { + expect( + getColumnLabel({ + sqlExpression: "case when 1 then 'a' else 'b' end", + }), + ).toEqual("case when 1 then 'a' else 'b' end"); + }); +}); diff --git a/plugins/plugin-chart-echarts/src/BoxPlot/buildQuery.ts b/plugins/plugin-chart-echarts/src/BoxPlot/buildQuery.ts index 30b3cdc053..aa74c93e7a 100644 --- a/plugins/plugin-chart-echarts/src/BoxPlot/buildQuery.ts +++ b/plugins/plugin-chart-echarts/src/BoxPlot/buildQuery.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { buildQueryContext, getMetricLabel } from '@superset-ui/core'; +import { buildQueryContext, getColumnLabel, getMetricLabel } from '@superset-ui/core'; import { BoxPlotQueryFormData, BoxPlotQueryObjectWhiskerType } from './types'; const PERCENTILE_REGEX = /(\d+)\/(\d+) percentiles/; @@ -49,7 +49,7 @@ export default function buildQuery(formData: BoxPlotQueryFormData) { options: { whisker_type: whiskerType, percentiles, - groupby: columns.filter(x => !distributionColumns.includes(x)), + groupby: columns.map(getColumnLabel).filter(x => !distributionColumns.includes(x)), metrics: metrics.map(getMetricLabel), }, }, diff --git a/plugins/plugin-chart-echarts/src/BoxPlot/transformProps.ts b/plugins/plugin-chart-echarts/src/BoxPlot/transformProps.ts index ad16c2b54d..29b61e181c 100644 --- a/plugins/plugin-chart-echarts/src/BoxPlot/transformProps.ts +++ b/plugins/plugin-chart-echarts/src/BoxPlot/transformProps.ts @@ -19,6 +19,7 @@ import { CategoricalColorNamespace, DataRecordValue, + getColumnLabel, getMetricLabel, getNumberFormatter, getTimeFormatter, @@ -43,8 +44,8 @@ export default function transformProps( const coltypeMapping = getColtypesMapping(queriesData[0]); const { colorScheme, - groupby = [], - metrics: formdataMetrics = [], + groupby: formDataGroupby = [], + metrics: formDataMetrics = [], numberFormat, dateFormat, xTicksLayout, @@ -52,13 +53,14 @@ export default function transformProps( } = formData as BoxPlotQueryFormData; const colorFn = CategoricalColorNamespace.getScale(colorScheme as string); const numberFormatter = getNumberFormatter(numberFormat); - const metricLabels = formdataMetrics.map(getMetricLabel); + const metricLabels = formDataMetrics.map(getMetricLabel); + const groupbyLabels = formDataGroupby.map(getColumnLabel); const transformedData = data .map((datum: any) => { const groupbyLabel = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping, timeFormatter: getTimeFormatter(dateFormat), }); @@ -91,7 +93,7 @@ export default function transformProps( metricLabels.map(metric => { const groupbyLabel = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping, timeFormatter: getTimeFormatter(dateFormat), }); @@ -106,7 +108,7 @@ export default function transformProps( tooltip: { formatter: (param: { data: [string, number] }) => { const [outlierName, stats] = param.data; - const headline = groupby + const headline = groupbyLabels.length ? `

${sanitizeHtml(outlierName)}

` : ''; return `${headline}${numberFormatter(stats)}`; @@ -124,13 +126,13 @@ export default function transformProps( const labelMap = data.reduce((acc: Record, datum) => { const label = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping, timeFormatter: getTimeFormatter(dateFormat), }); return { ...acc, - [label]: groupby.map(col => datum[col]), + [label]: groupbyLabels.map(col => datum[col]), }; }, {}); @@ -224,7 +226,7 @@ export default function transformProps( setDataMask, emitFilter, labelMap, - groupby, + groupby: groupbyLabels, selectedValues, }; } diff --git a/plugins/plugin-chart-echarts/src/Funnel/transformProps.ts b/plugins/plugin-chart-echarts/src/Funnel/transformProps.ts index 63c2464c5c..7450cf668c 100644 --- a/plugins/plugin-chart-echarts/src/Funnel/transformProps.ts +++ b/plugins/plugin-chart-echarts/src/Funnel/transformProps.ts @@ -24,6 +24,7 @@ import { getNumberFormatter, NumberFormats, NumberFormatter, + getColumnLabel, } from '@superset-ui/core'; import { CallbackDataParams } from 'echarts/types/src/util/types'; import { EChartsOption, FunnelSeriesOption } from 'echarts'; @@ -107,16 +108,19 @@ export default function transformProps( ...formData, }; const metricLabel = getMetricLabel(metric); - const keys = data.map(datum => extractGroupbyLabel({ datum, groupby, coltypeMapping: {} })); + const groupbyLabels = groupby.map(getColumnLabel); + const keys = data.map(datum => + extractGroupbyLabel({ datum, groupby: groupbyLabels, coltypeMapping: {} }), + ); const labelMap = data.reduce((acc: Record, datum) => { const label = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping: {}, }); return { ...acc, - [label]: groupby.map(col => datum[col]), + [label]: groupbyLabels.map(col => datum[col]), }; }, {}); @@ -126,7 +130,7 @@ export default function transformProps( const numberFormatter = getNumberFormatter(numberFormat); const transformedData: FunnelSeriesOption[] = data.map(datum => { - const name = extractGroupbyLabel({ datum, groupby, coltypeMapping: {} }); + const name = extractGroupbyLabel({ datum, groupby: groupbyLabels, coltypeMapping: {} }); const isFiltered = filterState.selectedValues && !filterState.selectedValues.includes(name); return { value: datum[metricLabel], @@ -214,7 +218,7 @@ export default function transformProps( setDataMask, emitFilter, labelMap, - groupby, + groupby: groupbyLabels, selectedValues, }; } diff --git a/plugins/plugin-chart-echarts/src/Funnel/types.ts b/plugins/plugin-chart-echarts/src/Funnel/types.ts index a265efca03..d766f62508 100644 --- a/plugins/plugin-chart-echarts/src/Funnel/types.ts +++ b/plugins/plugin-chart-echarts/src/Funnel/types.ts @@ -34,7 +34,7 @@ import { export type EchartsFunnelFormData = QueryFormData & EchartsLegendFormData & { colorScheme?: string; - groupby: string[]; + groupby: QueryFormData[]; labelLine: boolean; labelType: EchartsFunnelLabelTypeType; metric?: string; diff --git a/plugins/plugin-chart-echarts/src/Gauge/transformProps.ts b/plugins/plugin-chart-echarts/src/Gauge/transformProps.ts index ccd099376b..8ef03a4d61 100644 --- a/plugins/plugin-chart-echarts/src/Gauge/transformProps.ts +++ b/plugins/plugin-chart-echarts/src/Gauge/transformProps.ts @@ -24,6 +24,7 @@ import { getNumberFormatter, getMetricLabel, DataRecordValue, + getColumnLabel, } from '@superset-ui/core'; import { EChartsOption, GaugeSeriesOption } from 'echarts'; import { GaugeDataItemOption } from 'echarts/types/src/chart/gauge/GaugeSeries'; @@ -110,6 +111,7 @@ export default function transformProps( const axisLabelLength = Math.max( ...axisLabels.map(label => numberFormatter(label).length).concat([1]), ); + const groupbyLabels = groupby.map(getColumnLabel); const formatValue = (value: number) => valueFormatter.replace('{value}', numberFormatter(value)); const axisTickLength = FONT_SIZE_MULTIPLIERS.axisTickLength * fontSize; const splitLineLength = FONT_SIZE_MULTIPLIERS.splitLineLength * fontSize; @@ -124,10 +126,10 @@ export default function transformProps( const columnsLabelMap = new Map(); const transformedData: GaugeDataItemOption[] = data.map((data_point, index) => { - const name = groupby.map(column => `${column}: ${data_point[column]}`).join(', '); + const name = groupbyLabels.map(column => `${column}: ${data_point[column]}`).join(', '); columnsLabelMap.set( name, - groupby.map(col => data_point[col]), + groupbyLabels.map(col => data_point[col]), ); let item: GaugeDataItemOption = { value: data_point[getMetricLabel(metric as QueryFormMetric)] as number, diff --git a/plugins/plugin-chart-echarts/src/Pie/transformProps.ts b/plugins/plugin-chart-echarts/src/Pie/transformProps.ts index 1f621ebc35..0673fe3ae0 100644 --- a/plugins/plugin-chart-echarts/src/Pie/transformProps.ts +++ b/plugins/plugin-chart-echarts/src/Pie/transformProps.ts @@ -19,6 +19,7 @@ import { CategoricalColorNamespace, DataRecordValue, + getColumnLabel, getMetricLabel, getNumberFormatter, getTimeFormatter, @@ -107,6 +108,7 @@ export default function transformProps(chartProps: EchartsPieChartProps): PieCha emitFilter, }: EchartsPieFormData = { ...DEFAULT_LEGEND_FORM_DATA, ...DEFAULT_PIE_FORM_DATA, ...formData }; const metricLabel = getMetricLabel(metric); + const groupbyLabels = groupby.map(getColumnLabel); const minShowLabelAngle = (showLabelsThreshold || 0) * 3.6; const keys = data.map(datum => @@ -120,13 +122,13 @@ export default function transformProps(chartProps: EchartsPieChartProps): PieCha const labelMap = data.reduce((acc: Record, datum) => { const label = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping, timeFormatter: getTimeFormatter(dateFormat), }); return { ...acc, - [label]: groupby.map(col => datum[col]), + [label]: groupbyLabels.map(col => datum[col]), }; }, {}); diff --git a/plugins/plugin-chart-echarts/src/Radar/transformProps.ts b/plugins/plugin-chart-echarts/src/Radar/transformProps.ts index b83e76725a..4b1148ae15 100644 --- a/plugins/plugin-chart-echarts/src/Radar/transformProps.ts +++ b/plugins/plugin-chart-echarts/src/Radar/transformProps.ts @@ -20,6 +20,7 @@ import { CategoricalColorNamespace, DataRecordValue, ensureIsInt, + getColumnLabel, getMetricLabel, getNumberFormatter, getTimeFormatter, @@ -99,7 +100,8 @@ export default function transformProps( labelType, }); - const metricsLabel = metrics.map(metric => getMetricLabel(metric)); + const metricLabels = metrics.map(getMetricLabel); + const groupbyLabels = groupby.map(getColumnLabel); const metricLabelAndMaxValueMap = new Map(); const columnsLabelMap = new Map(); @@ -107,14 +109,14 @@ export default function transformProps( data.forEach(datum => { const joinedName = extractGroupbyLabel({ datum, - groupby, + groupby: groupbyLabels, coltypeMapping, timeFormatter: getTimeFormatter(dateFormat), }); // map(joined_name: [columnLabel_1, columnLabel_2, ...]) columnsLabelMap.set( joinedName, - groupby.map(col => datum[col]), + groupbyLabels.map(col => datum[col]), ); // put max value of series into metricLabelAndMaxValueMap @@ -138,7 +140,7 @@ export default function transformProps( // generate transformedData transformedData.push({ - value: metricsLabel.map(metricLabel => datum[metricLabel]), + value: metricLabels.map(metricLabel => datum[metricLabel]), name: joinedName, itemStyle: { color: colorFn(joinedName), @@ -166,7 +168,7 @@ export default function transformProps( {}, ); - const indicator = metricsLabel.map(metricLabel => { + const indicator = metricLabels.map(metricLabel => { const maxValueInControl = columnConfig?.[metricLabel]?.radarMetricMaxValue; // Ensure that 0 is at the center of the polar coordinates const metricValueAsMax = diff --git a/plugins/plugin-chart-echarts/src/utils/series.ts b/plugins/plugin-chart-echarts/src/utils/series.ts index 4863007a3f..7c92ec042f 100644 --- a/plugins/plugin-chart-echarts/src/utils/series.ts +++ b/plugins/plugin-chart-echarts/src/utils/series.ts @@ -21,6 +21,7 @@ import { ChartDataResponseResult, DataRecord, DataRecordValue, + ensureIsArray, GenericDataType, NumberFormatter, TimeFormatter, @@ -112,7 +113,7 @@ export function extractGroupbyLabel({ timeFormatter?: TimeFormatter; coltypeMapping: Record; }): string { - return (groupby || []) + return ensureIsArray(groupby) .map(val => formatSeriesName(datum[val], { numberFormatter, diff --git a/yarn.lock b/yarn.lock index a3bbc40625..aa3e5fc6b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4396,7 +4396,7 @@ d3-cloud "^1.2.1" prop-types "^15.6.2" -"@superset-ui/react-pivottable@^0.12.11": +"@superset-ui/react-pivottable@^0.12.12": version "0.12.12" resolved "https://registry.yarnpkg.com/@superset-ui/react-pivottable/-/react-pivottable-0.12.12.tgz#34055c263a695e6060d3858464837c55c0118326" integrity sha512-4+wx2kQy3IRKoWHTf2bIkXjlzDA0u/eN2k0FfLfJ5bdER2GuqZErWuKtiZzARsn5kSS9hPIrvt77uv52R3FnfQ==