From a4669323d66ed3f3a242556ec1a66e683c060da7 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Wed, 27 Apr 2022 23:36:19 +0800 Subject: [PATCH] feat: add Advanced Analytics into mixed time series chart (#19851) --- .../src/sections/advancedAnalytics.tsx | 26 +- .../superset-ui-chart-controls/src/types.ts | 5 +- .../src/MixedTimeseries/buildQuery.ts | 112 +++---- .../src/MixedTimeseries/controlPanel.tsx | 24 ++ .../src/utils/formDataSuffix.ts | 74 +++++ .../test/MixedTimeseries/buildQuery.test.ts | 277 ++++++++++++++++++ .../test/utils/formDataSuffix.test.ts | 57 ++++ 7 files changed, 503 insertions(+), 72 deletions(-) create mode 100644 superset-frontend/plugins/plugin-chart-echarts/src/utils/formDataSuffix.ts create mode 100644 superset-frontend/plugins/plugin-chart-echarts/test/MixedTimeseries/buildQuery.test.ts create mode 100644 superset-frontend/plugins/plugin-chart-echarts/test/utils/formDataSuffix.test.ts diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/sections/advancedAnalytics.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/sections/advancedAnalytics.tsx index 9d4c76bb58277..13804df20da0e 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/sections/advancedAnalytics.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/sections/advancedAnalytics.tsx @@ -59,9 +59,16 @@ export const advancedAnalyticsControls: ControlPanelSectionConfig = { 'Defines the size of the rolling window function, ' + 'relative to the time granularity selected', ), - visibility: ({ controls }) => - Boolean(controls?.rolling_type?.value) && - controls.rolling_type.value !== RollingType.Cumsum, + visibility: ({ controls }, { name }) => { + // `rolling_type_b` refer to rolling_type in mixed timeseries Query B + const rollingTypeControlName = name.endsWith('_b') + ? 'rolling_type_b' + : 'rolling_type'; + return ( + Boolean(controls[rollingTypeControlName]?.value) && + controls[rollingTypeControlName]?.value !== RollingType.Cumsum + ); + }, }, }, ], @@ -79,9 +86,16 @@ export const advancedAnalyticsControls: ControlPanelSectionConfig = { 'shown are the total of 7 periods. This will hide the "ramp up" ' + 'taking place over the first 7 periods', ), - visibility: ({ controls }) => - Boolean(controls?.rolling_type?.value) && - controls.rolling_type.value !== RollingType.Cumsum, + visibility: ({ controls }, { name }) => { + // `rolling_type_b` refer to rolling_type in mixed timeseries Query B + const rollingTypeControlName = name.endsWith('_b') + ? 'rolling_type_b' + : 'rolling_type'; + return ( + Boolean(controls[rollingTypeControlName]?.value) && + controls[rollingTypeControlName]?.value !== RollingType.Cumsum + ); + }, }, }, ], diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts index 4eec6e9c0b401..73985bfc743b9 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts @@ -213,7 +213,10 @@ export interface BaseControlConfig< // TODO: add strict `chartState` typing (see superset-frontend/src/explore/types) chartState?: AnyDict, ) => ExtraControlProps; - visibility?: (props: ControlPanelsContainerProps) => boolean; + visibility?: ( + props: ControlPanelsContainerProps, + controlData: AnyDict, + ) => boolean; } export interface ControlValueValidator< diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/buildQuery.ts index 9adc149489a27..a8255b7d999fb 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/buildQuery.ts @@ -21,83 +21,65 @@ import { QueryFormData, QueryObject, normalizeOrderBy, + PostProcessingPivot, } from '@superset-ui/core'; import { pivotOperator, renameOperator, flattenOperator, + isTimeComparison, + timeComparePivotOperator, + rollingWindowOperator, + timeCompareOperator, + resampleOperator, } from '@superset-ui/chart-controls'; +import { + retainFormDataSuffix, + removeFormDataSuffix, +} from '../utils/formDataSuffix'; export default function buildQuery(formData: QueryFormData) { - const { - adhoc_filters, - adhoc_filters_b, - groupby, - groupby_b, - limit, - limit_b, - timeseries_limit_metric, - timeseries_limit_metric_b, - metrics, - metrics_b, - order_desc, - order_desc_b, - ...baseFormData - } = formData; - baseFormData.is_timeseries = true; - const formData1 = { - ...baseFormData, - adhoc_filters, - columns: groupby, - limit, - timeseries_limit_metric, - metrics, - order_desc, - }; - const formData2 = { - ...baseFormData, - adhoc_filters: adhoc_filters_b, - columns: groupby_b, - limit: limit_b, - timeseries_limit_metric: timeseries_limit_metric_b, - metrics: metrics_b, - order_desc: order_desc_b, + const baseFormData = { + ...formData, + is_timeseries: true, + columns: formData.groupby, + columns_b: formData.groupby_b, }; + const formData1 = removeFormDataSuffix(baseFormData, '_b'); + const formData2 = retainFormDataSuffix(baseFormData, '_b'); + + const queryContexts = [formData1, formData2].map(fd => + buildQueryContext(fd, baseQueryObject => { + const queryObject = { + ...baseQueryObject, + is_timeseries: true, + }; - const queryContextA = buildQueryContext(formData1, baseQueryObject => { - const queryObjectA = { - ...baseQueryObject, - is_timeseries: true, - post_processing: [ - pivotOperator(formData1, { ...baseQueryObject, is_timeseries: true }), - renameOperator(formData1, { - ...baseQueryObject, - ...{ is_timeseries: true }, - }), - flattenOperator(formData1, baseQueryObject), - ], - } as QueryObject; - return [normalizeOrderBy(queryObjectA)]; - }); + const pivotOperatorInRuntime: PostProcessingPivot = isTimeComparison( + fd, + queryObject, + ) + ? timeComparePivotOperator(fd, queryObject) + : pivotOperator(fd, queryObject); - const queryContextB = buildQueryContext(formData2, baseQueryObject => { - const queryObjectB = { - ...baseQueryObject, - is_timeseries: true, - post_processing: [ - pivotOperator(formData2, { ...baseQueryObject, is_timeseries: true }), - renameOperator(formData2, { - ...baseQueryObject, - ...{ is_timeseries: true }, - }), - flattenOperator(formData2, baseQueryObject), - ], - } as QueryObject; - return [normalizeOrderBy(queryObjectB)]; - }); + const tmpQueryObject = { + ...queryObject, + time_offsets: isTimeComparison(fd, queryObject) ? fd.time_compare : [], + post_processing: [ + pivotOperatorInRuntime, + rollingWindowOperator(fd, queryObject), + timeCompareOperator(fd, queryObject), + resampleOperator(fd, queryObject), + renameOperator(fd, queryObject), + flattenOperator(fd, queryObject), + ], + } as QueryObject; + return [normalizeOrderBy(tmpQueryObject)]; + }), + ); return { - ...queryContextA, - queries: [...queryContextA.queries, ...queryContextB.queries], + ...queryContexts[0], + queries: [...queryContexts[0].queries, ...queryContexts[1].queries], }; } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/controlPanel.tsx index 97955eec3500c..e839637885e41 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/MixedTimeseries/controlPanel.tsx @@ -18,10 +18,12 @@ */ import React from 'react'; import { t } from '@superset-ui/core'; +import { cloneDeep } from 'lodash'; import { ControlPanelConfig, ControlPanelSectionConfig, ControlSetRow, + CustomControlItem, emitFilterControl, sections, sharedControls, @@ -253,11 +255,33 @@ function createCustomizeSection( ]; } +function createAdvancedAnalyticsSection( + label: string, + controlSuffix: string, +): ControlPanelSectionConfig { + const aaWithSuffix = cloneDeep(sections.advancedAnalyticsControls); + aaWithSuffix.label = label; + if (!controlSuffix) { + return aaWithSuffix; + } + aaWithSuffix.controlSetRows.forEach(row => + row.forEach((control: CustomControlItem) => { + if (control?.name) { + // eslint-disable-next-line no-param-reassign + control.name = `${control.name}${controlSuffix}`; + } + }), + ); + return aaWithSuffix; +} + const config: ControlPanelConfig = { controlPanelSections: [ sections.legacyTimeseriesTime, createQuerySection(t('Query A'), ''), + createAdvancedAnalyticsSection(t('Advanced analytics Query A'), ''), createQuerySection(t('Query B'), '_b'), + createAdvancedAnalyticsSection(t('Advanced analytics Query B'), '_b'), { label: t('Annotations and Layers'), expanded: false, diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/utils/formDataSuffix.ts b/superset-frontend/plugins/plugin-chart-echarts/src/utils/formDataSuffix.ts new file mode 100644 index 0000000000000..c256e6f874270 --- /dev/null +++ b/superset-frontend/plugins/plugin-chart-echarts/src/utils/formDataSuffix.ts @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { QueryFormData } from '@superset-ui/core'; + +export const retainFormDataSuffix = ( + formData: QueryFormData, + controlSuffix: string, +): QueryFormData => { + /* + * retain controls by suffix and return a new formData + * eg: + * > const fd = { metrics: ['foo', 'bar'], metrics_b: ['zee'], limit: 100, ... } + * > removeFormDataSuffix(fd, '_b') + * { metrics: ['zee'], limit: 100, ... } + * */ + const newFormData = {}; + + Object.entries(formData) + .sort(([a], [b]) => { + // items contained suffix before others + const weight_a = a.endsWith(controlSuffix) ? 1 : 0; + const weight_b = b.endsWith(controlSuffix) ? 1 : 0; + return weight_b - weight_a; + }) + .forEach(([key, value]) => { + if (key.endsWith(controlSuffix)) { + newFormData[key.slice(0, -controlSuffix.length)] = value; + } + + if (!key.endsWith(controlSuffix) && !(key in newFormData)) { + // ignore duplication + newFormData[key] = value; + } + }); + + return newFormData as QueryFormData; +}; + +export const removeFormDataSuffix = ( + formData: QueryFormData, + controlSuffix: string, +): QueryFormData => { + /* + * remove unused controls by suffix and return a new formData + * eg: + * > const fd = { metrics: ['foo', 'bar'], metrics_b: ['zee'], limit: 100, ... } + * > removeUnusedFormData(fd, '_b') + * { metrics: ['foo', 'bar'], limit: 100, ... } + * */ + const newFormData = {}; + Object.entries(formData).forEach(([key, value]) => { + if (!key.endsWith(controlSuffix)) { + newFormData[key] = value; + } + }); + + return newFormData as QueryFormData; +}; diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/MixedTimeseries/buildQuery.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/MixedTimeseries/buildQuery.test.ts new file mode 100644 index 0000000000000..72f16482cb771 --- /dev/null +++ b/superset-frontend/plugins/plugin-chart-echarts/test/MixedTimeseries/buildQuery.test.ts @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { + ComparisionType, + FreeFormAdhocFilter, + RollingType, + TimeGranularity, +} from '@superset-ui/core'; +import buildQuery from '../../src/MixedTimeseries/buildQuery'; + +const formDataMixedChart = { + datasource: 'dummy', + viz_type: 'my_chart', + // query + // -- common + time_range: '1980 : 2000', + time_grain_sqla: TimeGranularity.WEEK, + granularity_sqla: 'ds', + // -- query a + groupby: ['foo'], + metrics: ['sum(sales)'], + adhoc_filters: [ + { + clause: 'WHERE', + expressionType: 'SQL', + sqlExpression: "foo in ('a', 'b')", + } as FreeFormAdhocFilter, + ], + limit: 5, + row_limit: 10, + timeseries_limit_metric: 'count', + order_desc: true, + emit_filter: true, + // -- query b + groupby_b: [], + metrics_b: ['count'], + adhoc_filters_b: [ + { + clause: 'WHERE', + expressionType: 'SQL', + sqlExpression: "name in ('c', 'd')", + } as FreeFormAdhocFilter, + ], + limit_b: undefined, + row_limit_b: 100, + timeseries_limit_metric_b: undefined, + order_desc_b: false, + emit_filter_b: undefined, + // chart configs + show_value: false, + show_valueB: undefined, +}; +const formDataMixedChartWithAA = { + ...formDataMixedChart, + rolling_type: RollingType.Cumsum, + time_compare: ['1 years ago'], + comparison_type: ComparisionType.Values, + resample_rule: '1AS', + resample_method: 'zerofill', + + rolling_type_b: RollingType.Sum, + rolling_periods_b: 1, + min_periods_b: 1, + comparison_type_b: ComparisionType.Difference, + time_compare_b: ['3 years ago'], + resample_rule_b: '1A', + resample_method_b: 'asfreq', +}; + +test('should compile query object A', () => { + const query_a = buildQuery(formDataMixedChart).queries[0]; + expect(query_a).toEqual({ + time_range: '1980 : 2000', + since: undefined, + until: undefined, + granularity: 'ds', + filters: [], + extras: { + having: '', + having_druid: [], + time_grain_sqla: 'P1W', + where: "(foo in ('a', 'b'))", + }, + applied_time_extras: {}, + columns: ['foo'], + metrics: ['sum(sales)'], + annotation_layers: [], + row_limit: 10, + row_offset: undefined, + series_columns: undefined, + series_limit: undefined, + series_limit_metric: undefined, + timeseries_limit: 5, + url_params: {}, + custom_params: {}, + custom_form_data: {}, + is_timeseries: true, + time_offsets: [], + post_processing: [ + { + operation: 'pivot', + options: { + aggregates: { + 'sum(sales)': { + operator: 'mean', + }, + }, + columns: ['foo'], + drop_missing_columns: false, + flatten_columns: false, + index: ['__timestamp'], + reset_index: false, + }, + }, + undefined, + undefined, + undefined, + { + operation: 'rename', + options: { + columns: { + 'sum(sales)': null, + }, + inplace: true, + level: 0, + }, + }, + { + operation: 'flatten', + }, + ], + orderby: [['count', false]], + }); +}); + +test('should compile query object B', () => { + const query_a = buildQuery(formDataMixedChart).queries[1]; + expect(query_a).toEqual({ + time_range: '1980 : 2000', + since: undefined, + until: undefined, + granularity: 'ds', + filters: [], + extras: { + having: '', + having_druid: [], + time_grain_sqla: 'P1W', + where: "(name in ('c', 'd'))", + }, + applied_time_extras: {}, + columns: [], + metrics: ['count'], + annotation_layers: [], + row_limit: 100, + row_offset: undefined, + series_columns: undefined, + series_limit: undefined, + series_limit_metric: undefined, + timeseries_limit: 0, + url_params: {}, + custom_params: {}, + custom_form_data: {}, + is_timeseries: true, + time_offsets: [], + post_processing: [ + { + operation: 'pivot', + options: { + aggregates: { + count: { + operator: 'mean', + }, + }, + columns: [], + drop_missing_columns: false, + flatten_columns: false, + index: ['__timestamp'], + reset_index: false, + }, + }, + undefined, + undefined, + undefined, + undefined, + { + operation: 'flatten', + }, + ], + orderby: [['count', true]], + }); +}); + +test('should compile AA in query A', () => { + const query_a = buildQuery(formDataMixedChartWithAA).queries[0]; + // time comparison + expect(query_a?.time_offsets).toEqual(['1 years ago']); + + // cumsum + expect( + // prettier-ignore + query_a + .post_processing + ?.find(operator => operator?.operation === 'cum') + ?.operation, + ).toEqual('cum'); + + // resample + expect( + // prettier-ignore + query_a + .post_processing + ?.find(operator => operator?.operation === 'resample'), + ).toEqual({ + operation: 'resample', + options: { + method: 'asfreq', + rule: '1AS', + fill_value: 0, + }, + }); +}); + +test('should compile AA in query B', () => { + const query_b = buildQuery(formDataMixedChartWithAA).queries[1]; + // time comparison + expect(query_b?.time_offsets).toEqual(['3 years ago']); + + // rolling total + expect( + // prettier-ignore + query_b + .post_processing + ?.find(operator => operator?.operation === 'rolling'), + ).toEqual({ + operation: 'rolling', + options: { + rolling_type: 'sum', + window: 1, + min_periods: 1, + columns: { + count: 'count', + 'count__3 years ago': 'count__3 years ago', + }, + }, + }); + + // resample + expect( + // prettier-ignore + query_b + .post_processing + ?.find(operator => operator?.operation === 'resample'), + ).toEqual({ + operation: 'resample', + options: { + method: 'asfreq', + rule: '1A', + fill_value: null, + }, + }); +}); diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/utils/formDataSuffix.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/utils/formDataSuffix.test.ts new file mode 100644 index 0000000000000..2e22583c76c7c --- /dev/null +++ b/superset-frontend/plugins/plugin-chart-echarts/test/utils/formDataSuffix.test.ts @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { + retainFormDataSuffix, + removeFormDataSuffix, +} from '../../src/utils/formDataSuffix'; + +const formData = { + datasource: 'dummy', + viz_type: 'table', + metrics: ['a', 'b'], + columns: ['foo', 'bar'], + limit: 100, + metrics_b: ['c', 'd'], + columns_b: ['hello', 'world'], + limit_b: 200, +}; + +test('should keep controls with suffix', () => { + expect(retainFormDataSuffix(formData, '_b')).toEqual({ + datasource: 'dummy', + viz_type: 'table', + metrics: ['c', 'd'], + columns: ['hello', 'world'], + limit: 200, + }); + // no side effect + expect(retainFormDataSuffix(formData, '_b')).not.toEqual(formData); +}); + +test('should remove controls with suffix', () => { + expect(removeFormDataSuffix(formData, '_b')).toEqual({ + datasource: 'dummy', + viz_type: 'table', + metrics: ['a', 'b'], + columns: ['foo', 'bar'], + limit: 100, + }); + // no side effect + expect(removeFormDataSuffix(formData, '_b')).not.toEqual(formData); +});