-
Notifications
You must be signed in to change notification settings - Fork 12.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Transformations: Adding group by and aggregate on multiple fields tra…
…nsformation * Adding Occurences transformer * Adding test for Occurences Transformer * Cleanup. Adding a test. * Adding doc * Modifying UI to support custom calculations options * Implementing data transformation * Finalizing calculations implementation * Cleanup * Using Fields instead of arrays in data grouping * Renaming transformation to GroupBy * Adding some doc * Apply suggestions (solving TS typing errors) Co-authored-by: Marcus Andersson <[email protected]> * Tweaking UI * Preventing of selecting twice the same field name. * Removing console print. No calculations by default. * Forgot to add the current value to the GroupBy selector * Solving some typing issues and prettyfier errors * Cleanup * Updating test * Ensure proper copy of options (solves some issues) * Check if the fields exist in the data before processing * Adding missing import in test file * If group by field not specified, return all data untouched. * Adding another missing import in test * Minor updates * Implementing GroupBy multiple fields + Improve field typing * Removing console prints * Allowing the exact number of fields to be added as aggregation * Centering remove button icon * Cleanup * Correcting TS error * Chaging transformer options structure * Sorting so GroupBy fields appear on top * Cleanup * Simplifying some operations. Adding curly brackets. * Changing some labels on the UI * Updating test * Cleanup * Updating doc * Fixed field list. Storing options as Record instead of Array. * Update test * Cleaned up the group by editor UI code. * changed the transform to a table layout instead of a flexbox layout. * cleaned up group by transformer. * removed unused imports. * Added some more tests. * Added one more test and cleaned up code. * fixed failing test. * Fixed so we we have the proper casing on naming. * fixed so we don't wrap on the first row. Co-authored-by: Marcus Andersson <[email protected]> Co-authored-by: Torkel Ödegaard <[email protected]> Co-authored-by: Marcus Andersson <[email protected]>
- Loading branch information
4 people
authored
Aug 31, 2020
1 parent
fe6d399
commit 7db42f0
Showing
8 changed files
with
657 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
257 changes: 257 additions & 0 deletions
257
packages/grafana-data/src/transformations/transformers/groupBy.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,257 @@ | ||
| import { toDataFrame } from '../../dataframe/processDataFrame'; | ||
| import { groupByTransformer, GroupByTransformerOptions, GroupByOperationID } from './groupBy'; | ||
| import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; | ||
| import { transformDataFrame } from '../transformDataFrame'; | ||
| import { Field, FieldType } from '../../types'; | ||
| import { DataTransformerID } from './ids'; | ||
| import { ArrayVector } from '../../vector'; | ||
| import { ReducerID } from '../fieldReducer'; | ||
| import { DataTransformerConfig } from '@grafana/data'; | ||
|
|
||
| describe('GroupBy transformer', () => { | ||
| beforeAll(() => { | ||
| mockTransformationsRegistry([groupByTransformer]); | ||
| }); | ||
|
|
||
| it('should not apply transformation if config is missing group by fields', () => { | ||
| const testSeries = toDataFrame({ | ||
| name: 'A', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [1, 2, 2, 3, 3, 3] }, | ||
| ], | ||
| }); | ||
|
|
||
| const cfg: DataTransformerConfig<GroupByTransformerOptions> = { | ||
| id: DataTransformerID.groupBy, | ||
| options: { | ||
| fields: { | ||
| message: { | ||
| operation: GroupByOperationID.aggregate, | ||
| aggregations: [ReducerID.count], | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = transformDataFrame([cfg], [testSeries]); | ||
| expect(result[0]).toBe(testSeries); | ||
| }); | ||
|
|
||
| it('should group values by message', () => { | ||
| const testSeries = toDataFrame({ | ||
| name: 'A', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [1, 2, 2, 3, 3, 3] }, | ||
| ], | ||
| }); | ||
|
|
||
| const cfg: DataTransformerConfig<GroupByTransformerOptions> = { | ||
| id: DataTransformerID.groupBy, | ||
| options: { | ||
| fields: { | ||
| message: { | ||
| operation: GroupByOperationID.groupBy, | ||
| aggregations: [], | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = transformDataFrame([cfg], [testSeries]); | ||
|
|
||
| const expected: Field[] = [ | ||
| { | ||
| name: 'message', | ||
| type: FieldType.string, | ||
| values: new ArrayVector(['one', 'two', 'three']), | ||
| config: {}, | ||
| }, | ||
| ]; | ||
|
|
||
| expect(result[0].fields).toEqual(expected); | ||
| }); | ||
|
|
||
| it('should group values by message and summarize values', () => { | ||
| const testSeries = toDataFrame({ | ||
| name: 'A', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [1, 2, 2, 3, 3, 3] }, | ||
| ], | ||
| }); | ||
|
|
||
| const cfg: DataTransformerConfig<GroupByTransformerOptions> = { | ||
| id: DataTransformerID.groupBy, | ||
| options: { | ||
| fields: { | ||
| message: { | ||
| operation: GroupByOperationID.groupBy, | ||
| aggregations: [], | ||
| }, | ||
| values: { | ||
| operation: GroupByOperationID.aggregate, | ||
| aggregations: [ReducerID.sum], | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = transformDataFrame([cfg], [testSeries]); | ||
|
|
||
| const expected: Field[] = [ | ||
| { | ||
| name: 'message', | ||
| type: FieldType.string, | ||
| values: new ArrayVector(['one', 'two', 'three']), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'values (sum)', | ||
| type: FieldType.number, | ||
| values: new ArrayVector([1, 4, 9]), | ||
| config: {}, | ||
| }, | ||
| ]; | ||
|
|
||
| expect(result[0].fields).toEqual(expected); | ||
| }); | ||
|
|
||
| it('should group by and compute a few calculations for each group of values', () => { | ||
| const testSeries = toDataFrame({ | ||
| name: 'A', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [1, 2, 2, 3, 3, 3] }, | ||
| ], | ||
| }); | ||
|
|
||
| const cfg: DataTransformerConfig<GroupByTransformerOptions> = { | ||
| id: DataTransformerID.groupBy, | ||
| options: { | ||
| fields: { | ||
| message: { | ||
| operation: GroupByOperationID.groupBy, | ||
| aggregations: [], | ||
| }, | ||
| time: { | ||
| operation: GroupByOperationID.aggregate, | ||
| aggregations: [ReducerID.count, ReducerID.last], | ||
| }, | ||
| values: { | ||
| operation: GroupByOperationID.aggregate, | ||
| aggregations: [ReducerID.sum], | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = transformDataFrame([cfg], [testSeries]); | ||
|
|
||
| const expected: Field[] = [ | ||
| { | ||
| name: 'message', | ||
| type: FieldType.string, | ||
| values: new ArrayVector(['one', 'two', 'three']), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'time (count)', | ||
| type: FieldType.number, | ||
| values: new ArrayVector([1, 2, 3]), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'time (last)', | ||
| type: FieldType.time, | ||
| values: new ArrayVector([3000, 5000, 8000]), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'values (sum)', | ||
| type: FieldType.number, | ||
| values: new ArrayVector([1, 4, 9]), | ||
| config: {}, | ||
| }, | ||
| ]; | ||
|
|
||
| expect(result[0].fields).toEqual(expected); | ||
| }); | ||
|
|
||
| it('should group values in data frames induvidually', () => { | ||
| const testSeries = [ | ||
| toDataFrame({ | ||
| name: 'A', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [1, 2, 2, 3, 3, 3] }, | ||
| ], | ||
| }), | ||
| toDataFrame({ | ||
| name: 'B', | ||
| fields: [ | ||
| { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000, 7000, 8000] }, | ||
| { name: 'message', type: FieldType.string, values: ['one', 'two', 'two', 'three', 'three', 'three'] }, | ||
| { name: 'values', type: FieldType.string, values: [0, 2, 5, 3, 3, 2] }, | ||
| ], | ||
| }), | ||
| ]; | ||
|
|
||
| const cfg: DataTransformerConfig<GroupByTransformerOptions> = { | ||
| id: DataTransformerID.groupBy, | ||
| options: { | ||
| fields: { | ||
| message: { | ||
| operation: GroupByOperationID.groupBy, | ||
| aggregations: [], | ||
| }, | ||
| values: { | ||
| operation: GroupByOperationID.aggregate, | ||
| aggregations: [ReducerID.sum], | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const result = transformDataFrame([cfg], testSeries); | ||
|
|
||
| const expectedA: Field[] = [ | ||
| { | ||
| name: 'message', | ||
| type: FieldType.string, | ||
| values: new ArrayVector(['one', 'two', 'three']), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'values (sum)', | ||
| type: FieldType.number, | ||
| values: new ArrayVector([1, 4, 9]), | ||
| config: {}, | ||
| }, | ||
| ]; | ||
|
|
||
| const expectedB: Field[] = [ | ||
| { | ||
| name: 'message', | ||
| type: FieldType.string, | ||
| values: new ArrayVector(['one', 'two', 'three']), | ||
| config: {}, | ||
| }, | ||
| { | ||
| name: 'values (sum)', | ||
| type: FieldType.number, | ||
| values: new ArrayVector([0, 7, 8]), | ||
| config: {}, | ||
| }, | ||
| ]; | ||
|
|
||
| expect(result[0].fields).toEqual(expectedA); | ||
| expect(result[1].fields).toEqual(expectedB); | ||
| }); | ||
| }); |
Oops, something went wrong.