diff --git a/common/constants/index.ts b/common/constants/index.ts index 5dbb6657..3ca8dd41 100644 --- a/common/constants/index.ts +++ b/common/constants/index.ts @@ -31,3 +31,17 @@ export const ACCELERATION_TIME_INTERVAL = [ ]; export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; + +export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. +##### Skipping Index +- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. +##### Covering Index +- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. +##### Materialized View Index +- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. +##### Note: +- All user given index names must be in lowercase letters. Cannot begin with underscores or hyphens. Spaces, commas, and characters :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `; diff --git a/common/types/index.ts b/common/types/index.ts index 563bb438..e210d8ea 100644 --- a/common/types/index.ts +++ b/common/types/index.ts @@ -3,11 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -export type AggregationNameType = 'skipping' | 'covering' | 'materialized'; +export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min'; export interface MaterializedViewColumn { id: string; - functionName: AggregationNameType; + functionName: AggregationFunctionType; functionParam: string; fieldAlias?: string; } @@ -32,16 +32,26 @@ export interface RefreshIntervalType { export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; +export interface GroupByTumbleType { + timeField: string; + tumbleWindow: number; + tumbleInterval: string; +} + +export interface materializedViewQueryType { + ColumnsValues: MaterializedViewColumn[]; + GroupByTumbleValue: GroupByTumbleType; +} + export interface CreateAccelerationForm { dataSource: string; database: string; dataTable: string; dataTableFields: DataTableFieldsType[]; accelerationIndexType: AccelerationIndexType; - queryBuilderType: 'visual' | 'code'; skippingIndexQueryData: SkippingIndexRowType[]; coveringIndexQueryData: string; - materializedViewQueryData: string; + materializedViewQueryData: materializedViewQueryType; accelerationIndexName: string; primaryShardsCount: number; replicaShardsCount: number; diff --git a/public/components/acceleration/create/create_acceleration.tsx b/public/components/acceleration/create/create_acceleration.tsx index 7a8bd370..94c1c02b 100644 --- a/public/components/acceleration/create/create_acceleration.tsx +++ b/public/components/acceleration/create/create_acceleration.tsx @@ -18,7 +18,7 @@ import React, { useState } from 'react'; import { CreateAccelerationHeader } from './create_acceleration_header'; import { CautionBannerCallout } from './caution_banner_callout'; import { AccelerationDataSourceSelector } from '../selectors/source_selector'; -import { CreateAccelerationForm } from '../../../../common/types/'; +import { CreateAccelerationForm, materializedViewQueryType } from '../../../../common/types/'; import { QueryVisualEditor } from '../visual_editors/query_visual_editor'; import { accelerationQueryBuilder } from '../visual_editors/query_builder'; import { IndexSettingOptions } from '../selectors/index_setting_options'; @@ -41,10 +41,9 @@ export const CreateAcceleration = ({ database: '', dataTableFields: [], accelerationIndexType: 'skipping', - queryBuilderType: 'visual', skippingIndexQueryData: [], coveringIndexQueryData: '', - materializedViewQueryData: '', + materializedViewQueryData: {} as materializedViewQueryType, accelerationIndexName: '', primaryShardsCount: 5, replicaShardsCount: 1, diff --git a/public/components/acceleration/selectors/define_index_options.tsx b/public/components/acceleration/selectors/define_index_options.tsx index 4ac7399c..dc5d56cd 100644 --- a/public/components/acceleration/selectors/define_index_options.tsx +++ b/public/components/acceleration/selectors/define_index_options.tsx @@ -4,9 +4,27 @@ */ import React, { ChangeEvent, useEffect, useState } from 'react'; +import { + EuiButton, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiIconTip, + EuiLink, + EuiMarkdownFormat, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSpacer, + EuiText, + EuiHorizontalRule, +} from '@elastic/eui'; import { CreateAccelerationForm } from '../../../../common/types'; -import { EuiFieldText, EuiFormRow, EuiSpacer, EuiText, EuiToolTip } from '@elastic/eui'; import { validateIndexName } from '../create/utils'; +import { ACCELERATION_INDEX_NAME_INFO } from '../../../../common/constants'; interface DefineIndexOptionsProps { accelerationFormData: CreateAccelerationForm; @@ -18,6 +36,34 @@ export const DefineIndexOptions = ({ setAccelerationFormData, }: DefineIndexOptionsProps) => { const [indexName, setIndexName] = useState(''); + const [isModalVisible, setIsModalVisible] = useState(false); + + let modal; + + if (isModalVisible) { + modal = ( + setIsModalVisible(false)}> + + +

Acceleration index naming

+
+
+ + + + + {ACCELERATION_INDEX_NAME_INFO} + + + + + setIsModalVisible(false)} fill> + Close + + +
+ ); + } const onChangeIndexName = (e: ChangeEvent) => { setAccelerationFormData({ ...accelerationFormData, accelerationIndexName: e.target.value }); @@ -30,6 +76,28 @@ export const DefineIndexOptions = ({ : setIndexName(''); }, [accelerationFormData.accelerationIndexType]); + const getPreprend = () => { + const dataSource = + accelerationFormData.dataSource !== '' + ? accelerationFormData.dataSource + : '{Datasource Name}'; + const database = + accelerationFormData.database !== '' ? accelerationFormData.database : '{Database Name}'; + const dataTable = + accelerationFormData.dataTable !== '' ? accelerationFormData.dataTable : '{Table Name}'; + const prependValue = `flint_${dataSource}_${database}_${dataTable}_`; + return [ + prependValue, + , + ]; + }; + + const getAppend = () => { + const appendValue = + accelerationFormData.accelerationIndexType === 'materialized' ? '' : '_index'; + return appendValue; + }; + return ( <> @@ -40,27 +108,24 @@ export const DefineIndexOptions = ({ label="Index name" helpText='Must be in lowercase letters. Cannot begin with underscores or hyphens. Spaces, commas, and characters :, ", *, +, /, \, |, ?, #, >, or < are not allowed. Prefix and suffix are added to the name of generated OpenSearch index.' + labelAppend={ + + setIsModalVisible(true)}>Help + + } > - _index` and `flint_datasource_datatable_`' - } - > - - + + {modal} ); }; diff --git a/public/components/acceleration/selectors/index_setting_options.tsx b/public/components/acceleration/selectors/index_setting_options.tsx index acf1b135..d5ef266d 100644 --- a/public/components/acceleration/selectors/index_setting_options.tsx +++ b/public/components/acceleration/selectors/index_setting_options.tsx @@ -166,25 +166,26 @@ export const IndexSettingOptions = ({ /> )} - - - - + helpText="The HDFS compatible file system location path for incremental refresh job checkpoint. Applicable when auto refresh is enabled." + > + + + )} ); }; diff --git a/public/components/acceleration/visual_editors/covering_index/covering_index_builder.tsx b/public/components/acceleration/visual_editors/covering_index/covering_index_builder.tsx index e9b81235..6ec753f0 100644 --- a/public/components/acceleration/visual_editors/covering_index/covering_index_builder.tsx +++ b/public/components/acceleration/visual_editors/covering_index/covering_index_builder.tsx @@ -12,6 +12,7 @@ import { EuiText, EuiFlexItem, EuiFlexGroup, + EuiComboBoxOptionOption, } from '@elastic/eui'; import React, { useEffect, useState } from 'react'; import { CreateAccelerationForm } from '../../../../../common/types'; @@ -31,7 +32,7 @@ export const CoveringIndexBuilder = ({ const [columnsValue, setColumnsValue] = useState(''); const [selectedOptions, setSelected] = useState([]); - const onChange = (selectedOptions) => { + const onChange = (selectedOptions: EuiComboBoxOptionOption[]) => { setSelected(selectedOptions); }; @@ -51,6 +52,7 @@ export const CoveringIndexBuilder = ({ ) + ')'; } + setAccelerationFormData({ ...accelerationFormData, coveringIndexQueryData: expresseionValue }); setColumnsValue(expresseionValue); }, [selectedOptions]); diff --git a/public/components/acceleration/visual_editors/materialized_view/add_column_popover.tsx b/public/components/acceleration/visual_editors/materialized_view/add_column_popover.tsx index 1328d486..5dee9114 100644 --- a/public/components/acceleration/visual_editors/materialized_view/add_column_popover.tsx +++ b/public/components/acceleration/visual_editors/materialized_view/add_column_popover.tsx @@ -20,7 +20,7 @@ import { } from '@elastic/eui'; import { - AggregationNameType, + AggregationFunctionType, CreateAccelerationForm, MaterializedViewColumn, } from '../../../../../common/types'; @@ -55,6 +55,7 @@ export const AddColumnPopOver = ({ setSelectedField([{ label: defaultFieldName }]); } }; + const resetValues = () => { setSelectedFunction([ACCELERATION_AGGREGRATION_FUNCTIONS[0]]); resetSelectedField(); @@ -73,14 +74,6 @@ export const AddColumnPopOver = ({ resetSelectedField(); }, []); - const loadAggregationFields = () => { - let aggFields = _.map(accelerationFormData.dataTableFields, (x) => { - return { label: x.fieldName }; - }); - if (selectedField.length > 0 && selectedField[0].label === 'count') - aggFields = [{ label: '*' }, ...aggFields]; - return aggFields; - }; return ( { + return { label: x.fieldName }; + }), + ]} selectedOptions={selectedField} onChange={setSelectedField} /> @@ -139,7 +137,7 @@ export const AddColumnPopOver = ({ ...columnExpressionValues, { id: newId, - functionName: selectedFunction[0].label as AggregationNameType, + functionName: selectedFunction[0].label as AggregationFunctionType, functionParam: selectedField[0].label, fieldAlias: selectedAlias, }, diff --git a/public/components/acceleration/visual_editors/materialized_view/column_expression.tsx b/public/components/acceleration/visual_editors/materialized_view/column_expression.tsx index e1bdd729..0615b360 100644 --- a/public/components/acceleration/visual_editors/materialized_view/column_expression.tsx +++ b/public/components/acceleration/visual_editors/materialized_view/column_expression.tsx @@ -5,7 +5,11 @@ import React, { useEffect, useState } from 'react'; import { ACCELERATION_AGGREGRATION_FUNCTIONS } from '../../../../../common/constants'; -import { MaterializedViewColumn, CreateAccelerationForm } from '../../../../../common/types'; +import { + MaterializedViewColumn, + CreateAccelerationForm, + AggregationFunctionType, +} from '../../../../../common/types'; import _ from 'lodash'; import { EuiButtonIcon, @@ -51,12 +55,7 @@ export const ColumnExpression = ({ button={ { setIsAliasPopOverOpen(false); @@ -79,14 +78,13 @@ export const ColumnExpression = ({ selectedOptions={[ { label: currentColumnExpressionValue.functionName, - value: currentColumnExpressionValue.functionName, }, ]} onChange={(functionOption) => updateColumnExpressionValue( { ...currentColumnExpressionValue, - functionName: functionOption[0].value, + functionName: functionOption[0].label as AggregationFunctionType, }, index ) @@ -98,18 +96,23 @@ export const ColumnExpression = ({ { - return { label: x.fieldName, value: x.fieldName }; - })} + options={[ + { + label: '*', + disabled: currentColumnExpressionValue.functionName !== 'count', + }, + ..._.map(accelerationFormData.dataTableFields, (x) => { + return { label: x.fieldName }; + }), + ]} selectedOptions={[ { label: currentColumnExpressionValue.functionParam, - value: currentColumnExpressionValue.functionParam, }, ]} onChange={(fieldOption) => updateColumnExpressionValue( - { ...currentColumnExpressionValue, functionParam: fieldOption[0].value }, + { ...currentColumnExpressionValue, functionParam: fieldOption[0].label }, index ) } @@ -160,14 +163,12 @@ export const ColumnExpression = ({ { - console.log('before', columnExpressionValues); setColumnExpressionValues([ ..._.filter( columnExpressionValues, (o) => o.id !== currentColumnExpressionValue.id ), ]); - console.log('after', columnExpressionValues); }} iconType="trash" aria-label="delete-column-expression" diff --git a/public/components/acceleration/visual_editors/materialized_view/group_by_tumble_expression.tsx b/public/components/acceleration/visual_editors/materialized_view/group_by_tumble_expression.tsx index 56dc3297..98daa9c6 100644 --- a/public/components/acceleration/visual_editors/materialized_view/group_by_tumble_expression.tsx +++ b/public/components/acceleration/visual_editors/materialized_view/group_by_tumble_expression.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import { EuiComboBox, EuiExpression, @@ -16,16 +16,10 @@ import { EuiComboBoxOptionOption, } from '@elastic/eui'; import { ACCELERATION_TIME_INTERVAL } from '../../../../../common/constants'; -import { CreateAccelerationForm } from '../../../../../common/types'; +import { CreateAccelerationForm, GroupByTumbleType } from '../../../../../common/types'; import _ from 'lodash'; import { isTimePlural } from '../../create/utils'; -interface GroupByTumbleValues { - timeField: string; - tumbleWindow: number; - tumbleInterval: string; -} - interface GroupByTumbleExpressionProps { accelerationFormData: CreateAccelerationForm; setAccelerationFormData: React.Dispatch>; @@ -36,7 +30,7 @@ export const GroupByTumbleExpression = ({ setAccelerationFormData, }: GroupByTumbleExpressionProps) => { const [IsGroupPopOverOpen, setIsGroupPopOverOpen] = useState(false); - const [groupbyValues, setGroupByValues] = useState({ + const [groupbyValues, setGroupByValues] = useState({ timeField: '', tumbleWindow: 1, tumbleInterval: ACCELERATION_TIME_INTERVAL[0].text, @@ -55,6 +49,16 @@ export const GroupByTumbleExpression = ({ setGroupByValues({ ...groupbyValues, timeField: selectedOptions[0].label }); }; + useEffect(() => { + setAccelerationFormData({ + ...accelerationFormData, + materializedViewQueryData: { + ...accelerationFormData.materializedViewQueryData, + GroupByTumbleValue: groupbyValues, + }, + }); + }, [groupbyValues]); + return ( { + setAccelerationFormData({ + ...accelerationFormData, + materializedViewQueryData: { + ...accelerationFormData.materializedViewQueryData, + ColumnsValues: columnExpressionValues, + }, + }); + }, [columnExpressionValues]); + return ( <> diff --git a/public/components/acceleration/visual_editors/query_builder.tsx b/public/components/acceleration/visual_editors/query_builder.tsx index a366a5f9..101f3ba8 100644 --- a/public/components/acceleration/visual_editors/query_builder.tsx +++ b/public/components/acceleration/visual_editors/query_builder.tsx @@ -5,19 +5,45 @@ import _ from 'lodash'; import { CreateAccelerationForm, SkippingIndexRowType } from '../../../../common/types'; +import { isTimePlural } from '../create/utils'; -const buildSkippingIndexColumns = (skippingIndexQueryData: SkippingIndexRowType[]) => { - return _.reduce( - skippingIndexQueryData, - function (columns, n, index) { - const columnValue = columns + ` ${n.fieldName} ${n.accelerationMethod}`; - if (index !== skippingIndexQueryData.length - 1) return `${columnValue}, \n`; - else return `${columnValue} \n`; - }, - '' +/* Add index options to query */ +const buildIndexOptions = (accelerationformData: CreateAccelerationForm) => { + const indexOptions: string[] = []; + + // Add index settings option + indexOptions.push( + `index_settings = '{"number_of_shards":${accelerationformData.primaryShardsCount},"number_of_replicas":${accelerationformData.replicaShardsCount}}'` ); + + // Add auto refresh option + indexOptions.push(`auto_refresh = ${accelerationformData.refreshType === 'auto'}`); + + // Add refresh interval option + if (accelerationformData.refreshType === 'interval') { + const { refreshWindow, refreshInterval } = accelerationformData.refreshIntervalOptions; + indexOptions.push( + `refresh_interval = '${refreshWindow} ${refreshInterval}${isTimePlural(refreshWindow)}'` + ); + } + + // Add checkpoint location option + if (accelerationformData.checkpointLocation) { + indexOptions.push(`checkpoint_location = '${accelerationformData.checkpointLocation}'`); + } + + // Combine all options with commas and return as a single string + return `WITH (\n${indexOptions.join(',\n')}\n)`; +}; + +/* Add skipping index columns to query */ +const buildSkippingIndexColumns = (skippingIndexQueryData: SkippingIndexRowType[]) => { + return skippingIndexQueryData + .map((n) => ` ${n.fieldName} ${n.accelerationMethod}`) + .join(', \n'); }; +/* Builds create skipping index query */ const skippingIndexQueryBuilder = (accelerationformData: CreateAccelerationForm) => { /* * Skipping Index Example @@ -27,14 +53,22 @@ const skippingIndexQueryBuilder = (accelerationformData: CreateAccelerationForm) * field1 VALUE_SET, * field2 PARTITION, * field3 MIN_MAX, + * ) WITH ( + * auto_refresh = true, + * refresh_interval = '1 minute', + * checkpoint_location = 's3://test/', + * index_settings = '{"number_of_shards":9,"number_of_replicas":2}' * ) */ - console.log('index builder started'); - let codeQuery = 'CREATE SKIPPING INDEX ON ' + accelerationformData.dataTable; - codeQuery = codeQuery + '\n FOR COLUMNS ( \n'; - codeQuery = codeQuery + buildSkippingIndexColumns(accelerationformData.skippingIndexQueryData); - codeQuery = codeQuery + ')'; - console.log('index builder finished', codeQuery); + const { dataSource, database, dataTable, skippingIndexQueryData } = accelerationformData; + + const codeQuery = `CREATE SKIPPING INDEX +[IF NOT EXISTS] +ON ${dataSource}.${database}.${dataTable} +FOR COLUMNS ( + ${buildSkippingIndexColumns(skippingIndexQueryData)} + ) ${buildIndexOptions(accelerationformData)}`; + return codeQuery; }; @@ -47,7 +81,6 @@ const materializedQueryViewBuilder = (accelerationformData: CreateAccelerationFo }; export const accelerationQueryBuilder = (accelerationformData: CreateAccelerationForm) => { - console.log('accelerationQueryBuilder started', accelerationformData); switch (accelerationformData.accelerationIndexType) { case 'skipping': { return skippingIndexQueryBuilder(accelerationformData);