-
+
+
-
-
-
-
- No Data available
-
+
+
+ Loading tree data
+
+
+
+ class="euiFlexItem"
+ >
+
+
+
+ Loading may take over 30 seconds
+
+
+
+
-
- Failed to load databases
-
+
+
+
+ Status:
+ Scheduled
+
+
+
-
+
diff --git a/public/components/Main/main.test.tsx b/public/components/Main/main.test.tsx
index e583c2a..b0ab0dd 100644
--- a/public/components/Main/main.test.tsx
+++ b/public/components/Main/main.test.tsx
@@ -23,6 +23,46 @@ import { Main } from './main';
const setBreadcrumbsMock = jest.fn();
+jest.mock('../../dependencies/register_observability_dependencies', () => ({
+ getRenderAccelerationDetailsFlyout: jest.fn(() => jest.fn()),
+ getRenderCreateAccelerationFlyout: jest.fn(() => jest.fn()),
+ setRenderAccelerationDetailsFlyout: jest.fn(() => jest.fn()),
+ setRenderCreateAccelerationFlyout: jest.fn(() => jest.fn()),
+}));
+
+jest.mock('../../framework/catalog_cache_refs', () => ({
+ catalogCacheRefs: {
+ CatalogCacheManager: {
+ addOrUpdateAccelerationsByDataSource: () => ({
+ dbCache: { status: 'empty' },
+ }),
+ getOrCreateDataSource: () => ({
+ dsCache: { status: 'empty' },
+ }),
+ },
+ useLoadDatabasesToCache: () => ({
+ loadStatus: 'Scheduled',
+ startLoading: jest.fn(),
+ stopLoading: jest.fn(),
+ }),
+ useLoadTablesToCache: () => ({
+ loadStatus: 'Scheduled',
+ startLoading: jest.fn(),
+ stopLoading: jest.fn(),
+ }),
+ useLoadTableColumnsToCache: () => ({
+ loadStatus: 'Scheduled',
+ startLoading: jest.fn(),
+ stopLoading: jest.fn(),
+ }),
+ useLoadAccelerationsToCache: () => ({
+ loadStatus: 'Scheduled',
+ startLoading: jest.fn(),
+ stopLoading: jest.fn(),
+ }),
+ },
+}));
+
describe('
spec', () => {
it('renders the component', async () => {
const client = httpClientMock;
@@ -45,9 +85,9 @@ describe('
spec', () => {
const asyncTest = () => {
fireEvent.click(pplButton);
};
- waitFor(()=>{
+ waitFor(() => {
asyncTest();
- })
+ });
expect(document.body.children[0]).toMatchSnapshot();
});
it('renders the component and checks if Opensearch is selected', async () => {
@@ -68,7 +108,7 @@ describe('
spec', () => {
);
expect(getByText('OpenSearch')).toBeInTheDocument();
- fireEvent.click(getByText('OpenSearch'))
+ fireEvent.click(getByText('OpenSearch'));
await waitFor(() => {
expect(getByText('glue_1')).toBeInTheDocument();
});
@@ -81,9 +121,11 @@ describe('
spec', () => {
const { getByText } = await render(
);
- expect(getByText('OpenSearch')).toBeInTheDocument();
- fireEvent.click(getByText('OpenSearch'))
- fireEvent.click(getByText('glue_1'))
+ await waitFor(() => {
+ expect(getByText('OpenSearch')).toBeInTheDocument();
+ });
+ fireEvent.click(getByText('OpenSearch'));
+ fireEvent.click(getByText('glue_1'));
await waitFor(() => {
expect(getByText('Sample Query')).toBeInTheDocument();
});
@@ -100,7 +142,10 @@ describe('
spec', () => {
const { getByText } = await render(
);
- expect(getByText('.kibana_1')).toBeInTheDocument();
+
+ await waitFor(() => {
+ expect(getByText('.kibana_1')).toBeInTheDocument();
+ });
expect(document.body.children[0]).toMatchSnapshot();
});
@@ -217,4 +262,4 @@ describe('
spec', () => {
await asyncTest();
expect(document.body.children[0]).toMatchSnapshot();
});
-});
\ No newline at end of file
+});
diff --git a/public/components/Main/main.tsx b/public/components/Main/main.tsx
index 713fd54..74ca225 100644
--- a/public/components/Main/main.tsx
+++ b/public/components/Main/main.tsx
@@ -19,6 +19,7 @@ import {
EuiText,
} from '@elastic/eui';
import { IHttpResponse } from 'angular';
+import { createBrowserHistory } from 'history';
import _ from 'lodash';
import React from 'react';
import { ChromeBreadcrumb, CoreStart } from '../../../../../src/core/public';
@@ -39,7 +40,7 @@ import { QueryResults } from '../QueryResults/QueryResults';
import { CreateButton } from '../SQLPage/CreateButton';
import { DataSelect } from '../SQLPage/DataSelect';
import { SQLPage } from '../SQLPage/SQLPage';
-import { TableView } from '../SQLPage/table_view';
+import { CatalogTree } from '../SQLPage/sql_catalog_tree/catalog_tree';
interface ResponseData {
ok: boolean;
@@ -230,6 +231,7 @@ export function getQueryResultsForTable(
export class Main extends React.Component
{
httpClient: CoreStart['http'];
+ historyFromRedirection = createBrowserHistory();
constructor(props: MainProps) {
super(props);
@@ -759,6 +761,21 @@ export class Main extends React.Component {
});
}
+ checkHistoryState = () => {
+ if (!this.historyFromRedirection.location.state) return;
+
+ const { language, queryToRun }: any = this.historyFromRedirection.location.state;
+ if (language === 'sql') {
+ this.updateSQLQueries(queryToRun);
+
+ // Clear the state after use
+ this.historyFromRedirection.replace({
+ ...location,
+ state: null,
+ });
+ }
+ };
+
handleDataSelect = (selectedItems: EuiComboBoxOptionOption[]) => {
this.updateSQLQueries('');
this.updatePPLQueries('');
@@ -769,6 +786,7 @@ export class Main extends React.Component {
this.setState({
selectedDatasource: selectedItems,
});
+ this.checkHistoryState();
};
handleReloadTree = () => {
@@ -940,8 +958,7 @@ export class Main extends React.Component {
}}
>
- ({
+ getRenderAccelerationDetailsFlyout: jest.fn(() => jest.fn()),
+ getRenderCreateAccelerationFlyout: jest.fn(() => jest.fn()),
+ setRenderAccelerationDetailsFlyout: jest.fn(() => jest.fn()),
+ setRenderCreateAccelerationFlyout: jest.fn(() => jest.fn()),
+}));
+
describe(' {
@@ -55,9 +55,10 @@ export class SQLPage extends React.Component {
sqlQuery: this.props.sqlQuery,
translation: '',
isModalVisible: false,
- flyoutComponent: <>,
+ queryToRun: '',
};
}
+ renderCreateAccelerationFlyout = getRenderCreateAccelerationFlyout();
setIsModalVisible(visible: boolean): void {
this.setState({
@@ -65,21 +66,8 @@ export class SQLPage extends React.Component {
});
}
- resetFlyout = () => {
- this.setState({ flyoutComponent: <> });
- };
-
setAccelerationFlyout = () => {
- this.setState({
- flyoutComponent: (
- {
>
{
-
+
{
this.props.onClear();
}}
>
-
+
Clear
@@ -220,7 +212,9 @@ export class SQLPage extends React.Component {
+ this.renderCreateAccelerationFlyout(this.props.selectedDatasource[0].label)
+ }
isDisabled={this.props.asyncLoading}
>
Accelerate Table
@@ -230,8 +224,7 @@ export class SQLPage extends React.Component {
{modal}
- {this.state.flyoutComponent}
);
}
-}
\ No newline at end of file
+}
diff --git a/public/components/SQLPage/sql_catalog_tree/catalog_tree.tsx b/public/components/SQLPage/sql_catalog_tree/catalog_tree.tsx
new file mode 100644
index 0000000..b131ecf
--- /dev/null
+++ b/public/components/SQLPage/sql_catalog_tree/catalog_tree.tsx
@@ -0,0 +1,35 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { EuiComboBoxOptionOption } from '@elastic/eui';
+import React from 'react';
+import { OSTree } from './os_tree';
+import { S3Tree } from './s3_tree';
+
+interface CatalogTreeProps {
+ selectedItems: EuiComboBoxOptionOption[];
+ updateSQLQueries: (query: string) => void;
+ refreshTree: boolean;
+}
+
+export const CatalogTree = ({ selectedItems, updateSQLQueries, refreshTree }: CatalogTreeProps) => {
+ return (
+ <>
+ {selectedItems !== undefined && selectedItems[0].label === 'OpenSearch' ? (
+ ([]);
+ const [isTreeLoading, setIsTreeLoading] = useState({
+ status: false,
+ message: '',
+ });
+
+ const loadtree = async () => {
+ setTreeData([]);
+ setIsTreeLoading({
+ status: true,
+ message: 'Fetching associated objects ...',
+ });
+ const { treeContent, loadingStatus } = await getTreeContent(selectedItems);
+ setTreeData(treeContent);
+ setIsTreeLoading({ ...loadingStatus });
+ };
+
+ const treeLoadingStateRenderer = (
+
+
+
+ Loading tree data
+
+
+
+
+ Loading may take over 30 seconds
+
+
+
+
+ Status: {isTreeLoading.status}
+
+
+
+
+
+ );
+
+ const treeStateRenderer =
+ isTreeLoading.message === '' ? (
+
+
+
+ Failed to load the tree
+
+
+
+
+ {isTreeLoading.message}
+
+
+
+
+
+ );
+
+ const treeRenderer = (
+ <>
+ {isTreeLoading.status && isTreeLoading.message === ''
+ ? treeLoadingStateRenderer
+ : treeStateRenderer}
+
+ );
+
+ useEffect(() => {
+ loadtree();
+ }, [selectedItems, refreshTree]);
+
+ return {treeRenderer}
;
+};
diff --git a/public/components/SQLPage/sql_catalog_tree/os_tree_helpers.tsx b/public/components/SQLPage/sql_catalog_tree/os_tree_helpers.tsx
new file mode 100644
index 0000000..4073d56
--- /dev/null
+++ b/public/components/SQLPage/sql_catalog_tree/os_tree_helpers.tsx
@@ -0,0 +1,81 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { EuiComboBoxOptionOption, EuiIcon, EuiText, EuiToolTip } from '@elastic/eui';
+import { Node } from '@opensearch-project/oui/src/eui_components/tree_view/tree_view';
+import _ from 'lodash';
+import React from 'react';
+import {
+ FETCH_OPENSEARCH_INDICES_PATH,
+ LOAD_OPENSEARCH_INDICES_QUERY,
+} from '../../../../common/constants';
+import { coreRefs } from '../../../framework/core_refs';
+
+export const generateOpenSearchTree = (indices: string[]) => {
+ const openSearchIndicesTree = indices.map((indexName, idx) => ({
+ label: (
+
+
+ {_.truncate(indexName, { length: 50 })}
+ {' '}
+
+ ),
+ icon: ([]);
+ const [currentSelectedDatabase, setCurrentSelectedDatabase] = useState('');
+ const renderAccelerationDetailsFlyout = getRenderAccelerationDetailsFlyout();
+ const [isTreeLoading, setIsTreeLoading] = useState({
+ status: false,
+ message: '',
+ });
+ const [isObjectLoading, setIsObjectLoading] = useState({
+ tableStatus: false,
+ accelerationsStatus: false,
+ });
+ const {
+ loadStatus: loadDatabasesStatus,
+ startLoading: startDatabasesLoading,
+ stopLoading: stopDatabasesLoading,
+ } = catalogCacheRefs.useLoadDatabasesToCache();
+ const {
+ loadStatus: loadTablesStatus,
+ startLoading: startLoadingTables,
+ stopLoading: stopLoadingTables,
+ } = catalogCacheRefs.useLoadTablesToCache();
+ const {
+ loadStatus: loadAccelerationsStatus,
+ startLoading: startLoadingAccelerations,
+ stopLoading: stopLoadingAccelerations,
+ } = catalogCacheRefs.useLoadAccelerationsToCache();
+
+ const updateDatabaseState = (databaseName: string, isLoading: boolean, values?: TreeItem[]) => {
+ setTreeData(
+ produce((draft) => {
+ const databaseToUpdate = draft.find((database) => database.name === databaseName);
+ if (databaseToUpdate) {
+ databaseToUpdate.isExpanded = true;
+ databaseToUpdate.isLoading = isLoading;
+ if (values !== undefined) {
+ databaseToUpdate.values = databaseToUpdate.values
+ ? databaseToUpdate.values.concat(values)
+ : values;
+ }
+ }
+ })
+ );
+ };
+
+ const constructObjectTree = (
+ database: string,
+ tablesData: string[],
+ accelerationsData: any[]
+ ) => {
+ const tablesTreeItems = tablesData.map((table) => {
+ const indices = findSkippingAndCoveringIndexNames(accelerationsData, database, table);
+
+ const tableValues = indices.map((index) => {
+ return index === TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME
+ ? loadTreeItem(
+ [TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME],
+ TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME
+ )[0]
+ : loadTreeItem([index], TREE_ITEM_COVERING_INDEX_DEFAULT_NAME)[0];
+ });
+
+ const tableTreeItem = loadTreeItem(
+ [table],
+ TREE_ITEM_TABLE_NAME_DEFAULT_NAME,
+ tableValues
+ )[0];
+ return tableTreeItem;
+ });
+
+ const mvItems = findMaterializedViewsForDatabase(accelerationsData, database);
+ const mvTreeItems = loadTreeItem(mvItems, TREE_ITEM_MATERIALIZED_VIEW_DEFAULT_NAME);
+
+ updateDatabaseState(database, true, [...tablesTreeItems, ...mvTreeItems]);
+ };
+
+ const onClickDatabase = (database: TreeItem) => {
+ if (currentSelectedDatabase === '') {
+ setCurrentSelectedDatabase(database.name);
+ updateDatabaseState(database.name, true);
+ if (!isEitherObjectCacheEmpty(dataSource, database.name)) {
+ const tablesData = getTablesFromCache(dataSource, database.name);
+ const accelerationsData = getAccelerationsFromCache(dataSource);
+
+ constructObjectTree(database.name, tablesData, accelerationsData);
+ updateDatabaseState(database.name, false);
+ setCurrentSelectedDatabase('');
+ } else {
+ updateDatabaseState(database.name, true);
+ setIsObjectLoading({
+ tableStatus: true,
+ accelerationsStatus: true,
+ });
+ startLoadingTables(dataSource, database.name);
+ startLoadingAccelerations(dataSource);
+ }
+ } else {
+ setToast('Can only load one database at a time', 'warning');
+ }
+ };
+
+ const onClickAcceleration = (
+ databaseName: string,
+ tableName: string | undefined,
+ accelerationName: string
+ ) => {
+ const accelerationsData = getAccelerationsFromCache(dataSource);
+ const accelerationObject = findIndexObject(
+ accelerationsData,
+ databaseName,
+ tableName,
+ accelerationName
+ );
+ renderAccelerationDetailsFlyout(accelerationObject, dataSource);
+ };
+
+ const treeDataDatabases = treeData.map((database, index) => ({
+ label: createLabel(database, dataSource, database.name, index, updateSQLQueries),
+ icon: iconCreation(database),
+ id: 'element_' + index,
+ callback: () => {
+ if (database.values?.length === 0) {
+ onClickDatabase(database);
+ }
+ },
+ isSelectable: true,
+ isExpanded: database.isExpanded,
+ children: database.values?.map((table, idx) => ({
+ label: createLabel(table, dataSource, database.name, idx, updateSQLQueries),
+ id: `${database.name}_${table.name}`,
+ icon: iconCreation(table),
+ callback: () => {
+ if (table.type === TREE_ITEM_MATERIALIZED_VIEW_DEFAULT_NAME) {
+ onClickAcceleration(database.name, undefined, table.name);
+ }
+ },
+ isSelectable: true,
+ isExpanded: table.isExpanded,
+ children: table.values?.map((indexChild, idxValue) => ({
+ label: createLabel(indexChild, table.name, idxValue),
+ id: `${database.name}_${table.name}_${indexChild.name}`,
+ icon: iconCreation(indexChild),
+ callback: () => {
+ if (
+ indexChild.type === TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME ||
+ indexChild.type === TREE_ITEM_COVERING_INDEX_DEFAULT_NAME
+ ) {
+ onClickAcceleration(database.name, table.name, indexChild.name);
+ }
+ },
+ })),
+ })),
+ }));
+
+ const onRefreshTree = () => {
+ setIsTreeLoading({ status: true, message: '' });
+ startDatabasesLoading(dataSource);
+ };
+
+ const onLoadS3Tree = () => {
+ setIsTreeLoading({ status: true, message: '' });
+ const dsCache = catalogCacheRefs.CatalogCacheManager!.getOrCreateDataSource(dataSource);
+
+ if (dsCache.status === CachedDataSourceStatus.Updated && dsCache.databases.length > 0) {
+ const databases = dsCache.databases.map((db) => db.name);
+ setTreeData(loadTreeItem(databases, TREE_ITEM_DATABASE_NAME_DEFAULT_NAME));
+ setIsTreeLoading({ status: false, message: '' });
+ } else if (dsCache.status === CachedDataSourceStatus.Empty) {
+ startDatabasesLoading(dataSource);
+ }
+ };
+
+ useEffect(() => {
+ const status = loadDatabasesStatus.toLowerCase();
+ if (status === AsyncQueryStatus.Success) {
+ setIsTreeLoading({ status: false, message: '' });
+ } else if (status === AsyncQueryStatus.Failed || status === AsyncQueryStatus.Cancelled) {
+ setIsTreeLoading({ status: false, message: 'Failed to load databases' });
+ }
+ }, [loadDatabasesStatus]);
+
+ useEffect(() => {
+ const status = loadTablesStatus.toLowerCase();
+ if (status === AsyncQueryStatus.Success) {
+ setIsObjectLoading({ ...isObjectLoading, tableStatus: false });
+ } else if (status === AsyncQueryStatus.Failed || status === AsyncQueryStatus.Cancelled) {
+ setIsObjectLoading({ ...isObjectLoading, tableStatus: false });
+ }
+ }, [loadTablesStatus]);
+
+ useEffect(() => {
+ const status = loadAccelerationsStatus.toLowerCase();
+ if (status === AsyncQueryStatus.Success) {
+ setIsObjectLoading({ ...isObjectLoading, accelerationsStatus: false });
+ } else if (status === AsyncQueryStatus.Failed || status === AsyncQueryStatus.Cancelled) {
+ setIsObjectLoading({ ...isObjectLoading, accelerationsStatus: false });
+ }
+ }, [loadAccelerationsStatus]);
+
+ useEffect(() => {
+ onLoadS3Tree();
+ }, [dataSource]);
+
+ useEffect(() => {
+ setIsFirstRender(false);
+ }, []);
+
+ useEffect(() => {
+ if (isFirstRender) {
+ return;
+ }
+ // This will only execute on changes to refreshTree after the initial render
+ onRefreshTree();
+ }, [refreshTree]);
+
+ useEffect(() => {
+ if (
+ !(isObjectLoading.accelerationsStatus || isObjectLoading.tableStatus) &&
+ currentSelectedDatabase !== ''
+ ) {
+ const tablesData = getTablesFromCache(dataSource, currentSelectedDatabase);
+ const accelerationsData = getAccelerationsFromCache(dataSource);
+
+ constructObjectTree(currentSelectedDatabase, tablesData, accelerationsData);
+ updateDatabaseState(currentSelectedDatabase, false);
+ setCurrentSelectedDatabase('');
+ }
+ }, [isObjectLoading]);
+
+ useEffect(() => {
+ return () => {
+ stopDatabasesLoading();
+ stopLoadingTables();
+ stopLoadingAccelerations();
+ };
+ }, []);
+
+ const treeLoadingStateRenderer = (
+
+
+
+ Loading tree data
+
+
+
+
+ Loading may take over 30 seconds
+
+
+
+
+ Status: {loadDatabasesStatus}
+
+
+
+
+
+ );
+
+ const treeStateRenderer =
+ isTreeLoading.message === '' ? (
+
+
+
+ Failed to load the tree
+
+
+
+
+ {isTreeLoading.message}
+
+
+
+
+
+ );
+
+ const treeRenderer = (
+ <>
+ {isTreeLoading.status && isTreeLoading.message === ''
+ ? treeLoadingStateRenderer
+ : treeStateRenderer}
+
+ );
+ return {treeRenderer}
;
+};
diff --git a/public/components/SQLPage/sql_catalog_tree/s3_tree_helpers.tsx b/public/components/SQLPage/sql_catalog_tree/s3_tree_helpers.tsx
new file mode 100644
index 0000000..479b201
--- /dev/null
+++ b/public/components/SQLPage/sql_catalog_tree/s3_tree_helpers.tsx
@@ -0,0 +1,180 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+ EuiFlexGroup,
+ EuiFlexItem,
+ EuiIcon,
+ EuiLoadingSpinner,
+ EuiNotificationBadge,
+ EuiText,
+ EuiToolTip,
+} from '@elastic/eui';
+import _ from 'lodash';
+import React from 'react';
+import {
+ TREE_ITEM_COVERING_INDEX_DEFAULT_NAME,
+ TREE_ITEM_DATABASE_NAME_DEFAULT_NAME,
+ TREE_ITEM_MATERIALIZED_VIEW_DEFAULT_NAME,
+ TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME,
+ TREE_ITEM_TABLE_NAME_DEFAULT_NAME,
+} from '../../../../common/constants';
+import { CachedDataSourceStatus, TreeItem, TreeItemType } from '../../../../common/types';
+import { catalogCacheRefs } from '../../../framework/catalog_cache_refs';
+
+export const handleQuery = (
+ e: React.MouseEvent,
+ dataSource: string,
+ database: string,
+ tableName: string,
+ updateSQLQueries: (query: string) => void
+) => {
+ e.stopPropagation();
+ updateSQLQueries(`select * from \`${dataSource}\`.\`${database}\`.\`${tableName}\` limit 10`);
+};
+
+export const createLabel = (
+ node: TreeItem,
+ dataSource: string,
+ database: string,
+ index: number,
+ updateSQLQueries: (query: string) => void
+) => {
+ return (
+
+
+
+
+
+ {_.truncate(node.name, { length: 50 })}{' '}
+ {node.isLoading && handleQuery(e, dataSource, database, node.name, updateSQLQueries)}
+ />
+ )}
+
+
+
+
+
+ );
+};
+
+export const iconCreation = (node: TreeItem) => {
+ switch (node.type) {
+ case TREE_ITEM_MATERIALIZED_VIEW_DEFAULT_NAME:
+ return (
+
+ MV
+
+ );
+ case TREE_ITEM_TABLE_NAME_DEFAULT_NAME:
+ return