diff --git a/CHANGELOG.md b/CHANGELOG.md index 622b3580c2d4..1a7cae599d09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Use mirrors to download Node.js binaries to escape sporadic 404 errors ([#3619](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3619)) - [Multiple DataSource] Refactor dev tool console to use opensearch-js client to send requests ([#3544](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3544)) - [Data] Add geo shape filter field ([#3605](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3605)) +- [Multiple DataSource] Allow create and distinguish index pattern with same name but from different datasources ([#3571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3571)) ### 🐛 Bug Fixes diff --git a/src/plugins/data/common/index_patterns/fields/utils.test.ts b/src/plugins/data/common/index_patterns/fields/utils.test.ts new file mode 100644 index 000000000000..b23f9ebaa2e2 --- /dev/null +++ b/src/plugins/data/common/index_patterns/fields/utils.test.ts @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isFilterable } from '..'; +import { IFieldType } from '.'; + +const mockField = { + name: 'foo', + scripted: false, + searchable: true, + type: 'string', +} as IFieldType; + +describe('isFilterable', () => { + describe('types', () => { + it('should return true for filterable types', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type })).toBe(true); + }); + }); + + it('should return false for filterable types if the field is not searchable', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type, searchable: false })).toBe(false); + }); + }); + + it('should return false for un-filterable types', () => { + ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach( + (type) => { + expect(isFilterable({ ...mockField, type })).toBe(false); + } + ); + }); + }); + + it('should return true for scripted fields', () => { + expect(isFilterable({ ...mockField, scripted: true, searchable: false })).toBe(true); + }); + + it('should return true for the _id field', () => { + expect(isFilterable({ ...mockField, name: '_id' })).toBe(true); + }); +}); diff --git a/src/plugins/data/common/index_patterns/index.ts b/src/plugins/data/common/index_patterns/index.ts index 29ed26283a7c..f6ba691b767e 100644 --- a/src/plugins/data/common/index_patterns/index.ts +++ b/src/plugins/data/common/index_patterns/index.ts @@ -33,3 +33,4 @@ export * from './types'; export { IndexPatternsService } from './index_patterns'; export type { IndexPattern } from './index_patterns'; export * from './errors'; +export { validateDataSourceReference, getIndexPatternTitle } from './utils'; diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts index 5e4d80df6ccd..628e8c03f377 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts @@ -366,7 +366,7 @@ export class IndexPattern implements IIndexPattern { }; } - getSaveObjectReference() { + getSaveObjectReference = () => { return this.dataSourceRef ? [ { @@ -376,7 +376,7 @@ export class IndexPattern implements IIndexPattern { }, ] : []; - } + }; /** * Provide a field, get its formatter diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index bd5bc48bba7c..688605821097 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -29,6 +29,7 @@ */ import { i18n } from '@osd/i18n'; +import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; import { SavedObjectsClientCommon } from '../..'; import { createIndexPatternCache } from '.'; import { IndexPattern } from './index_pattern'; @@ -53,7 +54,7 @@ import { FieldFormatsStartCommon } from '../../field_formats'; import { UI_SETTINGS, SavedObject } from '../../../common'; import { SavedObjectNotFound } from '../../../../opensearch_dashboards_utils/common'; import { IndexPatternMissingIndices } from '../lib'; -import { findByTitle } from '../utils'; +import { findByTitle, getIndexPatternTitle } from '../utils'; import { DuplicateIndexPatternError } from '../errors'; const indexPatternCache = createIndexPatternCache(); @@ -118,8 +119,28 @@ export class IndexPatternsService { fields: ['title'], perPage: 10000, }); + + this.savedObjectsCache = await Promise.all( + this.savedObjectsCache.map(async (obj) => { + if (obj.type === 'index-pattern') { + const result = { ...obj }; + result.attributes.title = await getIndexPatternTitle( + obj.attributes.title, + obj.references, + this.getDataSource + ); + return result; + } else { + return obj; + } + }) + ); } + getDataSource = async (id: string) => { + return await this.savedObjectsClient.get('data-source', id); + }; + /** * Get list of index pattern ids * @param refresh Force refresh of index pattern list @@ -557,7 +578,11 @@ export class IndexPatternsService { */ async createSavedObject(indexPattern: IndexPattern, override = false) { - const dupe = await findByTitle(this.savedObjectsClient, indexPattern.title); + const dupe = await findByTitle( + this.savedObjectsClient, + indexPattern.title, + indexPattern.dataSourceRef?.id + ); if (dupe) { if (override) { await this.delete(dupe.id); diff --git a/src/plugins/data/common/index_patterns/utils.test.ts b/src/plugins/data/common/index_patterns/utils.test.ts index ab0424487b20..1c1b56df5ba0 100644 --- a/src/plugins/data/common/index_patterns/utils.test.ts +++ b/src/plugins/data/common/index_patterns/utils.test.ts @@ -9,63 +9,84 @@ * GitHub history for details. */ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { isFilterable } from '.'; -import { IFieldType } from './fields'; +import { AuthType, DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; +import { IndexPatternSavedObjectAttrs } from './index_patterns'; +import { SavedObject, SavedObjectReference } from './types'; +import { getIndexPatternTitle, validateDataSourceReference } from './utils'; -const mockField = { - name: 'foo', - scripted: false, - searchable: true, - type: 'string', -} as IFieldType; +describe('test validateDataSourceReference', () => { + const getIndexPatternSavedObjectMock = (mockedFields: any = {}) => + ({ ...mockedFields } as SavedObject); + let indexPatternSavedObjectMock; + const dataSourceId = 'fakeDataSourceId'; -describe('isFilterable', () => { - describe('types', () => { - it('should return true for filterable types', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type })).toBe(true); - }); + test('ivalidateDataSourceReference should return false when datasource reference does not exist in index pattern', () => { + indexPatternSavedObjectMock = getIndexPatternSavedObjectMock({ + references: [{ name: 'someReference' }], }); - it('should return false for filterable types if the field is not searchable', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type, searchable: false })).toBe(false); - }); - }); + expect(validateDataSourceReference(indexPatternSavedObjectMock)).toBe(false); + expect(validateDataSourceReference(indexPatternSavedObjectMock, dataSourceId)).toBe(false); + }); - it('should return false for un-filterable types', () => { - ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach( - (type) => { - expect(isFilterable({ ...mockField, type })).toBe(false); - } - ); + test('ivalidateDataSourceReference should return true when datasource reference exists in index pattern, and datasource id matches', () => { + indexPatternSavedObjectMock = getIndexPatternSavedObjectMock({ + references: [{ type: 'data-source', id: dataSourceId }], }); + + expect(validateDataSourceReference(indexPatternSavedObjectMock)).toBe(false); + expect(validateDataSourceReference(indexPatternSavedObjectMock, dataSourceId)).toBe(true); + }); +}); + +describe('test getIndexPatternTitle', () => { + const dataSourceMock: SavedObject = { + id: 'dataSourceId', + type: 'data-source', + attributes: { + title: 'dataSourceMockTitle', + endpoint: 'https://fakeendpoint.com', + auth: { + type: AuthType.NoAuth, + credentials: undefined, + }, + }, + references: [], + }; + const indexPatternMockTitle = 'indexPatternMockTitle'; + const referencesMock: SavedObjectReference[] = [{ type: 'data-source', id: 'dataSourceId' }]; + + let getDataSourceMock: jest.Mock; + + beforeEach(() => { + getDataSourceMock = jest.fn().mockResolvedValue(dataSourceMock); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('getIndexPatternTitle should concat datasource title with index pattern title', async () => { + const res = await getIndexPatternTitle( + indexPatternMockTitle, + referencesMock, + getDataSourceMock + ); + expect(res).toEqual('dataSourceMockTitle.indexPatternMockTitle'); }); - it('should return true for scripted fields', () => { - expect(isFilterable({ ...mockField, scripted: true, searchable: false })).toBe(true); + test('getIndexPatternTitle should return index pattern title, when index-pattern is not referenced to any datasource', async () => { + const res = await getIndexPatternTitle(indexPatternMockTitle, [], getDataSourceMock); + expect(res).toEqual('indexPatternMockTitle'); }); - it('should return true for the _id field', () => { - expect(isFilterable({ ...mockField, name: '_id' })).toBe(true); + test('getIndexPatternTitle should return index pattern title, when failing to fetch datasource info', async () => { + getDataSourceMock = jest.fn().mockRejectedValue(new Error('error')); + const res = await getIndexPatternTitle( + indexPatternMockTitle, + referencesMock, + getDataSourceMock + ); + expect(res).toEqual('dataSourceId.indexPatternMockTitle'); }); }); diff --git a/src/plugins/data/common/index_patterns/utils.ts b/src/plugins/data/common/index_patterns/utils.ts index a2d9b07b1c81..bba25bfd6df8 100644 --- a/src/plugins/data/common/index_patterns/utils.ts +++ b/src/plugins/data/common/index_patterns/utils.ts @@ -28,26 +28,81 @@ * under the License. */ +import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; import type { IndexPatternSavedObjectAttrs } from './index_patterns'; -import type { SavedObjectsClientCommon } from '../types'; +import type { SavedObject, SavedObjectReference, SavedObjectsClientCommon } from '../types'; /** * Returns an object matching a given title * * @param client {SavedObjectsClientCommon} * @param title {string} + * @param dataSourceId {string}{optional} * @returns {Promise} */ -export async function findByTitle(client: SavedObjectsClientCommon, title: string) { +export async function findByTitle( + client: SavedObjectsClientCommon, + title: string, + dataSourceId?: string +) { if (title) { - const savedObjects = await client.find({ - type: 'index-pattern', - perPage: 10, - search: `"${title}"`, - searchFields: ['title'], - fields: ['title'], + const savedObjects = ( + await client.find({ + type: 'index-pattern', + perPage: 10, + search: `"${title}"`, + searchFields: ['title'], + fields: ['title'], + }) + ).filter((obj) => { + return obj && obj.attributes && validateDataSourceReference(obj, dataSourceId); }); return savedObjects.find((obj) => obj.attributes.title.toLowerCase() === title.toLowerCase()); } } + +// This is used to validate datasource reference of index pattern +export const validateDataSourceReference = ( + indexPattern: SavedObject, + dataSourceId?: string +) => { + const references = indexPattern.references; + if (dataSourceId) { + return references.some((ref) => ref.id === dataSourceId && ref.type === 'data-source'); + } else { + // No datasource id passed as input meaning we are getting index pattern from default cluster, + // and it's supposed to be an empty array + return references.length === 0; + } +}; + +export const getIndexPatternTitle = async ( + indexPatternTitle: string, + references: SavedObjectReference[], + getDataSource: (id: string) => Promise> +): Promise => { + const DATA_SOURCE_INDEX_PATTERN_DELIMITER = '.'; + let dataSourceTitle; + const dataSourceReference = references.find((ref) => ref.type === 'data-source'); + + // If an index-pattern references datasource, prepend data source name with index pattern name for display purpose + if (dataSourceReference) { + const dataSourceId = dataSourceReference.id; + try { + const { + attributes: { title }, + error, + } = await getDataSource(dataSourceId); + dataSourceTitle = error ? dataSourceId : title; + } catch (e) { + // use datasource id as title when failing to fetch datasource + dataSourceTitle = dataSourceId; + } + + return dataSourceTitle.concat(DATA_SOURCE_INDEX_PATTERN_DELIMITER).concat(indexPatternTitle); + } else { + // if index pattern doesn't reference datasource, return as it is. + return indexPatternTitle; + } +}; diff --git a/src/plugins/data/opensearch_dashboards.json b/src/plugins/data/opensearch_dashboards.json index 1193def7cb91..9dff517d5524 100644 --- a/src/plugins/data/opensearch_dashboards.json +++ b/src/plugins/data/opensearch_dashboards.json @@ -5,7 +5,7 @@ "ui": true, "requiredPlugins": ["expressions", "uiActions"], "optionalPlugins": ["usageCollection", "dataSource"], - "extraPublicDirs": ["common", "common/utils/abort_utils"], + "extraPublicDirs": ["common", "common/utils/abort_utils", "common/index_patterns/utils.ts"], "requiredBundles": [ "usageCollection", "opensearchDashboardsUtils", diff --git a/src/plugins/data_source/server/routes/test_connection.ts b/src/plugins/data_source/server/routes/test_connection.ts index b32148a57420..f36702171b45 100644 --- a/src/plugins/data_source/server/routes/test_connection.ts +++ b/src/plugins/data_source/server/routes/test_connection.ts @@ -30,18 +30,19 @@ export const registerTestConnectionRoute = ( schema.literal(AuthType.NoAuth), schema.literal(AuthType.SigV4), ]), - credentials: schema.oneOf([ - schema.object({ - username: schema.string(), - password: schema.string(), - }), - schema.object({ - region: schema.string(), - accessKey: schema.string(), - secretKey: schema.string(), - }), - schema.literal(null), - ]), + credentials: schema.maybe( + schema.oneOf([ + schema.object({ + username: schema.string(), + password: schema.string(), + }), + schema.object({ + region: schema.string(), + accessKey: schema.string(), + secretKey: schema.string(), + }), + ]) + ), }) ), }), diff --git a/src/plugins/data_source_management/public/components/utils.ts b/src/plugins/data_source_management/public/components/utils.ts index 539edbca970b..5f2cfb2337ad 100644 --- a/src/plugins/data_source_management/public/components/utils.ts +++ b/src/plugins/data_source_management/public/components/utils.ts @@ -4,7 +4,7 @@ */ import { HttpStart, SavedObjectsClientContract } from 'src/core/public'; -import { AuthType, DataSourceAttributes, DataSourceTableItem } from '../types'; +import { DataSourceAttributes, DataSourceTableItem } from '../types'; export async function getDataSources(savedObjectsClient: SavedObjectsClientContract) { return savedObjectsClient diff --git a/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.test.tsx index 566a54b538f0..9298aef92cf0 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.test.tsx @@ -47,14 +47,14 @@ const indexPattern = { const indexPattern1 = { id: 'test1', attributes: { - title: 'test1 title', + title: 'test1 titleToDisplay', }, } as SavedObject; const indexPattern2 = { id: 'test2', attributes: { - title: 'test2 title', + title: 'test2 titleToDisplay', }, } as SavedObject; @@ -97,15 +97,15 @@ describe('DiscoverIndexPattern', () => { const instance = shallow(); expect(getIndexPatternPickerOptions(instance)!.map((option: any) => option.label)).toEqual([ - 'test1 title', - 'test2 title', + 'test1 titleToDisplay', + 'test2 titleToDisplay', ]); }); test('should switch data panel to target index pattern', () => { const instance = shallow(); - selectIndexPatternPickerOption(instance, 'test2 title'); + selectIndexPatternPickerOption(instance, 'test2 titleToDisplay'); expect(defaultProps.setIndexPattern).toHaveBeenCalledWith('test2'); }); }); diff --git a/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.tsx b/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.tsx index 18e874aecb2d..95154bec1939 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_index_pattern.tsx @@ -70,8 +70,10 @@ export function DiscoverIndexPattern({ }); useEffect(() => { const { id, title } = selectedIndexPattern; - setSelected({ id, title }); - }, [selectedIndexPattern]); + const indexPattern = indexPatternList.find((pattern) => pattern.id === id); + const titleToDisplay = indexPattern ? indexPattern.attributes!.title : title; + setSelected({ id, title: titleToDisplay }); + }, [indexPatternList, selectedIndexPattern]); if (!selectedId) { return null; } diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx index 76e47b0875e5..3853574f0ab7 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/components/step_index_pattern/step_index_pattern.tsx @@ -55,6 +55,7 @@ import { context as contextType } from '../../../../../../opensearch_dashboards_ import { IndexPatternCreationConfig } from '../../../../../../../plugins/index_pattern_management/public'; import { MatchedItem, StepInfo } from '../../types'; import { DataSourceRef, IndexPatternManagmentContextValue } from '../../../../types'; +import { validateDataSourceReference } from '../../../../../../../plugins/data/common'; interface StepIndexPatternProps { allIndices: MatchedItem[]; @@ -131,7 +132,7 @@ export class StepIndexPattern extends Component - obj && obj.attributes ? obj.attributes.title : '' + obj && obj.attributes && validateDataSourceReference(obj, this.props.dataSourceRef?.id) + ? obj.attributes.title + : '' ) as string[]; this.setState({ existingIndexPatterns }); @@ -461,7 +464,7 @@ export class StepIndexPattern extends Component {this.renderList(matchedIndices)} - {this.dataSrouceEnabled && this.renderGoToPrevious()} + {this.dataSourceEnabled && this.renderGoToPrevious()} ); } diff --git a/src/plugins/saved_objects/public/finder/saved_object_finder.tsx b/src/plugins/saved_objects/public/finder/saved_object_finder.tsx index f2ca3b883911..b0b1409bf8b7 100644 --- a/src/plugins/saved_objects/public/finder/saved_object_finder.tsx +++ b/src/plugins/saved_objects/public/finder/saved_object_finder.tsx @@ -60,6 +60,8 @@ import { SavedObjectsStart, } from 'src/core/public'; +import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; +import { getIndexPatternTitle } from '../../../data/common/index_patterns/utils'; import { LISTING_LIMIT_SETTING } from '../../common'; export interface SavedObjectMetaData { @@ -155,7 +157,28 @@ class SavedObjectFinderUi extends React.Component< defaultSearchOperator: 'AND', }); - resp.savedObjects = resp.savedObjects.filter((savedObject) => { + const getDataSource = async (id: string) => { + const client = this.props.savedObjects.client; + return await client.get('data-source', id); + }; + + const savedObjects = await Promise.all( + resp.savedObjects.map(async (obj) => { + if (obj.type === 'index-pattern') { + const result = { ...obj }; + result.attributes.title = await getIndexPatternTitle( + obj.attributes.title!, + obj.references, + getDataSource + ); + return result; + } else { + return obj; + } + }) + ); + + resp.savedObjects = savedObjects.filter((savedObject) => { const metaData = metaDataMap[savedObject.type]; if (metaData.showSavedObject) { return metaData.showSavedObject(savedObject); diff --git a/src/plugins/saved_objects_management/server/routes/find.ts b/src/plugins/saved_objects_management/server/routes/find.ts index c2002fc97a15..dd49fc7575df 100644 --- a/src/plugins/saved_objects_management/server/routes/find.ts +++ b/src/plugins/saved_objects_management/server/routes/find.ts @@ -30,6 +30,8 @@ import { schema } from '@osd/config-schema'; import { IRouter } from 'src/core/server'; +import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; +import { getIndexPatternTitle } from '../../../data/common/index_patterns/utils'; import { injectMetaAttributes } from '../lib'; import { ISavedObjectsManagement } from '../services'; @@ -84,13 +86,33 @@ export const registerFindRoute = ( } }); + const getDataSource = async (id: string) => { + return await client.get('data-source', id); + }; + const findResponse = await client.find({ ...req.query, fields: undefined, searchFields: [...searchFields], }); - const enhancedSavedObjects = findResponse.saved_objects + const savedObjects = await Promise.all( + findResponse.saved_objects.map(async (obj) => { + if (obj.type === 'index-pattern') { + const result = { ...obj }; + result.attributes.title = await getIndexPatternTitle( + obj.attributes.title, + obj.references, + getDataSource + ); + return result; + } else { + return obj; + } + }) + ); + + const enhancedSavedObjects = savedObjects .map((so) => injectMetaAttributes(so, managementService)) .map((obj) => { const result = { ...obj, attributes: {} as Record };