From 67e6b3065c5b03c636a9b0dd938fb586c3c23ac4 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Fri, 14 Jun 2024 00:04:05 +0000 Subject: [PATCH 1/2] Clean up files and add helper functions Signed-off-by: Kawika Avilla --- common/constants.ts | 36 ++++ common/index.ts | 26 +-- opensearch_dashboards.json | 4 +- package.json | 13 +- public/index.ts | 8 +- public/plugin.tsx | 12 +- public/search/index.ts | 7 + public/search/ppl_search_interceptor.ts | 13 +- public/search/sql_search_interceptor.ts | 15 +- public/services.ts | 5 + public/types.ts | 5 + server/index.ts | 12 ++ server/plugin.ts | 22 +-- server/routes/index.ts | 70 +++----- server/search/engine_plugin.ts | 156 ----------------- server/search/index.ts | 7 + server/search/ppl/ppl_datasource.ts | 84 --------- server/search/ppl/ppl_facet.ts | 43 ----- server/search/ppl/ppl_plugin.ts | 89 ---------- .../search/{ppl => }/ppl_search_strategy.ts | 17 +- server/search/sql/sql_facet.ts | 39 ----- .../search/{sql => }/sql_search_strategy.ts | 16 +- server/types.ts | 23 ++- server/ui_settings.ts | 25 --- server/utils/facet.ts | 58 +++++++ server/utils/index.ts | 9 + server/utils/plugins.ts | 159 ++++++++++++++++++ server/utils/shim_schema_row.ts | 28 +++ server/utils/shim_stats.ts | 48 ++++++ tsconfig.json | 2 +- 30 files changed, 495 insertions(+), 556 deletions(-) create mode 100644 common/constants.ts create mode 100644 public/search/index.ts delete mode 100644 server/search/engine_plugin.ts create mode 100644 server/search/index.ts delete mode 100644 server/search/ppl/ppl_datasource.ts delete mode 100644 server/search/ppl/ppl_facet.ts delete mode 100644 server/search/ppl/ppl_plugin.ts rename server/search/{ppl => }/ppl_search_strategy.ts (90%) delete mode 100644 server/search/sql/sql_facet.ts rename server/search/{sql => }/sql_search_strategy.ts (81%) delete mode 100644 server/ui_settings.ts create mode 100644 server/utils/facet.ts create mode 100644 server/utils/index.ts create mode 100644 server/utils/plugins.ts create mode 100644 server/utils/shim_schema_row.ts create mode 100644 server/utils/shim_stats.ts diff --git a/common/constants.ts b/common/constants.ts new file mode 100644 index 0000000..aa78e9a --- /dev/null +++ b/common/constants.ts @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const PLUGIN_ID = 'queryEnhancements'; +export const PLUGIN_NAME = 'queryEnhancements'; + +export const BASE_API = '/api/enhancements'; + +export const SEARCH_STRATEGY = { + PPL: 'ppl', + SQL: 'sql', +}; + +export const API = { + SEARCH: `${BASE_API}/search`, + PPL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.PPL}`, + SQL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.SQL}`, +}; + +export const URI = { + PPL: '/_plugins/_ppl', + SQL: '/_plugins/_sql', + ASYNC_QUERY: '/_plugins/_async_query', + ML: '/_plugins/_ml', + OBSERVABILITY: '/_plugins/_observability', + DATA_CONNECTIONS: '/_plugins/_query/_datasources', +}; + +export const OPENSEARCH_API = { + PANELS: `${URI.OBSERVABILITY}/object`, + DATA_CONNECTIONS: URI.DATA_CONNECTIONS, +}; + +export const UI_SETTINGS = {}; diff --git a/common/index.ts b/common/index.ts index 87b8314..bcc8722 100644 --- a/common/index.ts +++ b/common/index.ts @@ -3,29 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ -export const PLUGIN_ID = 'queryEnhancements'; -export const PLUGIN_NAME = 'queryEnhancements'; - -export const PPL_SEARCH_STRATEGY = 'ppl'; -export const SQL_SEARCH_STRATEGY = 'sql'; - -export const PPL_ENDPOINT = '/_plugins/_ppl'; -export const SQL_ENDPOINT = '/_plugins/_sql'; - -const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; -const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; -export const OPENSEARCH_PANELS_API = { - OBJECT: `${BASE_OBSERVABILITY_URI}/object`, -}; -export const OPENSEARCH_DATACONNECTIONS_API = { - DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, -}; - -export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; - -export const BASE_ML_COMMONS_URI = '/_plugins/_ml'; - -// Advanced Settings -export const QUERY_ENABLE_ENHANCEMENTS_SETTING = 'query:enableEnhancements'; - +export * from './constants'; export * from './utils'; diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index c2beeb6..9512432 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -1,10 +1,10 @@ { "id": "queryEnhancements", - "version": "1.0.0", + "version": "3.0.0.0", "opensearchDashboardsVersion": "opensearchDashboards", "server": true, "ui": true, "requiredPlugins": ["data"], "optionalPlugins": ["dataSource", "dataSourceManagement"], - "requiredBundles": ["opensearchDashboardsUtils"] + "requiredBundles": [] } diff --git a/package.json b/package.json index 13954d9..c920fb9 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,13 @@ { "name": "queryEnhancements", - "version": "0.0.0", - "private": true, + "license": "Apache-2.0", + "version": "3.0.0.0", + "main": "index.ts", "scripts": { "build": "yarn plugin-helpers build", - "plugin-helpers": "../../scripts/use_node ../../scripts/plugin_helpers", - "osd": "../../scripts/use_node ../../scripts/osd" - } + "plugin-helpers": "node ../../scripts/plugin_helpers", + "osd": "node ../../scripts/osd" + }, + "dependencies": {}, + "devDependencies": {} } diff --git a/public/index.ts b/public/index.ts index e02f44d..d2bb583 100644 --- a/public/index.ts +++ b/public/index.ts @@ -1,10 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import './index.scss'; import { QueryEnhancementsPlugin } from './plugin'; -// This exports static code and TypeScript types, -// as well as, OpenSearch Dashboards Platform `plugin()` initializer. export function plugin() { return new QueryEnhancementsPlugin(); } + export { QueryEnhancementsPluginSetup, QueryEnhancementsPluginStart } from './types'; diff --git a/public/plugin.tsx b/public/plugin.tsx index 7e43c36..353018a 100644 --- a/public/plugin.tsx +++ b/public/plugin.tsx @@ -1,8 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import moment from 'moment'; import { CoreSetup, CoreStart, Plugin } from '../../../src/core/public'; import { IStorageWrapper, Storage } from '../../../src/plugins/opensearch_dashboards_utils/public'; -import { PPLQlSearchInterceptor } from './search/ppl_search_interceptor'; -import { SQLQlSearchInterceptor } from './search/sql_search_interceptor'; +import { PPLSearchInterceptor, SQLSearchInterceptor } from './search'; import { setData, setStorage } from './services'; import { QueryEnhancementsPluginSetup, @@ -23,7 +27,7 @@ export class QueryEnhancementsPlugin core: CoreSetup, { data }: QueryEnhancementsPluginSetupDependencies ): QueryEnhancementsPluginSetup { - const pplSearchInterceptor = new PPLQlSearchInterceptor({ + const pplSearchInterceptor = new PPLSearchInterceptor({ toasts: core.notifications.toasts, http: core.http, uiSettings: core.uiSettings, @@ -31,7 +35,7 @@ export class QueryEnhancementsPlugin usageCollector: data.search.usageCollector, }); - const sqlSearchInterceptor = new SQLQlSearchInterceptor({ + const sqlSearchInterceptor = new SQLSearchInterceptor({ toasts: core.notifications.toasts, http: core.http, uiSettings: core.uiSettings, diff --git a/public/search/index.ts b/public/search/index.ts new file mode 100644 index 0000000..624e7cf --- /dev/null +++ b/public/search/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { PPLSearchInterceptor } from './ppl_search_interceptor'; +export { SQLSearchInterceptor } from './sql_search_interceptor'; diff --git a/public/search/ppl_search_interceptor.ts b/public/search/ppl_search_interceptor.ts index 00d84aa..f8a9e4a 100644 --- a/public/search/ppl_search_interceptor.ts +++ b/public/search/ppl_search_interceptor.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { trimEnd } from 'lodash'; import { Observable, from } from 'rxjs'; import { stringify } from '@osd/std'; @@ -20,10 +25,10 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../../src/plugins/data/public'; -import { formatDate, PPL_SEARCH_STRATEGY, removeKeyword } from '../../common'; +import { formatDate, SEARCH_STRATEGY, removeKeyword, API } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -export class PPLQlSearchInterceptor extends SearchInterceptor { +export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; @@ -42,7 +47,7 @@ export class PPLQlSearchInterceptor extends SearchInterceptor { strategy?: string ): Observable { const { id, ...searchRequest } = request; - const path = trimEnd('/api/pplql/search'); + const path = trimEnd(API.PPL_SEARCH); const { timefilter } = this.queryService; const dateRange = timefilter.timefilter.getTime(); const { fromDate, toDate } = formatTimePickerDate(dateRange, 'YYYY-MM-DD HH:mm:ss.SSS'); @@ -186,6 +191,6 @@ export class PPLQlSearchInterceptor extends SearchInterceptor { } public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, PPL_SEARCH_STRATEGY); + return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.PPL); } } diff --git a/public/search/sql_search_interceptor.ts b/public/search/sql_search_interceptor.ts index 24e4008..3fb7d28 100644 --- a/public/search/sql_search_interceptor.ts +++ b/public/search/sql_search_interceptor.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { trimEnd } from 'lodash'; import { Observable, from } from 'rxjs'; import { stringify } from '@osd/std'; @@ -10,10 +15,10 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../../src/plugins/data/public'; -import { SQL_SEARCH_STRATEGY } from '../../common'; +import { API, SEARCH_STRATEGY } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -export class SQLQlSearchInterceptor extends SearchInterceptor { +export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; @@ -32,7 +37,7 @@ export class SQLQlSearchInterceptor extends SearchInterceptor { strategy?: string ): Observable { const { id, ...searchRequest } = request; - const path = trimEnd('/api/sqlql/search'); + const path = trimEnd(API.SQL_SEARCH); const fetchDataFrame = (queryString: string, df = null) => { const body = stringify({ query: { qs: queryString, format: 'jdbc' }, df }); @@ -56,7 +61,7 @@ export class SQLQlSearchInterceptor extends SearchInterceptor { if (!df.body.error) return; const jsError = new Error(df.body.error.response); this.deps.toasts.addError(jsError, { - title: i18n.translate('dqlPlugin.sqlQueryError', { + title: i18n.translate('queryEnhancements.sqlQueryError', { defaultMessage: 'Could not complete the SQL query', }), toastMessage: df.body.error.msg, @@ -67,6 +72,6 @@ export class SQLQlSearchInterceptor extends SearchInterceptor { } public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, SQL_SEARCH_STRATEGY); + return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); } } diff --git a/public/services.ts b/public/services.ts index 7132f4a..92d2002 100644 --- a/public/services.ts +++ b/public/services.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { createGetterSetter } from '../../../src/plugins/opensearch_dashboards_utils/common'; import { IStorageWrapper } from '../../../src/plugins/opensearch_dashboards_utils/public'; import { DataPublicPluginStart } from '../../../src/plugins/data/public'; diff --git a/public/types.ts b/public/types.ts index 67ee3f3..bcfa21d 100644 --- a/public/types.ts +++ b/public/types.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { DataPublicPluginSetup, DataPublicPluginStart } from 'src/plugins/data/public'; // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/server/index.ts b/server/index.ts index ccebeaf..45aade2 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { PluginConfigDescriptor, PluginInitializerContext } from '../../../src/core/server'; import { QueryEnhancementsPlugin } from './plugin'; import { configSchema, ConfigSchema } from '../common/config'; @@ -11,4 +16,11 @@ export function plugin(initializerContext: PluginInitializerContext) { return new QueryEnhancementsPlugin(initializerContext); } +export { + Facet, + OpenSearchPPLPlugin, + OpenSearchObservabilityPlugin, + shimStats, + shimSchemaRow, +} from './utils'; export { QueryEnhancementsPluginSetup, QueryEnhancementsPluginStart } from './types'; diff --git a/server/plugin.ts b/server/plugin.ts index 3af5dc9..a0ce775 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { Observable } from 'rxjs'; import { CoreSetup, @@ -7,18 +12,15 @@ import { PluginInitializerContext, SharedGlobalConfig, } from '../../../src/core/server'; -import { PPL_SEARCH_STRATEGY, SQL_ASYNC_SEARCH_STRATEGY, SQL_SEARCH_STRATEGY } from '../common'; +import { SEARCH_STRATEGY } from '../common'; import { defineRoutes } from './routes'; -import { EnginePlugin } from './search/engine_plugin'; -import { PPLPlugin } from './search/ppl/ppl_plugin'; -import { pplSearchStrategyProvider } from './search/ppl/ppl_search_strategy'; -import { sqlSearchStrategyProvider } from './search/sql/sql_search_strategy'; +import { pplSearchStrategyProvider, sqlSearchStrategyProvider } from './search'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, QueryEnhancementsPluginStart, } from './types'; -import { uiSettings } from './ui_settings'; +import { OpenSearchPPLPlugin, OpenSearchObservabilityPlugin } from './utils'; export class QueryEnhancementsPlugin implements Plugin { @@ -34,16 +36,14 @@ export class QueryEnhancementsPlugin const router = core.http.createRouter(); // Register server side APIs const client = core.opensearch.legacy.createClient('opensearch_observability', { - plugins: [PPLPlugin, EnginePlugin], + plugins: [OpenSearchPPLPlugin, OpenSearchObservabilityPlugin], }); - core.uiSettings.register(uiSettings); - const pplSearchStrategy = pplSearchStrategyProvider(this.config$, this.logger, client); const sqlSearchStrategy = sqlSearchStrategyProvider(this.config$, this.logger, client); - data.search.registerSearchStrategy(PPL_SEARCH_STRATEGY, pplSearchStrategy); - data.search.registerSearchStrategy(SQL_SEARCH_STRATEGY, sqlSearchStrategy); + data.search.registerSearchStrategy(SEARCH_STRATEGY.PPL, pplSearchStrategy); + data.search.registerSearchStrategy(SEARCH_STRATEGY.SQL, sqlSearchStrategy); defineRoutes(this.logger, router, { ppl: pplSearchStrategy, diff --git a/server/routes/index.ts b/server/routes/index.ts index fbc315c..d9c0f65 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { schema } from '@osd/config-schema'; import { IOpenSearchDashboardsResponse, @@ -10,18 +15,21 @@ import { IOpenSearchDashboardsSearchRequest, } from '../../../../src/plugins/data/common'; import { ISearchStrategy } from '../../../../src/plugins/data/server'; +import { SEARCH_STRATEGY, API } from '../../common'; -export function defineRoutes( +function defineRoute( logger: Logger, router: IRouter, searchStrategies: Record< string, ISearchStrategy - > + >, + searchStrategyId: string ) { + const path = `${API.SEARCH}/${searchStrategyId}`; router.post( { - path: `/api/pplql/search`, + path, validate: { body: schema.object({ query: schema.object({ @@ -34,17 +42,12 @@ export function defineRoutes( }, async (context, req, res): Promise> => { try { - const queryRes: IDataFrameResponse = await searchStrategies.ppl.search( + const queryRes: IDataFrameResponse = await searchStrategies[searchStrategyId].search( context, req as any, {} ); - const result: any = { - body: { - ...queryRes, - }, - }; - return res.ok(result); + return res.ok({ body: { ...queryRes } }); } catch (err) { logger.error(err); return res.custom({ @@ -54,41 +57,16 @@ export function defineRoutes( } } ); +} - // sql - router.post( - { - path: `/api/sqlql/search`, - validate: { - body: schema.object({ - query: schema.object({ - qs: schema.string(), - format: schema.string(), - }), - df: schema.nullable(schema.object({}, { unknowns: 'allow' })), - }), - }, - }, - async (context, req, res): Promise> => { - try { - const queryRes: IDataFrameResponse = await searchStrategies.sql.search( - context, - req as any, - {} - ); - const result: any = { - body: { - ...queryRes, - }, - }; - return res.ok(result); - } catch (err) { - logger.error(err); - return res.custom({ - statusCode: 500, - body: err, - }); - } - } - ); +export function defineRoutes( + logger: Logger, + router: IRouter, + searchStrategies: Record< + string, + ISearchStrategy + > +) { + defineRoute(logger, router, searchStrategies, SEARCH_STRATEGY.PPL); + defineRoute(logger, router, searchStrategies, SEARCH_STRATEGY.SQL); } diff --git a/server/search/engine_plugin.ts b/server/search/engine_plugin.ts deleted file mode 100644 index ec7188d..0000000 --- a/server/search/engine_plugin.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { JOBS_ENDPOINT_BASE, OPENSEARCH_PANELS_API } from '../../common'; - -export const EnginePlugin = (client: any, config: any, components: any) => { - const clientAction = components.clientAction.factory; - - client.prototype.observability = components.clientAction.namespaceFactory(); - const observability = client.prototype.observability.prototype; - - // Get Object - observability.getObject = clientAction({ - url: { - fmt: OPENSEARCH_PANELS_API.OBJECT, - params: { - objectId: { - type: 'string', - }, - objectIdList: { - type: 'string', - }, - objectType: { - type: 'string', - }, - sortField: { - type: 'string', - }, - sortOrder: { - type: 'string', - }, - fromIndex: { - type: 'number', - }, - maxItems: { - type: 'number', - }, - name: { - type: 'string', - }, - lastUpdatedTimeMs: { - type: 'string', - }, - createdTimeMs: { - type: 'string', - }, - }, - }, - method: 'GET', - }); - - // Get Object by Id - observability.getObjectById = clientAction({ - url: { - fmt: `${OPENSEARCH_PANELS_API.OBJECT}/<%=objectId%>`, - req: { - objectId: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - // Create new Object - observability.createObject = clientAction({ - url: { - fmt: OPENSEARCH_PANELS_API.OBJECT, - }, - method: 'POST', - needBody: true, - }); - - // Update Object by Id - observability.updateObjectById = clientAction({ - url: { - fmt: `${OPENSEARCH_PANELS_API.OBJECT}/<%=objectId%>`, - req: { - objectId: { - type: 'string', - required: true, - }, - }, - }, - method: 'PUT', - needBody: true, - }); - - // Delete Object by Id - observability.deleteObjectById = clientAction({ - url: { - fmt: `${OPENSEARCH_PANELS_API.OBJECT}/<%=objectId%>`, - req: { - objectId: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - // Delete Object by Id List - observability.deleteObjectByIdList = clientAction({ - url: { - fmt: OPENSEARCH_PANELS_API.OBJECT, - params: { - objectIdList: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - // Get async job status - observability.getJobStatus = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, - req: { - queryId: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - // Delete async job - observability.deleteJob = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, - req: { - queryId: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - // Run async job - observability.runDirectQuery = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}`, - }, - method: 'POST', - needBody: true, - }); -}; diff --git a/server/search/index.ts b/server/search/index.ts new file mode 100644 index 0000000..69ad6ee --- /dev/null +++ b/server/search/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { pplSearchStrategyProvider } from './ppl_search_strategy'; +export { sqlSearchStrategyProvider } from './sql_search_strategy'; diff --git a/server/search/ppl/ppl_datasource.ts b/server/search/ppl/ppl_datasource.ts deleted file mode 100644 index c746749..0000000 --- a/server/search/ppl/ppl_datasource.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import _ from 'lodash'; -import { IPPLEventsDataSource, IPPLVisualizationDataSource } from '../../types'; - -type PPLResponse = IPPLEventsDataSource & IPPLVisualizationDataSource; - -export class PPLDataSource { - constructor(private pplDataSource: PPLResponse, private dataType: string) { - if (this.dataType === 'jdbc') { - this.addSchemaRowMapping(); - } else if (this.dataType === 'viz') { - this.addStatsMapping(); - } - } - - private addStatsMapping = () => { - const visData = this.pplDataSource; - - /** - * Add vis mapping for runtime fields - * json data structure added to response will be - * [{ - * agent: "mozilla", - * avg(bytes): 5756 - * ... - * }, { - * agent: "MSIE", - * avg(bytes): 5605 - * ... - * }, { - * agent: "chrome", - * avg(bytes): 5648 - * ... - * }] - */ - const res = []; - if (visData?.metadata?.fields) { - const queriedFields = visData.metadata.fields; - for (let i = 0; i < visData.size; i++) { - const entry: any = {}; - queriedFields.map((field: any) => { - const statsDataSet = visData?.data; - entry[field.name] = statsDataSet[field.name][i]; - }); - res.push(entry); - } - visData.jsonData = res; - } - }; - - /** - * Add 'schemaName: data' entries for UI rendering - */ - private addSchemaRowMapping = () => { - const pplRes = this.pplDataSource; - - const data: any[] = []; - - _.forEach(pplRes.datarows, (row) => { - const record: any = {}; - - for (let i = 0; i < pplRes.schema.length; i++) { - const cur = pplRes.schema[i]; - - if (typeof row[i] === 'object') { - record[cur.name] = JSON.stringify(row[i]); - } else if (typeof row[i] === 'boolean') { - record[cur.name] = row[i].toString(); - } else { - record[cur.name] = row[i]; - } - } - - data.push(record); - }); - pplRes.jsonData = data; - }; - - public getDataSource = (): PPLResponse => this.pplDataSource; -} diff --git a/server/search/ppl/ppl_facet.ts b/server/search/ppl/ppl_facet.ts deleted file mode 100644 index 5d186ea..0000000 --- a/server/search/ppl/ppl_facet.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import _ from 'lodash'; -import { PPLDataSource } from './ppl_datasource'; - -export class PPLFacet { - constructor(private client: any) { - this.client = client; - } - - private fetch = async (request: any, format: string, responseFormat: string) => { - const res = { - success: false, - data: {}, - }; - try { - const params = { - body: { - query: request.body.query, - }, - }; - if (request.body.format !== 'jdbc') { - params.format = request.body.format; - } - const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); - const pplDataSource = new PPLDataSource(queryRes, request.body.format); - res.success = true; - res.data = pplDataSource.getDataSource(); - } catch (err: any) { - // eslint-disable-next-line no-console - console.error('PPL query fetch err: ', err); - res.data = err; - } - return res; - }; - - describeQuery = async (request: any) => { - return this.fetch(request, 'ppl.pplQuery', 'json'); - }; -} diff --git a/server/search/ppl/ppl_plugin.ts b/server/search/ppl/ppl_plugin.ts deleted file mode 100644 index 85b1cd3..0000000 --- a/server/search/ppl/ppl_plugin.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { OPENSEARCH_DATACONNECTIONS_API, PPL_ENDPOINT, SQL_ENDPOINT } from '../../../common'; - -export const PPLPlugin = (client: any, config: any, components: any) => { - const ca = components.clientAction.factory; - client.prototype.ppl = components.clientAction.namespaceFactory(); - const ppl = client.prototype.ppl.prototype; - - ppl.pplQuery = ca({ - url: { - fmt: `${PPL_ENDPOINT}`, - params: { - format: { - type: 'string', - required: true, - }, - }, - }, - needBody: true, - method: 'POST', - }); - - ppl.sqlQuery = ca({ - url: { - fmt: `${SQL_ENDPOINT}`, - params: { - format: { - type: 'string', - required: true, - }, - }, - }, - needBody: true, - method: 'POST', - }); - - ppl.getDataConnectionById = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, - req: { - dataconnection: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - ppl.deleteDataConnection = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, - req: { - dataconnection: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - ppl.createDataSource = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - needBody: true, - method: 'POST', - }); - - ppl.modifyDataConnection = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - needBody: true, - method: 'PATCH', - }); - - ppl.getDataConnections = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - method: 'GET', - }); -}; diff --git a/server/search/ppl/ppl_search_strategy.ts b/server/search/ppl_search_strategy.ts similarity index 90% rename from server/search/ppl/ppl_search_strategy.ts rename to server/search/ppl_search_strategy.ts index 6153154..4d6cee1 100644 --- a/server/search/ppl/ppl_search_strategy.ts +++ b/server/search/ppl_search_strategy.ts @@ -1,3 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ import { first } from 'rxjs/operators'; import { SharedGlobalConfig, Logger, ILegacyClusterClient } from 'opensearch-dashboards/server'; import { Observable } from 'rxjs'; @@ -5,15 +9,16 @@ import { ISearchStrategy, getDefaultSearchParams, SearchUsage, -} from '../../../../../src/plugins/data/server'; +} from '../../../../src/plugins/data/server'; import { + IDataFrameError, IDataFrameResponse, IDataFrameWithAggs, IOpenSearchDashboardsSearchRequest, createDataFrame, -} from '../../../../../src/plugins/data/common'; -import { PPLFacet } from './ppl_facet'; -import { getFields } from '../../../common/utils'; +} from '../../../../src/plugins/data/common'; +import { getFields } from '../../common/utils'; +import { Facet } from '../utils'; export const pplSearchStrategyProvider = ( config$: Observable, @@ -21,7 +26,7 @@ export const pplSearchStrategyProvider = ( client: ILegacyClusterClient, usage?: SearchUsage ): ISearchStrategy => { - const pplFacet = new PPLFacet(client); + const pplFacet = new Facet(client, logger, 'ppl.pplQuery', true); const parseRequest = (query: string) => { const pipeMap = new Map(); @@ -78,7 +83,7 @@ export const pplSearchStrategyProvider = ( type: 'data_frame', body: { error: rawResponse.data }, took: rawResponse.took, - }; + } as IDataFrameError; } const dataFrame = createDataFrame({ diff --git a/server/search/sql/sql_facet.ts b/server/search/sql/sql_facet.ts deleted file mode 100644 index 09f3ca0..0000000 --- a/server/search/sql/sql_facet.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export class SQLFacet { - constructor(private client: any) { - this.client = client; - } - - private fetch = async (request: any, format: string, responseFormat: string) => { - const res = { - success: false, - data: {}, - }; - try { - const params = { - body: { - query: request.body.query, - }, - }; - if (request.body.format !== 'jdbc') { - params.format = request.body.format; - } - const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); - res.success = true; - res.data = queryRes; - } catch (err: any) { - // eslint-disable-next-line no-console - console.error('SQL query fetch err: ', err); - res.data = err; - } - return res; - }; - - describeQuery = async (request: any) => { - return this.fetch(request, 'ppl.sqlQuery', 'json'); - }; -} diff --git a/server/search/sql/sql_search_strategy.ts b/server/search/sql_search_strategy.ts similarity index 81% rename from server/search/sql/sql_search_strategy.ts rename to server/search/sql_search_strategy.ts index 25fb051..d1aaadd 100644 --- a/server/search/sql/sql_search_strategy.ts +++ b/server/search/sql_search_strategy.ts @@ -1,13 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + import { SharedGlobalConfig, Logger, ILegacyClusterClient } from 'opensearch-dashboards/server'; import { Observable } from 'rxjs'; -import { ISearchStrategy, SearchUsage } from '../../../../../src/plugins/data/server'; +import { ISearchStrategy, SearchUsage } from '../../../../src/plugins/data/server'; import { + IDataFrameError, IDataFrameResponse, IOpenSearchDashboardsSearchRequest, PartialDataFrame, createDataFrame, -} from '../../../../../src/plugins/data/common'; -import { SQLFacet } from './sql_facet'; +} from '../../../../src/plugins/data/common'; +import { Facet } from '../utils'; export const sqlSearchStrategyProvider = ( config$: Observable, @@ -15,7 +21,7 @@ export const sqlSearchStrategyProvider = ( client: ILegacyClusterClient, usage?: SearchUsage ): ISearchStrategy => { - const sqlFacet = new SQLFacet(client); + const sqlFacet = new Facet(client, logger, 'ppl.sqlQuery'); return { search: async (context, request: any, options) => { @@ -28,7 +34,7 @@ export const sqlSearchStrategyProvider = ( type: 'data_frame', body: { error: rawResponse.data }, took: rawResponse.took, - }; + } as IDataFrameError; } const partial: PartialDataFrame = { diff --git a/server/types.ts b/server/types.ts index 1f4abcd..7335075 100644 --- a/server/types.ts +++ b/server/types.ts @@ -10,6 +10,7 @@ import { DataSourcePluginStart } from '../../../src/plugins/data_source/server'; export interface QueryEnhancementsPluginSetup {} // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface QueryEnhancementsPluginStart {} + export interface QueryEnhancementsPluginSetupDependencies { data: DataPluginSetup; dataSource?: DataSourcePluginStart; @@ -20,16 +21,30 @@ export interface ISchema { type: string; } -export interface IPPLVisualizationDataSource { +export interface IPPLDataSource { + jsonData?: any[]; +} + +export interface IPPLVisualizationDataSource extends IPPLDataSource { data: any; metadata: any; - jsonData?: any[]; size: number; status: number; } -export interface IPPLEventsDataSource { +export interface IPPLEventsDataSource extends IPPLDataSource { schema: ISchema[]; datarows: any[]; - jsonData?: any[]; +} + +export interface FacetResponse { + success: boolean; + data: any; +} + +export interface FacetRequest { + body: { + query: string; + format?: string; + }; } diff --git a/server/ui_settings.ts b/server/ui_settings.ts deleted file mode 100644 index c659e52..0000000 --- a/server/ui_settings.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { i18n } from '@osd/i18n'; -import { schema } from '@osd/config-schema'; - -import { UiSettingsParams } from 'opensearch-dashboards/server'; -import { QUERY_ASSIST_DISABLED } from '../common'; - -export const uiSettings: Record = { - [QUERY_ASSIST_DISABLED]: { - name: i18n.translate('queryEnhancements.advancedSettings.queryAssistDisabledTitle', { - defaultMessage: 'Disable experimental query assist in the search bar', - }), - value: false, - description: i18n.translate('queryEnhancements.advancedSettings.queryAssistDisabledText', { - defaultMessage: `If the query assist is setup and enabled, the query bar will provide suggestions and auto-completions as you type. - This is an experimental feature and may not work as expected. If you encounter issues, you can disable it here.`, - }), - category: ['search'], - schema: schema.boolean(), - }, -}; diff --git a/server/utils/facet.ts b/server/utils/facet.ts new file mode 100644 index 0000000..7b0f884 --- /dev/null +++ b/server/utils/facet.ts @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Logger } from 'opensearch-dashboards/server'; +import { FacetResponse, IPPLEventsDataSource, IPPLVisualizationDataSource } from '../types'; +import { shimSchemaRow, shimStats } from '.'; + +export class Facet { + constructor( + private client: any, + private logger: Logger, + private endpoint: string, + private shimResponse: boolean = false + ) { + this.client = client; + this.logger = logger; + this.endpoint = endpoint; + this.shimResponse = shimResponse; + } + + protected fetch = async (request: any, endpoint: string): Promise => { + try { + const { query, format } = request.body; + const params = { + body: { query }, + ...(format !== 'jdbc' && { format }), + }; + const queryRes = await this.client.asScoped(request).callAsCurrentUser(endpoint, params); + return { + success: true, + data: queryRes, + }; + } catch (err: any) { + this.logger.error(`Facet fetch: ${endpoint}: ${err}`); + return { + success: false, + data: err, + }; + } + }; + + public describeQuery = async (request: any): Promise => { + const response = await this.fetch(request, this.endpoint); + if (!this.shimResponse) return response; + + const { format: dataType } = request.body; + const shimFunctions: { [key: string]: (data: any) => any } = { + jdbc: (data: any) => shimSchemaRow(data as IPPLEventsDataSource), + viz: (data: any) => shimStats(data as IPPLVisualizationDataSource), + }; + + return shimFunctions[dataType] + ? { ...response, data: shimFunctions[dataType](response.data) } + : response; + }; +} diff --git a/server/utils/index.ts b/server/utils/index.ts new file mode 100644 index 0000000..2486049 --- /dev/null +++ b/server/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { Facet } from './facet'; +export { OpenSearchPPLPlugin, OpenSearchObservabilityPlugin } from './plugins'; +export { shimStats } from './shim_stats'; +export { shimSchemaRow } from './shim_schema_row'; diff --git a/server/utils/plugins.ts b/server/utils/plugins.ts new file mode 100644 index 0000000..3f786dd --- /dev/null +++ b/server/utils/plugins.ts @@ -0,0 +1,159 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { OPENSEARCH_API, URI } from '../../common'; + +const createAction = ( + client: any, + components: any, + options: { + endpoint: string; + method: 'GET' | 'POST' | 'DELETE' | 'PATCH' | 'PUT'; + needBody?: boolean; + paramKey?: string; + params?: any; + } +) => { + const { endpoint, method, needBody = false, paramKey, params } = options; + let urlConfig; + + if (paramKey) { + urlConfig = { + fmt: `${endpoint}/<%=${paramKey}%>`, + req: { + [paramKey]: { + type: 'string', + required: true, + }, + }, + }; + } else if (params) { + urlConfig = { + fmt: endpoint, + params, + }; + } else { + urlConfig = { fmt: endpoint }; + } + + return components.clientAction.factory({ + url: urlConfig, + needBody, + method, + }); +}; + +export const OpenSearchPPLPlugin = (client: any, config: any, components: any) => { + client.prototype.ppl = components.clientAction.namespaceFactory(); + const ppl = client.prototype.ppl.prototype; + + ppl.pplQuery = createAction(client, components, { + endpoint: URI.PPL, + method: 'POST', + needBody: true, + }); + ppl.sqlQuery = createAction(client, components, { + endpoint: URI.SQL, + method: 'POST', + needBody: true, + }); + ppl.getDataConnectionById = createAction(client, components, { + endpoint: OPENSEARCH_API.DATA_CONNECTIONS, + method: 'GET', + paramKey: 'dataconnection', + }); + ppl.deleteDataConnection = createAction(client, components, { + endpoint: OPENSEARCH_API.DATA_CONNECTIONS, + method: 'DELETE', + paramKey: 'dataconnection', + }); + ppl.createDataSource = createAction(client, components, { + endpoint: OPENSEARCH_API.DATA_CONNECTIONS, + method: 'POST', + needBody: true, + }); + ppl.modifyDataConnection = createAction(client, components, { + endpoint: OPENSEARCH_API.DATA_CONNECTIONS, + method: 'PATCH', + needBody: true, + }); + ppl.getDataConnections = createAction(client, components, { + endpoint: OPENSEARCH_API.DATA_CONNECTIONS, + method: 'GET', + }); +}; + +export const OpenSearchObservabilityPlugin = (client: any, config: any, components: any) => { + client.prototype.observability = components.clientAction.namespaceFactory(); + const observability = client.prototype.observability.prototype; + + observability.getObject = createAction(client, components, { + endpoint: OPENSEARCH_API.PANELS, + method: 'GET', + params: { + objectId: { type: 'string' }, + objectIdList: { type: 'string' }, + objectType: { type: 'string' }, + sortField: { type: 'string' }, + sortOrder: { type: 'string' }, + fromIndex: { type: 'number' }, + maxItems: { type: 'number' }, + name: { type: 'string' }, + lastUpdatedTimeMs: { type: 'string' }, + createdTimeMs: { type: 'string' }, + }, + }); + + observability.getObjectById = createAction(client, components, { + endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, + method: 'GET', + paramKey: 'objectId', + }); + + observability.createObject = createAction(client, components, { + endpoint: OPENSEARCH_API.PANELS, + method: 'POST', + needBody: true, + }); + + observability.updateObjectById = createAction(client, components, { + endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, + method: 'PUT', + paramKey: 'objectId', + needBody: true, + }); + + observability.deleteObjectById = createAction(client, components, { + endpoint: `${OPENSEARCH_API.PANELS}/<%=objectId%>`, + method: 'DELETE', + paramKey: 'objectId', + }); + + observability.deleteObjectByIdList = createAction(client, components, { + endpoint: OPENSEARCH_API.PANELS, + method: 'DELETE', + params: { + objectIdList: { type: 'string', required: true }, + }, + }); + + observability.getJobStatus = createAction(client, components, { + endpoint: `${URI.ASYNC_QUERY}/<%=queryId%>`, + method: 'GET', + paramKey: 'queryId', + }); + + observability.deleteJob = createAction(client, components, { + endpoint: `${URI.ASYNC_QUERY}/<%=queryId%>`, + method: 'DELETE', + paramKey: 'queryId', + }); + + observability.runDirectQuery = createAction(client, components, { + endpoint: URI.ASYNC_QUERY, + method: 'POST', + needBody: true, + }); +}; diff --git a/server/utils/shim_schema_row.ts b/server/utils/shim_schema_row.ts new file mode 100644 index 0000000..a38d775 --- /dev/null +++ b/server/utils/shim_schema_row.ts @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IPPLEventsDataSource } from '../types'; + +export function shimSchemaRow(response: IPPLEventsDataSource) { + const schemaLength = response.schema.length; + + const data = response.datarows.map((row) => { + return row.reduce((record: any, item: any, index: number) => { + if (index < schemaLength) { + const cur = response.schema[index]; + const value = + typeof item === 'object' + ? JSON.stringify(item) + : typeof item === 'boolean' + ? item.toString() + : item; + record[cur.name] = value; + } + return record; + }, {}); + }); + + return { ...response, jsonData: data }; +} diff --git a/server/utils/shim_stats.ts b/server/utils/shim_stats.ts new file mode 100644 index 0000000..e616996 --- /dev/null +++ b/server/utils/shim_stats.ts @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IPPLVisualizationDataSource } from '../types'; + +/** + * Add vis mapping for runtime fields + * json data structure added to response will be + * [{ + * agent: "mozilla", + * avg(bytes): 5756 + * ... + * }, { + * agent: "MSIE", + * avg(bytes): 5605 + * ... + * }, { + * agent: "chrome", + * avg(bytes): 5648 + * ... + * }] + * + * @internal + */ +export function shimStats(response: IPPLVisualizationDataSource) { + if (!response?.metadata?.fields || !response?.data) { + return { ...response }; + } + + const { + data: statsDataSet, + metadata: { fields: queriedFields }, + size, + } = response; + const data = new Array(size).fill(null).map((_, i) => { + const entry: Record = {}; + queriedFields.forEach(({ name }: { name: string }) => { + if (statsDataSet[name] && i < statsDataSet[name].length) { + entry[name] = statsDataSet[name][i]; + } + }); + return entry; + }); + + return { ...response, jsonData: data }; +} diff --git a/tsconfig.json b/tsconfig.json index 3066fa1..c55d1ac 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../tsconfig.json", "compilerOptions": { "outDir": "./target", - "skipLibCheck": true + "skipLibCheck": true, }, "include": [ "index.ts", From e11a3841e869fbb5a133a845354f64b2eebc0053 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Fri, 14 Jun 2024 08:18:45 +0000 Subject: [PATCH 2/2] final touches Signed-off-by: Kawika Avilla --- opensearch_dashboards.json | 4 ++-- server/search/ppl_search_strategy.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index 9512432..710f289 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -1,10 +1,10 @@ { "id": "queryEnhancements", - "version": "3.0.0.0", + "version": "3.0.0", "opensearchDashboardsVersion": "opensearchDashboards", "server": true, "ui": true, "requiredPlugins": ["data"], "optionalPlugins": ["dataSource", "dataSourceManagement"], - "requiredBundles": [] + "requiredBundles": ["opensearchDashboardsUtils"] } diff --git a/server/search/ppl_search_strategy.ts b/server/search/ppl_search_strategy.ts index 4d6cee1..1f4dbb9 100644 --- a/server/search/ppl_search_strategy.ts +++ b/server/search/ppl_search_strategy.ts @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + import { first } from 'rxjs/operators'; import { SharedGlobalConfig, Logger, ILegacyClusterClient } from 'opensearch-dashboards/server'; import { Observable } from 'rxjs';