From 45f437fd9155993c0d4514bd4b9a04c1563f64cc Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 11 Oct 2024 00:54:08 +0000 Subject: [PATCH] add template creation to assests creation flow Signed-off-by: Eric --- .../getting_started_collectData.tsx | 12 +++- .../getting_started/components/utils.tsx | 60 ++++++++++++++++++- .../getting_started/getting_started_router.ts | 17 +++++- server/routes/getting_started/helper.ts | 45 ++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/public/components/getting_started/components/getting_started_collectData.tsx b/public/components/getting_started/components/getting_started_collectData.tsx index 0d1de005ab..261052b89d 100644 --- a/public/components/getting_started/components/getting_started_collectData.tsx +++ b/public/components/getting_started/components/getting_started_collectData.tsx @@ -48,7 +48,7 @@ interface CollectAndShipDataProps { selectedDataSourceLabel: string; } -interface CollectorOption { +export interface CollectorOption { label: string; value: string; } @@ -280,7 +280,15 @@ export const CollectAndShipData: React.FC = ({ { - await UploadAssets(specificMethod, selectedDataSourceId, selectedDataSourceLabel); + await UploadAssets( + specificMethod, + selectedDataSourceId, + selectedDataSourceLabel, + technologyJsonMap[specificMethod]?.['getting-started']?.schema || + technologyJsonMap[specificMethod]?.schema || + [], + collectorOptions + ); }} fill > diff --git a/public/components/getting_started/components/utils.tsx b/public/components/getting_started/components/utils.tsx index 3c50f30723..21566ddc85 100644 --- a/public/components/getting_started/components/utils.tsx +++ b/public/components/getting_started/components/utils.tsx @@ -3,8 +3,27 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { MappingTypeMapping } from '@opensearch-project/opensearch/api/types'; +import { EuiSelectableOption } from '@elastic/eui'; import { coreRefs } from '../../../framework/core_refs'; import { useToast } from '../../../../public/components/common/toast'; +import { CollectorOption } from './getting_started_collectData'; + +export interface ICollectorIndexTemplate { + name: string; + templatePath: string; + template: MappingTypeMapping; +} + +export interface ICollectorSchema { + alias: string; + content: string; + description: string; + 'index-pattern-name': string; + type: string; + 'index-template': string; + info: string[]; +} const fetchAssets = async (tutorialId: string, assetFilter?: 'dashboards' | 'indexPatterns') => { const assetFilterParam = assetFilter ? `${assetFilter}/` : ''; @@ -20,16 +39,55 @@ const fetchAssets = async (tutorialId: string, assetFilter?: 'dashboards' | 'ind return responeData; }; -export const UploadAssets = async (tutorialId: string, mdsId: string, mdsLabel: string) => { +export const UploadAssets = async ( + tutorialId: string, + mdsId: string, + mdsLabel: string, + schema: ICollectorSchema[], + collectorOptions: Array> +) => { const { setToast } = useToast(); const http = coreRefs.http; + let selectedIntegration: string | undefined; + const checkedCollector = collectorOptions.find((collector) => { + return !!collector.checked; + }); + + if (checkedCollector !== undefined) { + if (checkedCollector.value === 'otel') { + selectedIntegration = 'otel-services'; + } else if (checkedCollector.value === 'nginx') { + selectedIntegration = checkedCollector.value; + } + } + try { + // Auto-generate index templates based on the selected integration + let templates: ICollectorIndexTemplate[] = []; + if (selectedIntegration !== undefined) { + const indexTemplateMappings = await http!.get( + `/api/integrations/repository/${selectedIntegration}/schema` + ); + templates = schema.reduce((acc: ICollectorIndexTemplate[], sh) => { + const templateMapping = indexTemplateMappings?.data?.mappings?.[sh.type]; + if (!!templateMapping) { + acc.push({ + name: sh.content.match(/[^/]+$/)?.[0] || '', + templatePath: sh.content.match(/PUT\s+(.+)/)?.[1] || '', + template: templateMapping, + }); + } + return acc; + }, []); + } + const response = await http!.post(`/api/observability/gettingStarted/createAssets`, { body: JSON.stringify({ mdsId, mdsLabel, tutorialId, + indexTemplates: templates, }), }); diff --git a/server/routes/getting_started/getting_started_router.ts b/server/routes/getting_started/getting_started_router.ts index 38d8744a65..97d7cc4ee1 100644 --- a/server/routes/getting_started/getting_started_router.ts +++ b/server/routes/getting_started/getting_started_router.ts @@ -12,7 +12,7 @@ import { SavedObject, } from '../../../../../src/core/server'; import { createSavedObjectsStreamFromNdJson } from '../../../../../src/core/server/saved_objects/routes/utils'; -import { loadAssetsFromFile } from './helper'; +import { loadAssetsFromFile, createAllTemplatesSettled } from './helper'; import { getWorkspaceState } from '../../../../../src/core/server/utils'; export function registerGettingStartedRoutes(router: IRouter) { @@ -140,6 +140,14 @@ export function registerGettingStartedRoutes(router: IRouter) { mdsId: schema.string(), mdsLabel: schema.string(), tutorialId: schema.string(), + indexTemplates: schema.arrayOf( + schema.object({ + name: schema.string(), + templatePath: schema.string(), + template: schema.recordOf(schema.string(), schema.any()), + }), + { defaultValue: [] } + ), }), }, }, @@ -149,10 +157,15 @@ export function registerGettingStartedRoutes(router: IRouter) { response ): Promise> => { try { - const { mdsId, mdsLabel, tutorialId } = request.body; + const { mdsId, mdsLabel, tutorialId, indexTemplates } = request.body; const { requestWorkspaceId } = getWorkspaceState(request); const fileData = await loadAssetsFromFile(tutorialId); + // create related index templates + if (indexTemplates.length > 0) { + await createAllTemplatesSettled(context, indexTemplates, mdsId); + } + const objects = await createSavedObjectsStreamFromNdJson(Readable.from(fileData)); const loadedObjects = await objects.toArray(); diff --git a/server/routes/getting_started/helper.ts b/server/routes/getting_started/helper.ts index 82f3c8f92a..9723d7c8ce 100644 --- a/server/routes/getting_started/helper.ts +++ b/server/routes/getting_started/helper.ts @@ -5,6 +5,8 @@ import fs from 'fs'; import path from 'path'; +import { MappingTypeMapping } from '@opensearch-project/opensearch/api/types'; +import { RequestHandlerContext } from '../../../../../src/core/server'; export const assetMapper = (tutorialId: string) => { switch (tutorialId) { @@ -37,3 +39,46 @@ export const loadAssetsFromFile = async (tutorialId: string) => { throw new Error('Issue is loading asset'); } }; + +export const createAllTemplatesSettled = async ( + context: RequestHandlerContext, + indexTemplates: Array<{ name: string; template: MappingTypeMapping; templatePath: string }>, + dataSourceMDSId: string +) => { + const results = await Promise.allSettled( + indexTemplates.map(({ name, template, templatePath }) => + createIndexTemplate(context, name, template, dataSourceMDSId, templatePath) + ) + ); + + return results.map((result, index) => { + const templateName = indexTemplates[index].name; + if (result.status === 'fulfilled') { + return { name: templateName, success: true }; + } + return { name: templateName, success: false, reason: result.reason }; + }); +}; + +export const createIndexTemplate = async ( + context: RequestHandlerContext, + name: string, + template: MappingTypeMapping, + dataSourceMDSId: string, + templatePath: string +) => { + try { + const osClient = dataSourceMDSId + ? await context.dataSource.opensearch.getClient(dataSourceMDSId) + : context.core.opensearch.client.asCurrentUser; + + return await osClient.transport.request({ + method: 'PUT', + path: templatePath, + body: template, + }); + } catch (error) { + console.error(`Failed to create index template ${name}:`, error); + throw error; + } +};