[Enterprise Search] remove trained model names (#141713) (#141721)

Due to restrictions around Kibana spaces with trained models we do not want to return model names to the UI or include them in documents processed by ml inference. (cherry picked from commit 1bdfd0e) Co-authored-by: Rodney Norris <[email protected]>
elastic · Sep 24, 2022 · e92f098 · e92f098
1 parent 89749ae
commit e92f098
Show file tree

Hide file tree

Showing 8 changed files with 28 additions and 27 deletions.
diff --git a/x-pack/plugins/enterprise_search/common/types/pipelines.ts b/x-pack/plugins/enterprise_search/common/types/pipelines.ts
@@ -8,6 +8,5 @@
 export interface InferencePipeline {
   isDeployed: boolean;
   pipelineName: string;
-  trainedModelName: string;
   types: string[];
 }
diff --git a/...s/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx b/...s/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx
@@ -20,7 +20,6 @@ import {
   EuiPopover,
   EuiPopoverTitle,
   EuiText,
-  EuiTextColor,
   EuiTitle,
 } from '@elastic/eui';
 
@@ -35,7 +34,6 @@ import { PipelinesLogic } from './pipelines_logic';
 
 export const InferencePipelineCard: React.FC<InferencePipeline> = ({
   pipelineName,
-  trainedModelName,
   isDeployed,
   types,
 }) => {
@@ -129,9 +127,6 @@ export const InferencePipelineCard: React.FC<InferencePipeline> = ({
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiFlexGroup>
-            <EuiFlexItem>
-              <EuiTextColor color="subdued">{trainedModelName}</EuiTextColor>
-            </EuiFlexItem>
             <EuiFlexItem>
               <EuiFlexGroup gutterSize="m" justifyContent="flexEnd">
                 {isDeployed && (

diff --git a/...earch_content/components/search_index/pipelines/ml_inference_pipeline_processors_card.tsx b/...earch_content/components/search_index/pipelines/ml_inference_pipeline_processors_card.tsx
@@ -32,12 +32,7 @@ export const MlInferencePipelineProcessorsCard: React.FC = () => {
     <EuiFlexGroup direction="column" gutterSize="s">
       {inferencePipelines.map((item: InferencePipeline, index: number) => (
         <EuiFlexItem key={index}>
-          <InferencePipelineCard
-            trainedModelName={item.trainedModelName}
-            pipelineName={item.pipelineName}
-            isDeployed={item.isDeployed}
-            types={item.types}
-          />
+          <InferencePipelineCard {...item} />
         </EuiFlexItem>
       ))}
     </EuiFlexGroup>

diff --git a/...ugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.test.ts b/...ugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.test.ts
@@ -18,6 +18,7 @@ import {
   fetchMlInferencePipelineProcessorNames,
   fetchMlInferencePipelineProcessors,
   fetchPipelineProcessorInferenceData,
+  InferencePipelineData,
 } from './fetch_ml_inference_pipeline_processors';
 
 const mockGetPipeline = {
@@ -190,13 +191,11 @@ const trainedModelDataObject = {
   'trained-model-id-1': {
     isDeployed: false,
     pipelineName: 'ml-inference-pipeline-1',
-    trainedModelName: 'trained-model-id-1',
     types: ['lang_ident', 'ner'],
   },
   'trained-model-id-2': {
     isDeployed: true,
     pipelineName: 'ml-inference-pipeline-2',
-    trainedModelName: 'trained-model-id-2',
     types: ['pytorch', 'ner'],
   },
 };
@@ -391,7 +390,7 @@ describe('fetchAndAddTrainedModelData lib function', () => {
       Promise.resolve(mockGetTrainedModelStats)
     );
 
-    const pipelines: InferencePipeline[] = [
+    const pipelines: InferencePipelineData[] = [
       {
         isDeployed: false,
         pipelineName: 'ml-inference-pipeline-1',
@@ -406,7 +405,7 @@ describe('fetchAndAddTrainedModelData lib function', () => {
       },
     ];
 
-    const expected: InferencePipeline[] = [
+    const expected: InferencePipelineData[] = [
       {
         isDeployed: false,
         pipelineName: 'ml-inference-pipeline-1',
@@ -555,7 +554,6 @@ describe('fetchMlInferencePipelineProcessors lib function', () => {
         {
           isDeployed: false,
           pipelineName: 'ml-inference-pipeline-3',
-          trainedModelName: 'trained-model-id-1',
           types: ['lang_ident', 'ner'],
         },
       ];

diff --git a/...ck/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts b/...ck/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts
@@ -12,6 +12,10 @@ import { BUILT_IN_MODEL_TAG } from '@kbn/ml-plugin/common/constants/data_frame_a
 import { InferencePipeline } from '../../../common/types/pipelines';
 import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils';
 
+export type InferencePipelineData = InferencePipeline & {
+  trainedModelName: string;
+};
+
 export const fetchMlInferencePipelineProcessorNames = async (
   client: ElasticsearchClient,
   indexName: string
@@ -37,7 +41,7 @@ export const fetchMlInferencePipelineProcessorNames = async (
 export const fetchPipelineProcessorInferenceData = async (
   client: ElasticsearchClient,
   mlInferencePipelineProcessorNames: string[]
-): Promise<InferencePipeline[]> => {
+): Promise<InferencePipelineData[]> => {
   const mlInferencePipelineProcessorConfigs = await client.ingest.getPipeline({
     id: mlInferencePipelineProcessorNames.join(),
   });
@@ -62,7 +66,7 @@ export const fetchPipelineProcessorInferenceData = async (
 
       return pipelineProcessorData;
     },
-    [] as InferencePipeline[]
+    [] as InferencePipelineData[]
   );
 };
 
@@ -81,13 +85,13 @@ export const getMlModelTypesForModelConfig = (trainedModel: MlTrainedModelConfig
 export const getMlModelConfigsForModelIds = async (
   client: ElasticsearchClient,
   trainedModelNames: string[]
-): Promise<Record<string, InferencePipeline>> => {
+): Promise<Record<string, InferencePipelineData>> => {
   const [trainedModels, trainedModelsStats] = await Promise.all([
     client.ml.getTrainedModels({ model_id: trainedModelNames.join() }),
     client.ml.getTrainedModelsStats({ model_id: trainedModelNames.join() }),
   ]);
 
-  const modelConfigs: Record<string, InferencePipeline> = {};
+  const modelConfigs: Record<string, InferencePipelineData> = {};
 
   trainedModels.trained_model_configs.forEach((trainedModelData) => {
     const trainedModelName = trainedModelData.model_id;
@@ -115,8 +119,8 @@ export const getMlModelConfigsForModelIds = async (
 
 export const fetchAndAddTrainedModelData = async (
   client: ElasticsearchClient,
-  pipelineProcessorData: InferencePipeline[]
-): Promise<InferencePipeline[]> => {
+  pipelineProcessorData: InferencePipelineData[]
+): Promise<InferencePipelineData[]> => {
   const trainedModelNames = Array.from(
     new Set(pipelineProcessorData.map((pipeline) => pipeline.trainedModelName))
   );
@@ -160,5 +164,9 @@ export const fetchMlInferencePipelineProcessors = async (
   // inference processors, return early to avoid fetching all of the possible trained model data.
   if (pipelineProcessorInferenceData.length === 0) return [] as InferencePipeline[];
 
-  return await fetchAndAddTrainedModelData(client, pipelineProcessorInferenceData);
+  const pipelines = await fetchAndAddTrainedModelData(client, pipelineProcessorInferenceData);
+
+  // Due to restrictions with Kibana spaces we do not want to return the trained model name
+  // to the UI. So we remove it from the data structure here.
+  return pipelines.map(({ trainedModelName, ...pipeline }) => pipeline);
 };
diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts
@@ -39,6 +39,7 @@ describe('createIndexPipelineDefinitions util function', () => {
 });
 
 describe('formatMlPipelineBody util function', () => {
+  const pipelineName = 'ml-inference-my-ml-proc';
   const modelId = 'my-model-id';
   let modelInputField = 'my-model-input-field';
   const modelType = 'pytorch';
@@ -82,8 +83,8 @@ describe('formatMlPipelineBody util function', () => {
             field: '_source._ingest.processors',
             value: [
               {
-                model_id: modelId,
                 model_version: modelVersion,
+                pipeline: pipelineName,
                 processed_timestamp: '{{{ _ingest.timestamp }}}',
                 types: modelTypes,
               },
@@ -110,6 +111,7 @@ describe('formatMlPipelineBody util function', () => {
     };
     mockClient.ml.getTrainedModels.mockImplementation(() => Promise.resolve(mockResponse));
     const actualResult = await formatMlPipelineBody(
+      pipelineName,
       modelId,
       sourceField,
       destField,
@@ -123,6 +125,7 @@ describe('formatMlPipelineBody util function', () => {
     const mockError = new Error('No known trained model with model_id [my-model-id]');
     mockClient.ml.getTrainedModels.mockImplementation(() => Promise.reject(mockError));
     const asyncCall = formatMlPipelineBody(
+      pipelineName,
       modelId,
       sourceField,
       destField,
@@ -157,8 +160,8 @@ describe('formatMlPipelineBody util function', () => {
             field: '_source._ingest.processors',
             value: [
               {
-                model_id: modelId,
                 model_version: modelVersion,
+                pipeline: pipelineName,
                 processed_timestamp: '{{{ _ingest.timestamp }}}',
                 types: modelTypes,
               },
@@ -184,6 +187,7 @@ describe('formatMlPipelineBody util function', () => {
     };
     mockClient.ml.getTrainedModels.mockImplementation(() => Promise.resolve(mockResponse));
     const actualResult = await formatMlPipelineBody(
+      pipelineName,
       modelId,
       sourceField,
       destField,

diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts
@@ -228,6 +228,7 @@ export const createIndexPipelineDefinitions = (
  * @param esClient the Elasticsearch Client to use when retrieving model details.
  */
 export const formatMlPipelineBody = async (
+  pipelineName: string,
   modelId: string,
   sourceField: string,
   destinationField: string,
@@ -264,8 +265,8 @@ export const formatMlPipelineBody = async (
           field: '_source._ingest.processors',
           value: [
             {
-              model_id: modelId,
               model_version: modelVersion,
+              pipeline: pipelineName,
               processed_timestamp: '{{{ _ingest.timestamp }}}',
               types: modelTypes,
             },

diff --git a/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts
@@ -96,6 +96,7 @@ export const createMlInferencePipeline = async (
 
   // Generate pipeline with default processors
   const mlInferencePipeline = await formatMlPipelineBody(
+    inferencePipelineGeneratedName,
     modelId,
     sourceField,
     destinationField,