From 6fd19137fe1cb0293ce84f3bba8f8ae7fedd98c9 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 27 Sep 2024 18:31:47 +0100 Subject: [PATCH] [ML] Fix file upload when pipline is used (#194273) Fixes bug introduced in https://github.com/elastic/kibana/pull/193744 If a file is uploaded with an ingest pipeline, the pipeline is created, but not used when uploading the data. Adds tests to check that the data exists in the index and the expected fields exist and are populated. --- .../plugins/file_upload/server/import_data.ts | 1 + .../data_visualizer/file_data_visualizer.ts | 65 +++++++++++++++++++ .../services/ml/data_visualizer_file_based.ts | 42 ++++++++++++ 3 files changed, 108 insertions(+) diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index afcec55107ff9..7f69a2be14404 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -49,6 +49,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { createdPipelineId = pipelineId; } else { createdIndex = index; + createdPipelineId = pipelineId; } let failures: ImportFailure[] = []; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts index 9b4176d070c63..cdafe6b5f8a63 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts @@ -85,6 +85,24 @@ export default function ({ getService }: FtrProviderContext) { docCountFormatted: '19 (100%)', }, ], + allFields: [ + '@timestamp', + 'http', + 'http.request', + 'http.request.method', + 'http.response', + 'http.response.body', + 'http.response.body.bytes', + 'http.response.status_code', + 'http.version', + 'message', + 'source', + 'source.address', + 'url', + 'url.original', + 'user_agent', + 'user_agent.original', + ], visibleMetricFieldsCount: 3, totalMetricFieldsCount: 3, populatedFieldsCount: 9, @@ -127,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 7, }, ], + allFields: ['Coordinates', 'Location'], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -171,6 +190,7 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 3, }, ], + allFields: ['description', 'title', 'value'], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -207,6 +227,41 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 11, }, ], + allFields: [ + 'AvgTicketPrice', + 'Cancelled', + 'Carrier', + 'Dest', + 'DestAirportID', + 'DestCityName', + 'DestCountry', + 'DestLocation', + 'DestLocation.lat', + 'DestLocation.lat.keyword', + 'DestLocation.lon', + 'DestLocation.lon.keyword', + 'DestRegion', + 'DestWeather', + 'DistanceKilometers', + 'FlightDelayMin', + 'FlightDelayType', + 'FlightNum', + 'FlightTimeHour', + 'FlightTimeMin', + 'Origin', + 'OriginAirportID', + 'OriginCityName', + 'OriginCountry', + 'OriginLocation', + 'OriginLocation.lat', + 'OriginLocation.lat.keyword', + 'OriginLocation.lon', + 'OriginLocation.lon.keyword', + 'OriginRegion', + 'OriginWeather', + 'dayOfWeek', + 'timestamp', + ], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -349,6 +404,16 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('closes filebeat config'); await ml.dataVisualizerFileBased.closeCreateFilebeatConfig(); + + await ml.dataVisualizerFileBased.assertDocCountInIndex( + testData.indexName, + testData.expected.ingestedDocCount + ); + + await ml.dataVisualizerFileBased.assertFieldsFromIndex( + testData.indexName, + testData.expected.allFields + ); }); }); } diff --git a/x-pack/test/functional/services/ml/data_visualizer_file_based.ts b/x-pack/test/functional/services/ml/data_visualizer_file_based.ts index df95eddd957f8..d9d59e000d805 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_file_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_file_based.ts @@ -16,6 +16,7 @@ export function MachineLearningDataVisualizerFileBasedProvider( { getService, getPageObjects }: FtrProviderContext, mlCommonUI: MlCommonUI ) { + const es = getService('es'); const log = getService('log'); const retry = getService('retry'); const testSubjects = getService('testSubjects'); @@ -177,5 +178,46 @@ export function MachineLearningDataVisualizerFileBasedProvider( await testSubjects.click('fileBeatConfigFlyoutCloseButton'); await testSubjects.missingOrFail('fileDataVisFilebeatConfigPanel'); }, + + async assertDocCountInIndex(index: string, expectedCount: number) { + await retry.tryForTime(60 * 1000, async () => { + const count = await this.getDocCountFromIndex(index); + expect(count).to.eql( + expectedCount, + `Expected document count in index '${index}' to be '${expectedCount}' (got '${count}')` + ); + }); + }, + + async getDocCountFromIndex(index: string) { + const resp = await es.search({ + index, + body: { + size: 0, + query: { + match_all: {}, + }, + }, + }); + // @ts-expect-error incorrect type definition + return resp.hits.total?.value; + }, + + async assertFieldsFromIndex(index: string, fields: string[]) { + await retry.tryForTime(60 * 1000, async () => { + const sortedFields = fields.sort(); + const fieldCaps = await es.fieldCaps({ + index, + fields: '*', + filters: '-metadata', + include_empty_fields: false, + }); + const fieldsFromIndex = Object.keys(fieldCaps.fields).sort(); + expect(fieldsFromIndex).to.eql( + sortedFields, + `Expected fields to be ${sortedFields} (got ${fieldsFromIndex})` + ); + }); + }, }; }