diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index afcec55107ff9..7f69a2be14404 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -49,6 +49,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { createdPipelineId = pipelineId; } else { createdIndex = index; + createdPipelineId = pipelineId; } let failures: ImportFailure[] = []; diff --git a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts index 9b4176d070c63..cdafe6b5f8a63 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts @@ -85,6 +85,24 @@ export default function ({ getService }: FtrProviderContext) { docCountFormatted: '19 (100%)', }, ], + allFields: [ + '@timestamp', + 'http', + 'http.request', + 'http.request.method', + 'http.response', + 'http.response.body', + 'http.response.body.bytes', + 'http.response.status_code', + 'http.version', + 'message', + 'source', + 'source.address', + 'url', + 'url.original', + 'user_agent', + 'user_agent.original', + ], visibleMetricFieldsCount: 3, totalMetricFieldsCount: 3, populatedFieldsCount: 9, @@ -127,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 7, }, ], + allFields: ['Coordinates', 'Location'], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -171,6 +190,7 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 3, }, ], + allFields: ['description', 'title', 'value'], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -207,6 +227,41 @@ export default function ({ getService }: FtrProviderContext) { exampleCount: 11, }, ], + allFields: [ + 'AvgTicketPrice', + 'Cancelled', + 'Carrier', + 'Dest', + 'DestAirportID', + 'DestCityName', + 'DestCountry', + 'DestLocation', + 'DestLocation.lat', + 'DestLocation.lat.keyword', + 'DestLocation.lon', + 'DestLocation.lon.keyword', + 'DestRegion', + 'DestWeather', + 'DistanceKilometers', + 'FlightDelayMin', + 'FlightDelayType', + 'FlightNum', + 'FlightTimeHour', + 'FlightTimeMin', + 'Origin', + 'OriginAirportID', + 'OriginCityName', + 'OriginCountry', + 'OriginLocation', + 'OriginLocation.lat', + 'OriginLocation.lat.keyword', + 'OriginLocation.lon', + 'OriginLocation.lon.keyword', + 'OriginRegion', + 'OriginWeather', + 'dayOfWeek', + 'timestamp', + ], visibleMetricFieldsCount: 0, totalMetricFieldsCount: 0, populatedFieldsCount: 3, @@ -349,6 +404,16 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('closes filebeat config'); await ml.dataVisualizerFileBased.closeCreateFilebeatConfig(); + + await ml.dataVisualizerFileBased.assertDocCountInIndex( + testData.indexName, + testData.expected.ingestedDocCount + ); + + await ml.dataVisualizerFileBased.assertFieldsFromIndex( + testData.indexName, + testData.expected.allFields + ); }); }); } diff --git a/x-pack/test/functional/services/ml/data_visualizer_file_based.ts b/x-pack/test/functional/services/ml/data_visualizer_file_based.ts index df95eddd957f8..d9d59e000d805 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_file_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_file_based.ts @@ -16,6 +16,7 @@ export function MachineLearningDataVisualizerFileBasedProvider( { getService, getPageObjects }: FtrProviderContext, mlCommonUI: MlCommonUI ) { + const es = getService('es'); const log = getService('log'); const retry = getService('retry'); const testSubjects = getService('testSubjects'); @@ -177,5 +178,46 @@ export function MachineLearningDataVisualizerFileBasedProvider( await testSubjects.click('fileBeatConfigFlyoutCloseButton'); await testSubjects.missingOrFail('fileDataVisFilebeatConfigPanel'); }, + + async assertDocCountInIndex(index: string, expectedCount: number) { + await retry.tryForTime(60 * 1000, async () => { + const count = await this.getDocCountFromIndex(index); + expect(count).to.eql( + expectedCount, + `Expected document count in index '${index}' to be '${expectedCount}' (got '${count}')` + ); + }); + }, + + async getDocCountFromIndex(index: string) { + const resp = await es.search({ + index, + body: { + size: 0, + query: { + match_all: {}, + }, + }, + }); + // @ts-expect-error incorrect type definition + return resp.hits.total?.value; + }, + + async assertFieldsFromIndex(index: string, fields: string[]) { + await retry.tryForTime(60 * 1000, async () => { + const sortedFields = fields.sort(); + const fieldCaps = await es.fieldCaps({ + index, + fields: '*', + filters: '-metadata', + include_empty_fields: false, + }); + const fieldsFromIndex = Object.keys(fieldCaps.fields).sort(); + expect(fieldsFromIndex).to.eql( + sortedFields, + `Expected fields to be ${sortedFields} (got ${fieldsFromIndex})` + ); + }); + }, }; }