From 5548e120103a2a5b539ce8cb6693c36cbbda3fcb Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 18 Jul 2023 17:29:42 +0100 Subject: [PATCH] [ML] Fix flaky update_groups api test (#161326) Related to https://github.com/elastic/kibana/issues/161324 and https://github.com/elastic/kibana/issues/160370 Flaky test runner https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2649 I believe the problem lies with the function `cleanMLSavedObjects` only cleaning up saved objects in the default space and not in any other of the spaces which jobs or trained models may have been added to. This causes an intermittent clash where a job's saved object already exists, but is in a different space. I don't know why this doesn't fail on every run. The fix is to update `cleanMLSavedObjects` so it can take a list of additional space IDs to also clean. Any test which adds jobs or trained models to spaces other than `default` need to call this function and supply the list of space IDs it is using. I've updated every test I could find in this PR. --- .../ml/anomaly_detectors/close_with_spaces.ts | 2 +- .../anomaly_detectors/delete_with_spaces.ts | 2 +- .../ml/anomaly_detectors/open_with_spaces.ts | 2 +- .../ml/data_frame_analytics/delete_spaces.ts | 2 +- .../apis/ml/jobs/close_jobs_spaces.ts | 2 +- .../apis/ml/jobs/delete_jobs_spaces.ts | 2 +- .../ml/jobs/force_start_datafeeds_spaces.ts | 2 +- .../test/api_integration/apis/ml/jobs/jobs.ts | 4 +++ .../api_integration/apis/ml/jobs/reset.ts | 3 +- .../apis/ml/jobs/stop_datafeeds_spaces.ts | 2 +- .../apis/ml/jobs/update_groups.ts | 1 + .../apis/ml/results/get_anomaly_search.ts | 4 +-- .../remove_from_current_space.ts | 4 +-- .../ml/saved_objects/update_jobs_spaces.ts | 2 +- .../update_trained_model_spaces.ts | 2 +- .../notifications/notification_list.ts | 2 +- .../ml/stack_management_jobs/manage_spaces.ts | 2 +- .../functional/services/ml/test_resources.ts | 28 +++++++++++++------ 18 files changed, 41 insertions(+), 27 deletions(-) diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/close_with_spaces.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/close_with_spaces.ts index d4d243e13e509..b1c1f650274c5 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/close_with_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/close_with_spaces.ts @@ -50,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.closeAnomalyDetectionJob(jobIdSpace1); await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/delete_with_spaces.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/delete_with_spaces.ts index d100d6060c5d1..f9044b4863f0d 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/delete_with_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/delete_with_spaces.ts @@ -46,7 +46,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/open_with_spaces.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/open_with_spaces.ts index 8cf0ab02507a0..ae28e3d514f6e 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/open_with_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/open_with_spaces.ts @@ -49,7 +49,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.closeAnomalyDetectionJob(jobIdSpace1); await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete_spaces.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete_spaces.ts index eb3b4cb93340b..d843fecc38f00 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete_spaces.ts @@ -50,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/close_jobs_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/close_jobs_spaces.ts index 510b3990d45a0..203ba0dcec789 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/close_jobs_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/close_jobs_spaces.ts @@ -58,7 +58,7 @@ export default ({ getService }: FtrProviderContext) => { await ml.api.closeAnomalyDetectionJob(jobIdSpace1); await ml.api.closeAnomalyDetectionJob(jobIdSpace2); await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/delete_jobs_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/delete_jobs_spaces.ts index 72f87c0a56489..64ce10a3ec414 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/delete_jobs_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/delete_jobs_spaces.ts @@ -53,7 +53,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts index 132685ba96572..457e352089c07 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/force_start_datafeeds_spaces.ts @@ -87,7 +87,7 @@ export default ({ getService }: FtrProviderContext) => { await ml.api.deleteAnomalyDetectionJobES(jobIdSpace1); await ml.api.deleteAnomalyDetectionJobES(jobIdSpace2); await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/jobs.ts b/x-pack/test/api_integration/apis/ml/jobs/jobs.ts index 3929ed377a51c..a6c423bd8474d 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/jobs.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/jobs.ts @@ -16,6 +16,7 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const supertest = getService('supertestWithoutAuth'); const ml = getService('ml'); + const spacesService = getService('spaces'); const idSpace1 = 'space1'; @@ -95,6 +96,7 @@ export default ({ getService }: FtrProviderContext) => { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); await ml.testResources.setKibanaTimeZoneToUTC(); + await spacesService.create({ id: idSpace1, name: 'space_one', disabledFeatures: [] }); for (const job of testSetupJobConfigs) { await ml.api.createAnomalyDetectionJob(job); @@ -123,7 +125,9 @@ export default ({ getService }: FtrProviderContext) => { }); after(async () => { + await spacesService.delete(idSpace1); await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); }); it('returns expected list of combined jobs with stats in default space', async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/reset.ts b/x-pack/test/api_integration/apis/ml/jobs/reset.ts index 574124448b1e9..0b69097a907e9 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/reset.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/reset.ts @@ -52,8 +52,7 @@ export default ({ getService }: FtrProviderContext) => { [MULTI_METRIC_JOB_CONFIG.job_id]: { reset: true, task: 'cannot be predicted' }, }; - // Failing: See https://github.com/elastic/kibana/issues/160370 - describe.skip('reset_jobs', function () { + describe('reset_jobs', function () { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); diff --git a/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts index 9e8c342c8ced4..d42b94073d104 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/stop_datafeeds_spaces.ts @@ -90,7 +90,7 @@ export default ({ getService }: FtrProviderContext) => { await ml.api.deleteAnomalyDetectionJobES(jobIdSpace1); await ml.api.deleteAnomalyDetectionJobES(jobIdSpace2); await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/jobs/update_groups.ts b/x-pack/test/api_integration/apis/ml/jobs/update_groups.ts index 20ebb75a219e8..5507716eeae91 100644 --- a/x-pack/test/api_integration/apis/ml/jobs/update_groups.ts +++ b/x-pack/test/api_integration/apis/ml/jobs/update_groups.ts @@ -70,6 +70,7 @@ export default ({ getService }: FtrProviderContext) => { after(async () => { await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); }); it('returns expected list of groups after update', async () => { diff --git a/x-pack/test/api_integration/apis/ml/results/get_anomaly_search.ts b/x-pack/test/api_integration/apis/ml/results/get_anomaly_search.ts index 5cfe728abb8b7..9238fe5cce1c6 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_anomaly_search.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_anomaly_search.ts @@ -68,10 +68,10 @@ export default ({ getService }: FtrProviderContext) => { }); after(async () => { - await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); await spacesService.delete(idSpace1); await spacesService.delete(idSpace2); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); }); it('should see results in current space', async () => { diff --git a/x-pack/test/api_integration/apis/ml/saved_objects/remove_from_current_space.ts b/x-pack/test/api_integration/apis/ml/saved_objects/remove_from_current_space.ts index 44232ef2dd01b..9cf0f50a8cbb7 100644 --- a/x-pack/test/api_integration/apis/ml/saved_objects/remove_from_current_space.ts +++ b/x-pack/test/api_integration/apis/ml/saved_objects/remove_from_current_space.ts @@ -69,10 +69,10 @@ export default ({ getService }: FtrProviderContext) => { }); after(async () => { - await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); await spacesService.delete(idSpace1); await spacesService.delete(idSpace2); + await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); }); it('should remove AD job from current space', async () => { diff --git a/x-pack/test/api_integration/apis/ml/saved_objects/update_jobs_spaces.ts b/x-pack/test/api_integration/apis/ml/saved_objects/update_jobs_spaces.ts index cbe3c584e89ff..1b79fdccb2feb 100644 --- a/x-pack/test/api_integration/apis/ml/saved_objects/update_jobs_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/saved_objects/update_jobs_spaces.ts @@ -61,7 +61,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/api_integration/apis/ml/saved_objects/update_trained_model_spaces.ts b/x-pack/test/api_integration/apis/ml/saved_objects/update_trained_model_spaces.ts index 69f5acdbcb03d..a5a87b7015c04 100644 --- a/x-pack/test/api_integration/apis/ml/saved_objects/update_trained_model_spaces.ts +++ b/x-pack/test/api_integration/apis/ml/saved_objects/update_trained_model_spaces.ts @@ -55,7 +55,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await ml.api.cleanMlIndices(); - await ml.testResources.cleanMLSavedObjects(); + await ml.testResources.cleanMLSavedObjects([idSpace1, idSpace2]); }); after(async () => { diff --git a/x-pack/test/functional/apps/ml/short_tests/notifications/notification_list.ts b/x-pack/test/functional/apps/ml/short_tests/notifications/notification_list.ts index 37cb71ba3c042..062b2809c375d 100644 --- a/x-pack/test/functional/apps/ml/short_tests/notifications/notification_list.ts +++ b/x-pack/test/functional/apps/ml/short_tests/notifications/notification_list.ts @@ -52,8 +52,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await ml.api.deleteAnomalyDetectionJobES(jobId); } await spacesService.delete(idSpace1); - await ml.testResources.cleanMLSavedObjects(); await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects(); await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); }); diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/manage_spaces.ts b/x-pack/test/functional/apps/ml/stack_management_jobs/manage_spaces.ts index 03f235dcd814a..55b2bdfa12063 100644 --- a/x-pack/test/functional/apps/ml/stack_management_jobs/manage_spaces.ts +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/manage_spaces.ts @@ -135,8 +135,8 @@ export default function ({ getService }: FtrProviderContext) { await spacesService.delete(spaceId); } } - await ml.testResources.cleanMLSavedObjects(); await ml.api.cleanMlIndices(); + await ml.testResources.cleanMLSavedObjects([spaceIds.idSpace1]); await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); await ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier'); }); diff --git a/x-pack/test/functional/services/ml/test_resources.ts b/x-pack/test/functional/services/ml/test_resources.ts index e8fb9b4e8a152..bf5f6aed44789 100644 --- a/x-pack/test/functional/services/ml/test_resources.ts +++ b/x-pack/test/functional/services/ml/test_resources.ts @@ -101,12 +101,14 @@ export function MachineLearningTestResourcesProvider( log.debug(` > Not found`); }, - async getSavedObjectIdsByType(objectType: SavedObjectType): Promise { + async getSavedObjectIdsByType(objectType: SavedObjectType, space?: string): Promise { const savedObjectIds: string[] = []; log.debug(`Searching for '${objectType}' ...`); const { body: findResponse, status } = await supertest - .get(`/api/saved_objects/_find?type=${objectType}&per_page=10000`) + .get( + `${space ? `/s/${space}` : ''}/api/saved_objects/_find?type=${objectType}&per_page=10000` + ) .set(getCommonRequestHeader('1')); mlApi.assertResponseStatusCode(200, status, findResponse); @@ -514,17 +516,23 @@ export function MachineLearningTestResourcesProvider( await this.assertSavedObjectExistsById(id, SavedObjectType.DASHBOARD); }, - async deleteMlSavedObjectByJobId(jobId: string, jobType: JobType) { + async deleteMlSavedObjectByJobId(jobId: string, jobType: JobType, space?: string) { const savedObjectId = `${jobType}-${jobId}`; - await this.deleteSavedObjectById(savedObjectId, SavedObjectType.ML_JOB, true); + await this.deleteSavedObjectById(savedObjectId, SavedObjectType.ML_JOB, true, space); }, - async cleanMLSavedObjects() { + async cleanMLSavedObjects(additionalSpaces: string[] = []) { + // clean default space await this.cleanMLJobSavedObjects(); await this.cleanMLTrainedModelsSavedObjects(); + + for (const space of additionalSpaces) { + await this.cleanMLJobSavedObjects(space); + await this.cleanMLTrainedModelsSavedObjects(space); + } }, - async cleanMLJobSavedObjects() { + async cleanMLJobSavedObjects(space?: string) { log.debug('Deleting ML job saved objects ...'); const savedObjectIds = await this.getSavedObjectIdsByType(SavedObjectType.ML_JOB); for (const id of savedObjectIds) { @@ -533,10 +541,11 @@ export function MachineLearningTestResourcesProvider( log.debug('> ML job saved objects deleted.'); }, - async cleanMLTrainedModelsSavedObjects() { + async cleanMLTrainedModelsSavedObjects(space?: string) { log.debug('Deleting ML trained model saved objects ...'); const savedObjectIds = await this.getSavedObjectIdsByType( - SavedObjectType.ML_TRAINED_MODEL_SAVED_OBJECT_TYPE + SavedObjectType.ML_TRAINED_MODEL_SAVED_OBJECT_TYPE, + space ); for (const id of savedObjectIds) { if (mlApi.isInternalModelId(id)) { @@ -546,7 +555,8 @@ export function MachineLearningTestResourcesProvider( await this.deleteSavedObjectById( id, SavedObjectType.ML_TRAINED_MODEL_SAVED_OBJECT_TYPE, - true + true, + space ); } log.debug('> ML trained model saved objects deleted.');