Skip to content

Commit

Permalink
[8.7] [ML] Fixing ML saved object cache (#151122) (#151589)
Browse files Browse the repository at this point in the history 
# Backport

This will backport the following commits from `main` to `8.7`:
- [[ML] Fixing ML saved object cache
(#151122)](#151122)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"James
Gowdy","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-02-20T09:41:14Z","message":"[ML]
Fixing ML saved object cache (#151122)\n\nPR
#146155 introduced a cache
of\r\nthe ML saved objects to the `mlSavedObjectService` to
improve\r\nperformance.\r\n\r\nThis can cause a problem for the module
`setup` function when called\r\nmore than once from an external plugin
in a single request.\r\nA single instance of the `mlSavedObjectService`
is used per request and\r\nso for each `setup` call the same cache is
used. Any saved objects\r\ncreated, updated or removed in the first
`setup` call are missing from\r\nthe cache in subsequent
calls.\r\n\r\nThis means any jobs being created in the second call to
`setup` cannot\r\nbe opened as do not exist in the cache.\r\n\r\nThis PR
clears the cache after every write action to the saved object\r\nclient
causing it to be repopulated the next time it is
read.","sha":"f6bb0f4fccab76791ef292ebd90df581c3fbdac6","branchLabelMapping":{"^v8.8.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","non-issue",":ml","Feature:Anomaly
Detection","release_note:skip","v8.7.0","v8.8.0"],"number":151122,"url":"https://github.com/elastic/kibana/pull/151122","mergeCommit":{"message":"[ML]
Fixing ML saved object cache (#151122)\n\nPR
#146155 introduced a cache
of\r\nthe ML saved objects to the `mlSavedObjectService` to
improve\r\nperformance.\r\n\r\nThis can cause a problem for the module
`setup` function when called\r\nmore than once from an external plugin
in a single request.\r\nA single instance of the `mlSavedObjectService`
is used per request and\r\nso for each `setup` call the same cache is
used. Any saved objects\r\ncreated, updated or removed in the first
`setup` call are missing from\r\nthe cache in subsequent
calls.\r\n\r\nThis means any jobs being created in the second call to
`setup` cannot\r\nbe opened as do not exist in the cache.\r\n\r\nThis PR
clears the cache after every write action to the saved object\r\nclient
causing it to be repopulated the next time it is
read.","sha":"f6bb0f4fccab76791ef292ebd90df581c3fbdac6"}},"sourceBranch":"main","suggestedTargetBranches":["8.7"],"targetPullRequestStates":[{"branch":"8.7","label":"v8.7.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.8.0","labelRegex":"^v8.8.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/151122","number":151122,"mergeCommit":{"message":"[ML]
Fixing ML saved object cache (#151122)\n\nPR
#146155 introduced a cache
of\r\nthe ML saved objects to the `mlSavedObjectService` to
improve\r\nperformance.\r\n\r\nThis can cause a problem for the module
`setup` function when called\r\nmore than once from an external plugin
in a single request.\r\nA single instance of the `mlSavedObjectService`
is used per request and\r\nso for each `setup` call the same cache is
used. Any saved objects\r\ncreated, updated or removed in the first
`setup` call are missing from\r\nthe cache in subsequent
calls.\r\n\r\nThis means any jobs being created in the second call to
`setup` cannot\r\nbe opened as do not exist in the cache.\r\n\r\nThis PR
clears the cache after every write action to the saved object\r\nclient
causing it to be repopulated the next time it is
read.","sha":"f6bb0f4fccab76791ef292ebd90df581c3fbdac6"}}]}] BACKPORT-->

Co-authored-by: James Gowdy <[email protected]>
  • Loading branch information
@kibanamachine @jgowdyelastic
kibanamachine and jgowdyelastic authored Feb 20, 2023
1 parent 868f13e commit ae5666b
Showing 1 changed file with 26 additions and 2 deletions.
28 changes: 26 additions & 2 deletions x-pack/plugins/ml/server/saved_objects/service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ export function mlSavedObjectServiceFactory(
(options: SavedObjectsFindOptions) => JSON.stringify(options)
);

function _clearSavedObjectsClientCache() {
if (_savedObjectsClientFindMemo.cache.clear) {
_savedObjectsClientFindMemo.cache.clear();
}
}

async function _getJobObjects(
jobType?: JobType,
jobId?: string,
Expand Down Expand Up @@ -118,6 +124,7 @@ export function mlSavedObjectServiceFactory(
} else {
// the saved object has no spaces, this is unexpected, attempt a normal delete
await savedObjectsClient.delete(ML_JOB_SAVED_OBJECT_TYPE, id, { force: true });
_clearSavedObjectsClientCache();
}
}
} catch (error) {
Expand All @@ -128,18 +135,21 @@ export function mlSavedObjectServiceFactory(
await savedObjectsClient.create<JobObject>(ML_JOB_SAVED_OBJECT_TYPE, job, {
id,
});
_clearSavedObjectsClientCache();
}

async function _bulkCreateJobs(jobs: Array<{ job: JobObject; namespaces: string[] }>) {
await isMlReady();
return await savedObjectsClient.bulkCreate<JobObject>(
const results = await savedObjectsClient.bulkCreate<JobObject>(
jobs.map((j) => ({
type: ML_JOB_SAVED_OBJECT_TYPE,
id: _jobSavedObjectId(j.job),
attributes: j.job,
initialNamespaces: j.namespaces,
}))
);
_clearSavedObjectsClientCache();
return results;
}

function _jobSavedObjectId(job: JobObject) {
Expand All @@ -154,6 +164,7 @@ export function mlSavedObjectServiceFactory(
}

await savedObjectsClient.delete(ML_JOB_SAVED_OBJECT_TYPE, job.id, { force: true });
_clearSavedObjectsClientCache();
}

async function _forceDeleteJob(jobType: JobType, jobId: string, namespace: string) {
Expand All @@ -169,6 +180,7 @@ export function mlSavedObjectServiceFactory(
namespace: namespace === '*' ? undefined : namespace,
force: true,
});
_clearSavedObjectsClientCache();
}

async function createAnomalyDetectionJob(jobId: string, datafeedId?: string) {
Expand Down Expand Up @@ -250,6 +262,7 @@ export function mlSavedObjectServiceFactory(
const jobObject = job.attributes;
jobObject.datafeed_id = datafeedId;
await savedObjectsClient.update<JobObject>(ML_JOB_SAVED_OBJECT_TYPE, job.id, jobObject);
_clearSavedObjectsClientCache();
}

async function deleteDatafeed(datafeedId: string) {
Expand All @@ -262,6 +275,7 @@ export function mlSavedObjectServiceFactory(
const jobObject = job.attributes;
jobObject.datafeed_id = null;
await savedObjectsClient.update<JobObject>(ML_JOB_SAVED_OBJECT_TYPE, job.id, jobObject);
_clearSavedObjectsClientCache();
}

async function _getIds(jobType: JobType, idType: keyof JobObject) {
Expand Down Expand Up @@ -372,6 +386,8 @@ export function mlSavedObjectServiceFactory(
spacesToAdd,
spacesToRemove
);
_clearSavedObjectsClientCache();

updateResult.objects.forEach(({ id: objectId, error }) => {
const jobId = jobObjectIdMap.get(objectId)!;
if (error) {
Expand Down Expand Up @@ -492,6 +508,7 @@ export function mlSavedObjectServiceFactory(
await savedObjectsClient.delete(ML_TRAINED_MODEL_SAVED_OBJECT_TYPE, modelId, {
force: true,
});
_clearSavedObjectsClientCache();
}
}
} catch (error) {
Expand All @@ -518,6 +535,7 @@ export function mlSavedObjectServiceFactory(
...(initialNamespaces ? { initialNamespaces } : {}),
}
);
_clearSavedObjectsClientCache();
}

async function _bulkCreateTrainedModel(models: TrainedModelObject[], namespaceFallback?: string) {
Expand All @@ -528,7 +546,7 @@ export function mlSavedObjectServiceFactory(
return acc;
}, {} as Record<string, string[] | undefined>);

return await savedObjectsClient.bulkCreate<TrainedModelObject>(
const results = await savedObjectsClient.bulkCreate<TrainedModelObject>(
models.map((m) => {
let initialNamespaces = m.job && namespacesPerJob[m.job.job_id];
if (!initialNamespaces?.length && namespaceFallback) {
Expand All @@ -546,6 +564,8 @@ export function mlSavedObjectServiceFactory(
};
})
);
_clearSavedObjectsClientCache();
return results;
}

async function getAllTrainedModelObjectsForAllSpaces(modelIds?: string[]) {
Expand Down Expand Up @@ -577,6 +597,7 @@ export function mlSavedObjectServiceFactory(
}

await savedObjectsClient.delete(ML_TRAINED_MODEL_SAVED_OBJECT_TYPE, model.id, { force: true });
_clearSavedObjectsClientCache();
}

async function _forceDeleteTrainedModel(modelId: string, namespace: string) {
Expand All @@ -586,6 +607,7 @@ export function mlSavedObjectServiceFactory(
namespace: namespace === '*' ? undefined : namespace,
force: true,
});
_clearSavedObjectsClientCache();
}

async function filterTrainedModelsForSpace<T>(list: T[], field: keyof T): Promise<T[]> {
Expand Down Expand Up @@ -729,6 +751,8 @@ export function mlSavedObjectServiceFactory(
spacesToAdd,
spacesToRemove
);
_clearSavedObjectsClientCache();

updateResult.objects.forEach(({ id: objectId, error }) => {
const model = trainedModelObjectIdMap.get(objectId)!;
if (error) {
Expand Down

0 comments on commit ae5666b

Please sign in to comment.