From 5c1344e6a48bfa242539a6fc4492631e0ac00b2b Mon Sep 17 00:00:00 2001 From: nreese Date: Tue, 24 Jan 2017 16:41:05 -0700 Subject: [PATCH 1/5] defer loading visualization saved objects so they can be loaded in a single _mget --- .../kibana/public/dashboard/panel/load_saved_object.js | 3 ++- src/ui/public/courier/saved_object/saved_object.js | 10 ++++++++-- .../public/courier/saved_object/saved_object_loader.js | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js b/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js index d9df46dbdcd37..99224321fdf3c 100644 --- a/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js +++ b/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js @@ -10,6 +10,7 @@ export function loadSavedObject(loaders, panel) { if (!loader) { throw new Error(`No loader for object of type ${panel.type}`); } - return loader.get(panel.id) + const isDefered = true; + return loader.get(panel.id, isDefered) .then(savedObj => ({ savedObj, editUrl: loader.urlFor(panel.id) })); } diff --git a/src/ui/public/courier/saved_object/saved_object.js b/src/ui/public/courier/saved_object/saved_object.js index 8ccf2df854365..16b07ca379079 100644 --- a/src/ui/public/courier/saved_object/saved_object.js +++ b/src/ui/public/courier/saved_object/saved_object.js @@ -127,7 +127,7 @@ export default function SavedObjectFactory(esAdmin, kbnIndex, Promise, Private, * @return {Promise} * @resolved {SavedObject} */ - this.init = _.once(() => { + this.init = _.once((isDefered) => { // ensure that the type is defined if (!type) throw new Error('You must define a type name to use SavedObject objects.'); @@ -166,7 +166,13 @@ export default function SavedObjectFactory(esAdmin, kbnIndex, Promise, Private, } // fetch the object from ES - return docSource.fetch().then(this.applyESResp); + if (isDefered) { + const defer = Promise.defer(); + docSource._createRequest(defer); + return defer.promise.then(this.applyESResp); + } else { + return docSource.fetch().then(this.applyESResp); + } }) .then(() => { return customInit.call(this); diff --git a/src/ui/public/courier/saved_object/saved_object_loader.js b/src/ui/public/courier/saved_object/saved_object_loader.js index fee5279a36258..7b947826967ab 100644 --- a/src/ui/public/courier/saved_object/saved_object_loader.js +++ b/src/ui/public/courier/saved_object/saved_object_loader.js @@ -29,8 +29,8 @@ export class SavedObjectLoader { * @param id * @returns {Promise} */ - get(id) { - return (new this.Class(id)).init(); + get(id, isDefered) { + return (new this.Class(id)).init(isDefered); } urlFor(id) { From 9a02e5b34903e759527e031ce21f0e780488394b Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Thu, 26 Jan 2017 10:57:34 -0500 Subject: [PATCH 2/5] Don't request field stats more than once for the same index pattern --- src/ui/public/courier/fetch/strategy/search.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ui/public/courier/fetch/strategy/search.js b/src/ui/public/courier/fetch/strategy/search.js index e1b3cd6cd7d1f..dd7d7ba47068a 100644 --- a/src/ui/public/courier/fetch/strategy/search.js +++ b/src/ui/public/courier/fetch/strategy/search.js @@ -15,6 +15,8 @@ export default function FetchStrategyForSearch(Private, Promise, timefilter, kbn * @return {Promise} - a promise that is fulfilled by the request body */ reqsFetchParamsToBody: function (reqsFetchParams) { + const indexToListMapping = {}; + return Promise.map(reqsFetchParams, function (fetchParams) { return Promise.resolve(fetchParams.index) .then(function (indexList) { @@ -23,7 +25,11 @@ export default function FetchStrategyForSearch(Private, Promise, timefilter, kbn } const timeBounds = timefilter.getBounds(); - return indexList.toIndexList(timeBounds.min, timeBounds.max); + + if (!indexToListMapping[indexList.id]) { + indexToListMapping[indexList.id] = indexList.toIndexList(timeBounds.min, timeBounds.max); + } + return indexToListMapping[indexList.id]; }) .then(function (indexList) { let body = fetchParams.body || {}; From 20d55fe60237459eb40aa7ab8e9d15b6c575c3ea Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 6 Feb 2017 15:13:23 -0700 Subject: [PATCH 3/5] [ui/courier] batch fetch requests for all searches and docs --- .../dashboard/panel/load_saved_object.js | 3 +- src/ui/public/courier/fetch/fetch.js | 29 ++++++++++------ .../public/courier/fetch/request/request.js | 34 +++++++++++++++++-- .../courier/saved_object/saved_object.js | 10 ++---- .../saved_object/saved_object_loader.js | 4 +-- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js b/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js index 99224321fdf3c..d9df46dbdcd37 100644 --- a/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js +++ b/src/core_plugins/kibana/public/dashboard/panel/load_saved_object.js @@ -10,7 +10,6 @@ export function loadSavedObject(loaders, panel) { if (!loader) { throw new Error(`No loader for object of type ${panel.type}`); } - const isDefered = true; - return loader.get(panel.id, isDefered) + return loader.get(panel.id) .then(savedObj => ({ savedObj, editUrl: loader.urlFor(panel.id) })); } diff --git a/src/ui/public/courier/fetch/fetch.js b/src/ui/public/courier/fetch/fetch.js index edcc1a04cb483..20f14ffdd1a17 100644 --- a/src/ui/public/courier/fetch/fetch.js +++ b/src/ui/public/courier/fetch/fetch.js @@ -8,23 +8,32 @@ import ReqStatusProvider from './req_status'; export default function fetchService(Private, Promise) { const requestQueue = Private(RequestQueueProvider); - const fetchThese = Private(FetchTheseProvider); - + const immediatelyFetchThese = Private(FetchTheseProvider); const callResponseHandlers = Private(CallResponseHandlersProvider); const INCOMPLETE = Private(ReqStatusProvider).INCOMPLETE; - function fetchQueued(strategy) { - const requests = requestQueue.getStartable(strategy); - if (!requests.length) return Promise.resolve(); - else return fetchThese(requests); - } + const debouncedFetchThese = _.debounce(() => { + const requests = requestQueue.get().filter(req => req.isFetchRequested()); + immediatelyFetchThese(requests); + }, { + wait: 10, + maxWait: 50 + }); - this.fetchQueued = fetchQueued; + const fetchTheseSoon = (requests) => { + requests.forEach(req => req._setFetchRequested()); + debouncedFetchThese(); + return Promise.all(requests.map(req => req.defer.promise)); + }; + + this.fetchQueued = (strategy) => { + return fetchTheseSoon(requestQueue.getStartable(strategy)); + }; function fetchASource(source, strategy) { const defer = Promise.defer(); - fetchThese([ + fetchTheseSoon([ source._createRequest(defer) ]); @@ -50,7 +59,7 @@ export default function fetchService(Private, Promise) { * @param {array} reqs - the requests to fetch * @async */ - this.these = fetchThese; + this.these = fetchTheseSoon; /** * Send responses to a list of requests, used when requests diff --git a/src/ui/public/courier/fetch/request/request.js b/src/ui/public/courier/fetch/request/request.js index 110d09e7564a8..2c024903e366d 100644 --- a/src/ui/public/courier/fetch/request/request.js +++ b/src/ui/public/courier/fetch/request/request.js @@ -15,12 +15,42 @@ export default function AbstractReqProvider(Private, Promise) { this.source = source; this.defer = defer || Promise.defer(); this._whenAbortedHandlers = []; - requestQueue.push(this); } + /** + * Called by the loopers to find requests that should be sent to the + * fetch() module. When a module is sent to fetch() it's _fetchRequested flag + * is set, and this consults that flag so requests are not send to fetch() + * multiple times. + * + * @return {Boolean} + */ canStart() { - return Boolean(!this.stopped && !this.source._fetchDisabled); + return !this._fetchRequested && !this.stopped && !this.source._fetchDisabled; + } + + /** + * Used to find requests that were previously sent to the fetch() module but + * have not been started yet, so they can be started. + * + * @return {Boolean} + */ + isFetchRequestedAndPending() { + return !!this._fetchRequested && !this.started; + } + + /** + * Called by the fetch() module when this request has been sent to + * be fetched. At that point the request is somewhere between `ready-to-start` + * and `started`. The fetch module then waits a short period of time to + * allow requests to build up in the request queue, and then immediately + * fetches all requests that return true from `isFetchRequested()` + * + * @return {undefined} + */ + _setFetchRequested() { + this._fetchRequested = true; } start() { diff --git a/src/ui/public/courier/saved_object/saved_object.js b/src/ui/public/courier/saved_object/saved_object.js index 16b07ca379079..8ccf2df854365 100644 --- a/src/ui/public/courier/saved_object/saved_object.js +++ b/src/ui/public/courier/saved_object/saved_object.js @@ -127,7 +127,7 @@ export default function SavedObjectFactory(esAdmin, kbnIndex, Promise, Private, * @return {Promise} * @resolved {SavedObject} */ - this.init = _.once((isDefered) => { + this.init = _.once(() => { // ensure that the type is defined if (!type) throw new Error('You must define a type name to use SavedObject objects.'); @@ -166,13 +166,7 @@ export default function SavedObjectFactory(esAdmin, kbnIndex, Promise, Private, } // fetch the object from ES - if (isDefered) { - const defer = Promise.defer(); - docSource._createRequest(defer); - return defer.promise.then(this.applyESResp); - } else { - return docSource.fetch().then(this.applyESResp); - } + return docSource.fetch().then(this.applyESResp); }) .then(() => { return customInit.call(this); diff --git a/src/ui/public/courier/saved_object/saved_object_loader.js b/src/ui/public/courier/saved_object/saved_object_loader.js index 7b947826967ab..fee5279a36258 100644 --- a/src/ui/public/courier/saved_object/saved_object_loader.js +++ b/src/ui/public/courier/saved_object/saved_object_loader.js @@ -29,8 +29,8 @@ export class SavedObjectLoader { * @param id * @returns {Promise} */ - get(id, isDefered) { - return (new this.Class(id)).init(isDefered); + get(id) { + return (new this.Class(id)).init(); } urlFor(id) { From f5bd5ca0f37c8dbbdc9e0e32ea3b3227e2b1971a Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 8 Feb 2017 14:22:32 -0700 Subject: [PATCH 4/5] [ui/courier] remove remaining mentions of req.isFetchRequested() --- src/ui/public/courier/fetch/fetch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/public/courier/fetch/fetch.js b/src/ui/public/courier/fetch/fetch.js index 20f14ffdd1a17..001e7be1847f0 100644 --- a/src/ui/public/courier/fetch/fetch.js +++ b/src/ui/public/courier/fetch/fetch.js @@ -13,7 +13,7 @@ export default function fetchService(Private, Promise) { const INCOMPLETE = Private(ReqStatusProvider).INCOMPLETE; const debouncedFetchThese = _.debounce(() => { - const requests = requestQueue.get().filter(req => req.isFetchRequested()); + const requests = requestQueue.get().filter(req => req.isFetchRequestedAndPending()); immediatelyFetchThese(requests); }, { wait: 10, From eb5446d89abe02baa83ebb9f1031d39434340e88 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 9 Feb 2017 13:48:48 -0700 Subject: [PATCH 5/5] [courier/fetch/request] remove unneceessary !! --- src/ui/public/courier/fetch/request/request.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ui/public/courier/fetch/request/request.js b/src/ui/public/courier/fetch/request/request.js index 2c024903e366d..6c9aa65c28645 100644 --- a/src/ui/public/courier/fetch/request/request.js +++ b/src/ui/public/courier/fetch/request/request.js @@ -37,7 +37,7 @@ export default function AbstractReqProvider(Private, Promise) { * @return {Boolean} */ isFetchRequestedAndPending() { - return !!this._fetchRequested && !this.started; + return this._fetchRequested && !this.started; } /** @@ -45,7 +45,7 @@ export default function AbstractReqProvider(Private, Promise) { * be fetched. At that point the request is somewhere between `ready-to-start` * and `started`. The fetch module then waits a short period of time to * allow requests to build up in the request queue, and then immediately - * fetches all requests that return true from `isFetchRequested()` + * fetches all requests that return true from `isFetchRequestedAndPending()` * * @return {undefined} */