From 2ac88caf2138692fc22c3fddd8aefdfeccebac94 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Mon, 20 May 2019 16:39:46 -0400 Subject: [PATCH] [Monitoring/Telemetry] Force collectors to indicate when they are ready (#36153) (#36706) * Initial code to force collectors to indicate when they are ready * Add and fix tests * Remove debug * Add ready check in api call * Fix prettier complaints * Return 503 if not all collectors are ready * PR feedback * Add retry logic for usage collection in the reporting tests * Fix incorrect boomify usage * Fix more issues with the tests * Just add debug I guess * More debug * Try and handle this exception * Try and make the tests more defensive and remove console logs * Retry logic here too * Debug for the reporting tests failure * I don't like this, but lets see if it works * Move the retry logic into the collector set directly * Add support for this new collector * Localize this * This shouldn't be static on the class, but rather static for the entire runtime --- .../make_kql_usage_collector.js | 1 + .../ui_metric/server/usage/collector.ts | 1 + src/legacy/server/config/schema.js | 4 + .../server/sample_data/usage/collector.ts | 1 + .../collectors/get_ops_stats_collector.js | 1 + .../status/routes/api/register_stats.js | 18 ++++- src/legacy/server/usage/classes/collector.js | 9 ++- .../server/usage/classes/collector_set.js | 53 ++++++++++++- src/legacy/server/usage/index.js | 4 +- .../apm_telemetry/make_apm_usage_collector.ts | 3 +- .../plugins/canvas/server/usage/collector.js | 1 + .../cloud/get_cloud_usage_collector.ts | 1 + .../infra/server/usage/usage_collector.ts | 1 + .../maps_telemetry/maps_usage_collector.js | 78 ++++++++++++------- .../ml_telemetry/make_ml_usage_collector.ts | 1 + .../__tests__/bulk_uploader.js | 57 ++++++++++++++ .../server/kibana_monitoring/bulk_uploader.js | 11 ++- .../collectors/get_kibana_usage_collector.js | 2 +- .../collectors/get_ops_stats_collector.js | 1 + .../collectors/get_settings_collector.js | 1 + .../collectors/ops_buffer/event_roller.js | 4 + .../collectors/ops_buffer/ops_buffer.js | 4 + .../visualizations/get_usage_collector.ts | 65 +++++++++++----- x-pack/plugins/reporting/index.js | 9 ++- .../usage/get_reporting_usage_collector.js | 4 +- .../plugins/rollup/server/usage/collector.js | 1 + .../server/lib/get_spaces_usage_collector.ts | 1 + .../server/lib/telemetry/usage_collector.ts | 1 + .../lib/get_localization_usage_collector.ts | 1 + .../usage/telemetry_usage_collector.ts | 1 + x-pack/test/functional/config.js | 1 + 31 files changed, 279 insertions(+), 62 deletions(-) diff --git a/src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js b/src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js index 534cc266d83a3..19fb64b7ecc74 100644 --- a/src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js +++ b/src/legacy/core_plugins/kibana/server/lib/kql_usage_collector/make_kql_usage_collector.js @@ -25,6 +25,7 @@ export function makeKQLUsageCollector(server) { const kqlUsageCollector = server.usage.collectorSet.makeUsageCollector({ type: 'kql', fetch, + isReady: () => true, }); server.usage.collectorSet.register(kqlUsageCollector); diff --git a/src/legacy/core_plugins/ui_metric/server/usage/collector.ts b/src/legacy/core_plugins/ui_metric/server/usage/collector.ts index 2053c6cee1dd5..bbb7b1af8e7c7 100644 --- a/src/legacy/core_plugins/ui_metric/server/usage/collector.ts +++ b/src/legacy/core_plugins/ui_metric/server/usage/collector.ts @@ -52,6 +52,7 @@ export function registerUiMetricUsageCollector(server: any) { return uiMetricsByAppName; }, + isReady: () => true, }); server.usage.collectorSet.register(collector); diff --git a/src/legacy/server/config/schema.js b/src/legacy/server/config/schema.js index 0d136d831a6b3..832e7099ff310 100644 --- a/src/legacy/server/config/schema.js +++ b/src/legacy/server/config/schema.js @@ -175,6 +175,10 @@ export default () => Joi.object({ pollInterval: Joi.number().default(1500), }).default(), + stats: Joi.object({ + maximumWaitTimeForAllCollectorsInS: Joi.number().default(60) + }).default(), + optimize: Joi.object({ enabled: Joi.boolean().default(true), bundleFilter: Joi.string().default('!tests'), diff --git a/src/legacy/server/sample_data/usage/collector.ts b/src/legacy/server/sample_data/usage/collector.ts index 4ed7487807eec..8561a6c3f1007 100644 --- a/src/legacy/server/sample_data/usage/collector.ts +++ b/src/legacy/server/sample_data/usage/collector.ts @@ -36,6 +36,7 @@ export function makeSampleDataUsageCollector(server: KbnServer) { server.usage.collectorSet.makeUsageCollector({ type: 'sample-data', fetch: fetchProvider(index), + isReady: () => true, }) ); } diff --git a/src/legacy/server/status/collectors/get_ops_stats_collector.js b/src/legacy/server/status/collectors/get_ops_stats_collector.js index 0ebb6b3539c0e..aded85384fd85 100644 --- a/src/legacy/server/status/collectors/get_ops_stats_collector.js +++ b/src/legacy/server/status/collectors/get_ops_stats_collector.js @@ -45,6 +45,7 @@ export function getOpsStatsCollector(server, kbnServer) { ...kbnServer.metrics // latest metrics captured from the ops event listener in src/legacy/server/status/index }; }, + isReady: () => true, ignoreForInternalUploader: true, // Ignore this one from internal uploader. A different stats collector is used there. }); } diff --git a/src/legacy/server/status/routes/api/register_stats.js b/src/legacy/server/status/routes/api/register_stats.js index 60545dc3b66bd..30fa2ca86940e 100644 --- a/src/legacy/server/status/routes/api/register_stats.js +++ b/src/legacy/server/status/routes/api/register_stats.js @@ -18,10 +18,15 @@ */ import Joi from 'joi'; -import { boomify } from 'boom'; +import boom from 'boom'; +import { i18n } from '@kbn/i18n'; import { wrapAuthConfig } from '../../wrap_auth_config'; import { KIBANA_STATS_TYPE } from '../../constants'; +const STATS_NOT_READY_MESSAGE = i18n.translate('server.stats.notReadyMessage', { + defaultMessage: 'Stats are not ready yet. Please try again later.', +}); + /* * API for Kibana meta info and accumulated operations stats * Including ?extended in the query string fetches Elasticsearch cluster_uuid and server.usage.collectorSet data @@ -69,6 +74,11 @@ export function registerStatsApi(kbnServer, server, config) { if (isExtended) { const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); const callCluster = (...args) => callWithRequest(req, ...args); + const collectorsReady = await collectorSet.areAllCollectorsReady(); + + if (shouldGetUsage && !collectorsReady) { + return boom.serverUnavailable(STATS_NOT_READY_MESSAGE); + } const usagePromise = shouldGetUsage ? getUsage(callCluster) : Promise.resolve(); try { @@ -77,7 +87,6 @@ export function registerStatsApi(kbnServer, server, config) { getClusterUuid(callCluster), ]); - let modifiedUsage = usage; if (isLegacy) { // In an effort to make telemetry more easily augmented, we need to ensure @@ -123,7 +132,7 @@ export function registerStatsApi(kbnServer, server, config) { }); } } catch (e) { - throw boomify(e); + throw boom.boomify(e); } } @@ -131,6 +140,9 @@ export function registerStatsApi(kbnServer, server, config) { * for health-checking Kibana and fetch does not rely on fetching data * from ES */ const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE); + if (!await kibanaStatsCollector.isReady()) { + return boom.serverUnavailable(STATS_NOT_READY_MESSAGE); + } let kibanaStats = await kibanaStatsCollector.fetch(); kibanaStats = collectorSet.toApiFieldNames(kibanaStats); diff --git a/src/legacy/server/usage/classes/collector.js b/src/legacy/server/usage/classes/collector.js index f103126740e13..40b004f51e49a 100644 --- a/src/legacy/server/usage/classes/collector.js +++ b/src/legacy/server/usage/classes/collector.js @@ -28,7 +28,7 @@ export class Collector { * @param {Function} options.formatForBulkUpload - optional * @param {Function} options.rest - optional other properties */ - constructor(server, { type, init, fetch, formatForBulkUpload = null, ...options } = {}) { + constructor(server, { type, init, fetch, formatForBulkUpload = null, isReady = null, ...options } = {}) { if (type === undefined) { throw new Error('Collector must be instantiated with a options.type string property'); } @@ -49,6 +49,9 @@ export class Collector { const defaultFormatterForBulkUpload = result => ({ type, payload: result }); this._formatForBulkUpload = formatForBulkUpload || defaultFormatterForBulkUpload; + if (typeof isReady === 'function') { + this.isReady = isReady; + } } /* @@ -69,4 +72,8 @@ export class Collector { formatForBulkUpload(result) { return this._formatForBulkUpload(result); } + + isReady() { + throw `isReady() must be implemented in ${this.type} collector`; + } } diff --git a/src/legacy/server/usage/classes/collector_set.js b/src/legacy/server/usage/classes/collector_set.js index c7140b7abb486..a6b7e69b31261 100644 --- a/src/legacy/server/usage/classes/collector_set.js +++ b/src/legacy/server/usage/classes/collector_set.js @@ -23,18 +23,19 @@ import { getCollectorLogger } from '../lib'; import { Collector } from './collector'; import { UsageCollector } from './usage_collector'; +let _waitingForAllCollectorsTimestamp = null; + /* * A collector object has types registered into it with the register(type) * function. Each type that gets registered defines how to fetch its own data * and optionally, how to combine it into a unified payload for bulk upload. */ export class CollectorSet { - /* * @param {Object} server - server object * @param {Array} collectors to initialize, usually as a result of filtering another CollectorSet instance */ - constructor(server, collectors = []) { + constructor(server, collectors = [], config = null) { this._log = getCollectorLogger(server); this._collectors = collectors; @@ -44,7 +45,9 @@ export class CollectorSet { */ this.makeStatsCollector = options => new Collector(server, options); this.makeUsageCollector = options => new UsageCollector(server, options); - this._makeCollectorSetFromArray = collectorsArray => new CollectorSet(server, collectorsArray); + this._makeCollectorSetFromArray = collectorsArray => new CollectorSet(server, collectorsArray, config); + + this._maximumWaitTimeForAllCollectorsInS = config ? config.get('stats.maximumWaitTimeForAllCollectorsInS') : 60; } /* @@ -73,6 +76,40 @@ export class CollectorSet { return x instanceof UsageCollector; } + async areAllCollectorsReady(collectorSet = this) { + if (!(collectorSet instanceof CollectorSet)) { + throw new Error(`areAllCollectorsReady method given bad collectorSet parameter: ` + typeof collectorSet); + } + + const collectorTypesNotReady = []; + let allReady = true; + await collectorSet.asyncEach(async collector => { + if (!await collector.isReady()) { + allReady = false; + collectorTypesNotReady.push(collector.type); + } + }); + + if (!allReady && this._maximumWaitTimeForAllCollectorsInS >= 0) { + const nowTimestamp = +new Date(); + _waitingForAllCollectorsTimestamp = _waitingForAllCollectorsTimestamp || nowTimestamp; + const timeWaitedInMS = nowTimestamp - _waitingForAllCollectorsTimestamp; + const timeLeftInMS = (this._maximumWaitTimeForAllCollectorsInS * 1000) - timeWaitedInMS; + if (timeLeftInMS <= 0) { + this._log.debug(`All collectors are not ready (waiting for ${collectorTypesNotReady.join(',')}) ` + + `but we have waited the required ` + + `${this._maximumWaitTimeForAllCollectorsInS}s and will return data from all collectors that are ready.`); + return true; + } else { + this._log.debug(`All collectors are not ready. Waiting for ${timeLeftInMS}ms longer.`); + } + } else { + _waitingForAllCollectorsTimestamp = null; + } + + return allReady; + } + /* * Call a bunch of fetch methods and then do them in bulk * @param {CollectorSet} collectorSet - a set of collectors to fetch. Default to all registered collectors @@ -155,4 +192,14 @@ export class CollectorSet { map(mapFn) { return this._collectors.map(mapFn); } + + some(someFn) { + return this._collectors.some(someFn); + } + + async asyncEach(eachFn) { + for (const collector of this._collectors) { + await eachFn(collector); + } + } } diff --git a/src/legacy/server/usage/index.js b/src/legacy/server/usage/index.js index 7cf9ae5b55f52..2a02070a55f95 100644 --- a/src/legacy/server/usage/index.js +++ b/src/legacy/server/usage/index.js @@ -19,8 +19,8 @@ import { CollectorSet } from './classes'; -export function usageMixin(kbnServer, server) { - const collectorSet = new CollectorSet(server); +export function usageMixin(kbnServer, server, config) { + const collectorSet = new CollectorSet(server, undefined, config); /* * expose the collector set object on the server diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts index 6ea2bdb1eb944..960b833ebb677 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts @@ -37,7 +37,8 @@ export function makeApmUsageCollector(core: CoreSetupWithUsageCollector) { } catch (err) { return createApmTelementry(); } - } + }, + isReady: () => true }); server.usage.collectorSet.register(apmUsageCollector); } diff --git a/x-pack/plugins/canvas/server/usage/collector.js b/x-pack/plugins/canvas/server/usage/collector.js index 83bfbfa80b4c2..0926f33543d8b 100644 --- a/x-pack/plugins/canvas/server/usage/collector.js +++ b/x-pack/plugins/canvas/server/usage/collector.js @@ -136,6 +136,7 @@ export function registerCanvasUsageCollector(server) { const index = server.config().get('kibana.index'); const collector = server.usage.collectorSet.makeUsageCollector({ type: CANVAS_USAGE_TYPE, + isReady: () => true, fetch: async callCluster => { const searchParams = { size: 10000, // elasticsearch index.max_result_window default value diff --git a/x-pack/plugins/cloud/get_cloud_usage_collector.ts b/x-pack/plugins/cloud/get_cloud_usage_collector.ts index 2784a85176ba6..5ce7be59a1c9c 100644 --- a/x-pack/plugins/cloud/get_cloud_usage_collector.ts +++ b/x-pack/plugins/cloud/get_cloud_usage_collector.ts @@ -37,6 +37,7 @@ export function getCloudUsageCollector(server: KibanaHapiServer) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_CLOUD_STATS_TYPE, + isReady: () => true, fetch: createCollectorFetch(server), }); } diff --git a/x-pack/plugins/infra/server/usage/usage_collector.ts b/x-pack/plugins/infra/server/usage/usage_collector.ts index b50d627d2ce09..018c903009bbe 100644 --- a/x-pack/plugins/infra/server/usage/usage_collector.ts +++ b/x-pack/plugins/infra/server/usage/usage_collector.ts @@ -22,6 +22,7 @@ export class UsageCollector { return collectorSet.makeUsageCollector({ type: KIBANA_REPORTING_TYPE, + isReady: () => true, fetch: async () => { return this.getReport(); }, diff --git a/x-pack/plugins/maps/server/maps_telemetry/maps_usage_collector.js b/x-pack/plugins/maps/server/maps_telemetry/maps_usage_collector.js index 88713a4471123..9c76be739fbe2 100644 --- a/x-pack/plugins/maps/server/maps_telemetry/maps_usage_collector.js +++ b/x-pack/plugins/maps/server/maps_telemetry/maps_usage_collector.js @@ -13,37 +13,63 @@ export function initTelemetryCollection(server) { registerMapsUsageCollector(server); } -export function buildCollectorObj(server) { - return { - type: 'maps', - fetch: async () => { - let docs; - try { - ({ docs } = await server.taskManager.fetch({ - query: { - bool: { - filter: { - term: { - _id: TASK_ID - } - } +async function isTaskManagerReady(server) { + const result = await fetch(server); + return result !== null; +} + +async function fetch(server) { + let docs; + try { + ({ docs } = await server.taskManager.fetch({ + query: { + bool: { + filter: { + term: { + _id: TASK_ID } } - })); - } catch (err) { - const errMessage = err && err.message ? err.message : err.toString(); - /* - * The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager - * has to wait for all plugins to initialize first. - * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) - */ - if (errMessage.indexOf('NotInitialized') >= 0) { - docs = {}; - } else { - throw err; } } + })); + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + /* + * The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager + * has to wait for all plugins to initialize first. + * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) + */ + if (errMessage.indexOf('NotInitialized') >= 0) { + return null; + } else { + throw err; + } + } + + return docs; +} + +export function buildCollectorObj(server) { + let isCollectorReady = false; + async function determineIfTaskManagerIsReady() { + let isReady = false; + try { + isReady = await isTaskManagerReady(server); + } catch (err) {} // eslint-disable-line + + if (isReady) { + isCollectorReady = true; + } else { + setTimeout(determineIfTaskManagerIsReady, 500); + } + } + determineIfTaskManagerIsReady(); + return { + type: 'maps', + isReady: () => isCollectorReady, + fetch: async () => { + const docs = await fetch(server); return _.get(docs, '[0].state.stats'); }, }; diff --git a/x-pack/plugins/ml/server/lib/ml_telemetry/make_ml_usage_collector.ts b/x-pack/plugins/ml/server/lib/ml_telemetry/make_ml_usage_collector.ts index fe0e11826246e..e012b7a06e91d 100644 --- a/x-pack/plugins/ml/server/lib/ml_telemetry/make_ml_usage_collector.ts +++ b/x-pack/plugins/ml/server/lib/ml_telemetry/make_ml_usage_collector.ts @@ -27,6 +27,7 @@ interface KibanaHapiServer extends Server { export function makeMlUsageCollector(server: KibanaHapiServer): void { const mlUsageCollector = server.usage.collectorSet.makeUsageCollector({ type: 'ml', + isReady: () => true, fetch: async (): Promise => { try { const savedObjectsClient = getSavedObjectsClient(server); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js index fe5a3f51dff46..5e2adbb2d65ab 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js @@ -20,6 +20,9 @@ class MockCollectorSet { isUsageCollector(x) { return !!x.isUsageCollector; } + areAllCollectorsReady() { + return this.mockCollectors.every(collector => collector.isReady()); + } getCollectorByType(type) { return this.mockCollectors.find(collector => collector.type === type) || this.mockCollectors[0]; } @@ -29,6 +32,9 @@ class MockCollectorSet { async bulkFetch() { return this.mockCollectors.map(({ fetch }) => fetch()); } + some(someFn) { + return this.mockCollectors.some(someFn); + } } describe('BulkUploader', () => { @@ -61,6 +67,7 @@ describe('BulkUploader', () => { { type: 'type_collector_test', fetch: noop, // empty payloads, + isReady: () => true, formatForBulkUpload: result => result, } ]); @@ -94,10 +101,56 @@ describe('BulkUploader', () => { }, CHECK_DELAY); }); + it('should not upload if some collectors are not ready', done => { + const collectors = new MockCollectorSet(server, [ + { + type: 'type_collector_test', + fetch: noop, // empty payloads, + isReady: () => false, + formatForBulkUpload: result => result, + }, + { + type: 'type_collector_test2', + fetch: noop, // empty payloads, + isReady: () => true, + formatForBulkUpload: result => result, + } + ]); + + const uploader = new BulkUploader(server, { + interval: FETCH_INTERVAL + }); + + uploader.start(collectors); + + // allow interval to tick a few times + setTimeout(() => { + uploader.stop(); + + const loggingCalls = server.log.getCalls(); + expect(loggingCalls.length).to.be.greaterThan(2); // should be 3-5: start, fetch, skip, fetch, skip + expect(loggingCalls[0].args).to.eql([ + ['info', 'monitoring', 'kibana-monitoring'], + 'Starting monitoring stats collection', + ]); + expect(loggingCalls[1].args).to.eql([ + ['debug', 'monitoring', 'kibana-monitoring'], + 'Skipping bulk uploading because not all collectors are ready', + ]); + expect(loggingCalls[loggingCalls.length - 1].args).to.eql([ + ['info', 'monitoring', 'kibana-monitoring'], + 'Monitoring stats collection is stopped', + ]); + + done(); + }, CHECK_DELAY); + }); + it('should run the bulk upload handler', done => { const collectors = new MockCollectorSet(server, [ { fetch: () => ({ type: 'type_collector_test', result: { testData: 12345 } }), + isReady: () => true, formatForBulkUpload: result => result } ]); @@ -135,11 +188,13 @@ describe('BulkUploader', () => { const collectors = new MockCollectorSet(server, [ { fetch: usageCollectorFetch, + isReady: () => true, formatForBulkUpload: result => result, isUsageCollector: true, }, { fetch: collectorFetch, + isReady: () => true, formatForBulkUpload: result => result, isUsageCollector: false, } @@ -166,11 +221,13 @@ describe('BulkUploader', () => { const collectors = new MockCollectorSet(server, [ { fetch: usageCollectorFetch, + isReady: () => true, formatForBulkUpload: result => result, isUsageCollector: true, }, { fetch: collectorFetch, + isReady: () => true, formatForBulkUpload: result => result, isUsageCollector: false, } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index b16ca429f3407..ea4f8e4c46144 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -69,7 +69,6 @@ export class BulkUploader { this._log.info('Starting monitoring stats collection'); const filterCollectorSet = _collectorSet => { const filterUsage = this._lastFetchUsageTime && this._lastFetchUsageTime + this._usageInterval > Date.now(); - this._lastFetchWithUsage = !filterUsage; if (!filterUsage) { this._lastFetchUsageTime = Date.now(); } @@ -123,6 +122,16 @@ export class BulkUploader { * @return {Promise} - resolves to undefined */ async _fetchAndUpload(collectorSet) { + const collectorsReady = await collectorSet.areAllCollectorsReady(); + if (!collectorsReady) { + this._log.debug('Skipping bulk uploading because not all collectors are ready'); + if (collectorSet.some(collectorSet.isUsageCollector)) { + this._lastFetchUsageTime = null; + this._log.debug('Resetting lastFetchWithUsage because not all collectors are ready'); + } + return; + } + const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser); const payload = this.toBulkUploadFormat(compact(data), collectorSet); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js index a81db5aca586a..96d0c98cf3f05 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js @@ -23,7 +23,7 @@ export function getKibanaUsageCollector(server) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_USAGE_TYPE, - + isReady: () => true, async fetch(callCluster) { const index = server.config().get('kibana.index'); const savedObjectCountSearchParams = { diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js index 0c202b1c9e365..509689df84d8c 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js @@ -80,6 +80,7 @@ export function getOpsStatsCollector(server, kbnServer) { return collectorSet.makeStatsCollector({ type: KIBANA_STATS_TYPE_MONITORING, init: opsMonitor.start, + isReady: () => buffer.hasEvents(), fetch: async () => { return await buffer.flush(); } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js index 4b9fb5a51efb3..11129fe94b579 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js @@ -86,6 +86,7 @@ export function getSettingsCollector(server) { return collectorSet.makeStatsCollector({ type: KIBANA_SETTINGS_TYPE, + isReady: () => true, async fetch(callCluster) { let kibanaSettingsData; const defaultAdminEmail = await checkForEmailValue(config, callCluster, this.log); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/event_roller.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/event_roller.js index 4f88bd0b75093..4b25fd9b706cf 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/event_roller.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/event_roller.js @@ -22,6 +22,10 @@ export class EventRoller { return get(this.rollup, path); } + hasEvents() { + return this.rollup !== null; + } + rollupEvent(event) { const heapStats = v8.getHeapStatistics(); const requests = mapRequests(event.requests); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js index b8c670edf1297..457213c784ad6 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js @@ -29,6 +29,10 @@ export function opsBuffer(server) { server.log(['debug', LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG], 'Received Kibana Ops event data'); }, + hasEvents() { + return eventRoller.hasEvents(); + }, + async flush() { let cloud; // a property that will be left out of the result if the details are undefined const cloudDetails = cloudDetector.getCloudDetails(); diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts index 159ca7eec8063..8976cffc8ea40 100644 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -8,30 +8,55 @@ import { get } from 'lodash'; import { HapiServer } from '../../../../'; import { PLUGIN_ID, VIS_TELEMETRY_TASK, VIS_USAGE_TYPE } from '../../../../constants'; -export function getUsageCollector(server: HapiServer) { +async function isTaskManagerReady(server: HapiServer) { + const result = await fetch(server); + return result !== null; +} + +async function fetch(server: HapiServer) { const { taskManager } = server; + + let docs; + try { + ({ docs } = await taskManager.fetch({ + query: { bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } } }, + })); + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + /* + The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) + */ + if (errMessage.includes('NotInitialized')) { + docs = null; + } else { + throw err; + } + } + + return docs; +} + +export function getUsageCollector(server: HapiServer) { + let isCollectorReady = false; + async function determineIfTaskManagerIsReady() { + let isReady = false; + try { + isReady = await isTaskManagerReady(server); + } catch (err) {} // eslint-disable-line + + if (isReady) { + isCollectorReady = true; + } else { + setTimeout(determineIfTaskManagerIsReady, 500); + } + } + determineIfTaskManagerIsReady(); + return { type: VIS_USAGE_TYPE, + isReady: () => isCollectorReady, fetch: async () => { - let docs; - try { - ({ docs } = await taskManager.fetch({ - query: { bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } } }, - })); - } catch (err) { - const errMessage = err && err.message ? err.message : err.toString(); - /* - * The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager - * has to wait for all plugins to initialize first. - * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) - */ - if (errMessage.includes('NotInitialized')) { - docs = {}; - } else { - throw err; - } - } - + const docs = await fetch(server); // get the accumulated state from the recurring task return get(docs, '[0].state.stats'); }, diff --git a/x-pack/plugins/reporting/index.js b/x-pack/plugins/reporting/index.js index 7e8d370aa7df7..d7837528034e0 100644 --- a/x-pack/plugins/reporting/index.js +++ b/x-pack/plugins/reporting/index.js @@ -146,6 +146,11 @@ export const reporting = (kibana) => { }, init: async function (server) { + let isCollectorReady = false; + const isReady = () => isCollectorReady; + // Register a function with server to manage the collection of usage stats + server.usage.collectorSet.register(getReportingUsageCollector(server, isReady)); + const exportTypesRegistry = await exportTypesRegistryFactory(server); const browserFactory = await createBrowserDriverFactory(server); server.expose('exportTypesRegistry', exportTypesRegistry); @@ -167,8 +172,8 @@ export const reporting = (kibana) => { xpackMainPlugin.info.feature(this.id).registerLicenseCheckResultsGenerator(checkLicense); }); - // Register a function with server to manage the collection of usage stats - server.usage.collectorSet.register(getReportingUsageCollector(server)); + // Post initialization of the above code, the collector is now ready to fetch its data + isCollectorReady = true; server.expose('browserDriverFactory', browserFactory); server.expose('queue', createQueueFactory(server)); diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js index d5f056f32ad03..36cd4f6ac4ffb 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js @@ -114,11 +114,11 @@ async function getReportingUsageWithinRange(callCluster, server, reportingAvaila * @param {Object} server * @return {Object} kibana usage stats type collection object */ -export function getReportingUsageCollector(server) { +export function getReportingUsageCollector(server, isReady) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_REPORTING_TYPE, - + isReady, fetch: async callCluster => { const xpackInfo = server.plugins.xpack_main.info; const config = server.config(); diff --git a/x-pack/plugins/rollup/server/usage/collector.js b/x-pack/plugins/rollup/server/usage/collector.js index f7d70f04d4750..977253dfa53fb 100644 --- a/x-pack/plugins/rollup/server/usage/collector.js +++ b/x-pack/plugins/rollup/server/usage/collector.js @@ -168,6 +168,7 @@ export function registerRollupUsageCollector(server) { const collector = server.usage.collectorSet.makeUsageCollector({ type: ROLLUP_USAGE_TYPE, + isReady: () => true, fetch: async callCluster => { const rollupIndexPatterns = await fetchRollupIndexPatterns(kibanaIndex, callCluster); const rollupIndexPatternToFlagMap = createIdToFlagMap(rollupIndexPatterns); diff --git a/x-pack/plugins/spaces/server/lib/get_spaces_usage_collector.ts b/x-pack/plugins/spaces/server/lib/get_spaces_usage_collector.ts index 693fcea699449..d7c1fe2bdf111 100644 --- a/x-pack/plugins/spaces/server/lib/get_spaces_usage_collector.ts +++ b/x-pack/plugins/spaces/server/lib/get_spaces_usage_collector.ts @@ -98,6 +98,7 @@ export function getSpacesUsageCollector(server: any) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_SPACES_STATS_TYPE, + isReady: () => true, fetch: async (callCluster: any) => { const xpackInfo = server.plugins.xpack_main.info; const config = server.config(); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/telemetry/usage_collector.ts b/x-pack/plugins/upgrade_assistant/server/lib/telemetry/usage_collector.ts index 333d1cb2190bc..a52ee1c01c526 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/telemetry/usage_collector.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/telemetry/usage_collector.ts @@ -100,6 +100,7 @@ export function makeUpgradeAssistantUsageCollector(server: UpgradeAssistantTelem const kbnServer = server as UpgradeAssistantTelemetryServer; const upgradeAssistantUsageCollector = kbnServer.usage.collectorSet.makeUsageCollector({ type: UPGRADE_ASSISTANT_TYPE, + isReady: () => true, fetch: async (callCluster: any) => fetchUpgradeAssistantMetrics(callCluster, server), }); diff --git a/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts index 61c7b8e77458c..f1b280919b5aa 100644 --- a/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts +++ b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts @@ -48,6 +48,7 @@ export function getLocalizationUsageCollector(server: any) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_LOCALIZATION_STATS_TYPE, + isReady: () => true, fetch: createCollectorFetch(server), }); } diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/usage/telemetry_usage_collector.ts b/x-pack/plugins/xpack_main/server/lib/telemetry/usage/telemetry_usage_collector.ts index f14870b4c5099..99f76112bb91b 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/usage/telemetry_usage_collector.ts +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/usage/telemetry_usage_collector.ts @@ -83,6 +83,7 @@ export async function readTelemetryFile(path: string): Promise true, fetch: async () => { const configPath: string = server.config().get('xpack.xpack_main.telemetry.config'); const telemetryPath = join(dirname(configPath), 'telemetry.yml'); diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 7fa3441457904..165fc7884617b 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -192,6 +192,7 @@ export default async function ({ readConfigFile }) { '--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d', '--xpack.xpack_main.telemetry.enabled=false', '--xpack.maps.showMapsInspectorAdapter=true', + '--stats.maximumWaitTimeForAllCollectorsInS=0', '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions '--xpack.code.security.enableGitCertCheck=false', // Disable git certificate check '--timelion.ui.enabled=true',