diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 2cd06f13a885..c9e2f22fa19a 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -289,9 +289,9 @@ export class TelemetryCollectionManagerPlugin return stats.map((stat) => { const license = licenses[stat.cluster_uuid]; return { + collectionSource: collection.title, ...(license ? { license } : {}), ...stat, - collectionSource: collection.title, }; }); } diff --git a/x-pack/plugins/monitoring/kibana.json b/x-pack/plugins/monitoring/kibana.json index a1e28985a352..a3d886b14cdf 100644 --- a/x-pack/plugins/monitoring/kibana.json +++ b/x-pack/plugins/monitoring/kibana.json @@ -13,7 +13,6 @@ ], "optionalPlugins": [ "infra", - "telemetryCollectionManager", "usageCollection", "home", "cloud", diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 8a658619e08f..331f1dc0fbb3 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -20,8 +20,6 @@ import { CoreStart, CustomHttpResponseOptions, ResponseError, - IClusterClient, - SavedObjectsServiceStart, } from 'kibana/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; import { @@ -41,7 +39,7 @@ import { initInfraSource } from './lib/logs/init_infra_source'; import { mbSafeQuery } from './lib/mb_safe_query'; import { instantiateClient } from './es_client/instantiate_client'; import { registerCollectors } from './kibana_monitoring/collectors'; -import { registerMonitoringCollection } from './telemetry_collection'; +import { registerMonitoringTelemetryCollection } from './telemetry_collection'; import { LicenseService } from './license_service'; import { AlertsFactory } from './alerts'; import { @@ -77,8 +75,6 @@ export class Plugin { private monitoringCore = {} as MonitoringCore; private legacyShimDependencies = {} as LegacyShimDependencies; private bulkUploader: IBulkUploader = {} as IBulkUploader; - private telemetryElasticsearchClient: IClusterClient | undefined; - private telemetrySavedObjectsService: SavedObjectsServiceStart | undefined; constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; @@ -148,19 +144,6 @@ export class Plugin { plugins.alerts?.registerType(alert.getAlertType()); } - // Initialize telemetry - if (plugins.telemetryCollectionManager) { - registerMonitoringCollection({ - telemetryCollectionManager: plugins.telemetryCollectionManager, - esCluster: this.cluster, - esClientGetter: () => this.telemetryElasticsearchClient, - soServiceGetter: () => this.telemetrySavedObjectsService, - customContext: { - maxBucketSize: config.ui.max_bucket_size, - }, - }); - } - // Register collector objects for stats to show up in the APIs if (plugins.usageCollection) { core.savedObjects.registerType({ @@ -177,6 +160,11 @@ export class Plugin { }); registerCollectors(plugins.usageCollection, config, cluster); + registerMonitoringTelemetryCollection( + plugins.usageCollection, + cluster, + config.ui.max_bucket_size + ); } // Always create the bulk uploader @@ -256,16 +244,7 @@ export class Plugin { }; } - start({ elasticsearch, savedObjects }: CoreStart) { - // TODO: For the telemetry plugin to work, we need to provide the new ES client. - // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using - // the new client in Monitoring Telemetry collection yet, setting the local client allows progress for now. - // The usage collector `fetch` method has been refactored to accept a `collectorFetchContext` object, - // exposing both es clients and the saved objects client. - // We will update the client in a follow up PR. - this.telemetryElasticsearchClient = elasticsearch.client; - this.telemetrySavedObjectsService = savedObjects; - } + start() {} stop() { if (this.cluster) { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts index a119686afe66..aa2033b64973 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts @@ -9,13 +9,10 @@ import { getStackStats, getAllStats, handleAllStats } from './get_all_stats'; import { ESClusterStats } from './get_es_stats'; import { KibanaStats } from './get_kibana_stats'; import { ClustersHighLevelStats } from './get_high_level_stats'; -import { coreMock } from 'src/core/server/mocks'; describe('get_all_stats', () => { const timestamp = Date.now(); const callCluster = sinon.stub(); - const esClient = sinon.stub(); - const soClient = sinon.stub(); const esClusters = [ { cluster_uuid: 'a' }, @@ -172,24 +169,7 @@ describe('get_all_stats', () => { .onCall(4) .returns(Promise.resolve({})); // Beats state - expect( - await getAllStats( - [{ clusterUuid: 'a' }], - { - callCluster: callCluster as any, - esClient: esClient as any, - soClient: soClient as any, - usageCollection: {} as any, - kibanaRequest: undefined, - timestamp, - }, - { - logger: coreMock.createPluginInitializerContext().logger.get('test'), - version: 'version', - maxBucketSize: 1, - } - ) - ).toStrictEqual(allClusters); + expect(await getAllStats(['a'], callCluster, timestamp, 1)).toStrictEqual(allClusters); }); it('returns empty clusters', async () => { @@ -199,24 +179,7 @@ describe('get_all_stats', () => { callCluster.withArgs('search').returns(Promise.resolve(clusterUuidsResponse)); - expect( - await getAllStats( - [], - { - callCluster: callCluster as any, - esClient: esClient as any, - soClient: soClient as any, - usageCollection: {} as any, - kibanaRequest: undefined, - timestamp, - }, - { - logger: coreMock.createPluginInitializerContext().logger.get('test'), - version: 'version', - maxBucketSize: 1, - } - ) - ).toStrictEqual([]); + expect(await getAllStats([], callCluster, timestamp, 1)).toStrictEqual([]); }); }); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts index b6b2023b2af1..1f194b75e200 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts @@ -7,8 +7,8 @@ import { set } from '@elastic/safer-lodash-set'; import { get, merge } from 'lodash'; -import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server'; import moment from 'moment'; +import { LegacyAPICaller } from 'kibana/server'; import { LOGSTASH_SYSTEM_ID, KIBANA_SYSTEM_ID, @@ -20,24 +20,20 @@ import { getKibanaStats, KibanaStats } from './get_kibana_stats'; import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats'; import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats'; -export interface CustomContext { - maxBucketSize: number; -} /** * Get statistics for all products joined by Elasticsearch cluster. * Returns the array of clusters joined with the Kibana and Logstash instances. * */ -export const getAllStats: StatsGetter = async ( - clustersDetails, - { callCluster, timestamp }, - { maxBucketSize } -) => { +export async function getAllStats( + clusterUuids: string[], + callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates + timestamp: number, + maxBucketSize: number +) { const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString(); const end = moment(timestamp).toISOString(); - const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid); - const [esClusters, kibana, logstash, beats] = await Promise.all([ getElasticsearchStats(callCluster, clusterUuids, maxBucketSize), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version getKibanaStats(callCluster, clusterUuids, start, end, maxBucketSize), // stack_stats.kibana @@ -46,7 +42,7 @@ export const getAllStats: StatsGetter = async ( ]); return handleAllStats(esClusters, { kibana, logstash, beats }); -}; +} /** * Combine the statistics from the stack to create "cluster" stats that associate all products together based on the cluster diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts index b296ff090aed..18a87296f786 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts @@ -5,7 +5,6 @@ */ import sinon from 'sinon'; -import { elasticsearchServiceMock, savedObjectsRepositoryMock } from 'src/core/server/mocks'; import { getClusterUuids, fetchClusterUuids, @@ -13,10 +12,7 @@ import { } from './get_cluster_uuids'; describe('get_cluster_uuids', () => { - const kibanaRequest = undefined; const callCluster = sinon.stub(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const soClient = savedObjectsRepositoryMock.create(); const response = { aggregations: { cluster_uuids: { @@ -24,36 +20,20 @@ describe('get_cluster_uuids', () => { }, }, }; - const expectedUuids = response.aggregations.cluster_uuids.buckets - .map((bucket) => bucket.key) - .map((expectedUuid) => ({ clusterUuid: expectedUuid })); + const expectedUuids = response.aggregations.cluster_uuids.buckets.map((bucket) => bucket.key); const timestamp = Date.now(); describe('getClusterUuids', () => { it('returns cluster UUIDs', async () => { callCluster.withArgs('search').returns(Promise.resolve(response)); - expect( - await getClusterUuids( - { callCluster, esClient, soClient, timestamp, kibanaRequest, usageCollection: {} as any }, - { - maxBucketSize: 1, - } as any - ) - ).toStrictEqual(expectedUuids); + expect(await getClusterUuids(callCluster, timestamp, 1)).toStrictEqual(expectedUuids); }); }); describe('fetchClusterUuids', () => { it('searches for clusters', async () => { callCluster.returns(Promise.resolve(response)); - expect( - await fetchClusterUuids( - { callCluster, esClient, soClient, timestamp, kibanaRequest, usageCollection: {} as any }, - { - maxBucketSize: 1, - } as any - ) - ).toStrictEqual(response); + expect(await fetchClusterUuids(callCluster, timestamp, 1)).toStrictEqual(response); }); }); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts index 5f471851b662..32cda4ebdac9 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts @@ -6,33 +6,31 @@ import { get } from 'lodash'; import moment from 'moment'; -import { - ClusterDetailsGetter, - StatsCollectionConfig, - ClusterDetails, -} from 'src/plugins/telemetry_collection_manager/server'; +import { LegacyAPICaller } from 'kibana/server'; import { createQuery } from './create_query'; import { INDEX_PATTERN_ELASTICSEARCH, CLUSTER_DETAILS_FETCH_INTERVAL, } from '../../common/constants'; -import { CustomContext } from './get_all_stats'; + /** * Get a list of Cluster UUIDs that exist within the specified timespan. */ -export const getClusterUuids: ClusterDetailsGetter = async ( - config, - { maxBucketSize } -) => { - const response = await fetchClusterUuids(config, maxBucketSize); +export async function getClusterUuids( + callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates + timestamp: number, + maxBucketSize: number +) { + const response = await fetchClusterUuids(callCluster, timestamp, maxBucketSize); return handleClusterUuidsResponse(response); -}; +} /** * Fetch the aggregated Cluster UUIDs from the monitoring cluster. */ export async function fetchClusterUuids( - { callCluster, timestamp }: StatsCollectionConfig, + callCluster: LegacyAPICaller, + timestamp: number, maxBucketSize: number ) { const start = moment(timestamp).subtract(CLUSTER_DETAILS_FETCH_INTERVAL, 'ms').toISOString(); @@ -66,10 +64,7 @@ export async function fetchClusterUuids( * @param {Object} response The aggregation response * @return {Array} Strings; each representing a Cluster's UUID. */ -export function handleClusterUuidsResponse(response: any): ClusterDetails[] { +export function handleClusterUuidsResponse(response: any): string[] { const uuidBuckets: any[] = get(response, 'aggregations.cluster_uuids.buckets', []); - - return uuidBuckets.map((uuidBucket) => ({ - clusterUuid: uuidBucket.key as string, - })); + return uuidBuckets.map((uuidBucket) => uuidBucket.key); } diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts index 4812d9522d7a..8db563cebac0 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts @@ -19,7 +19,7 @@ describe('get_licenses', () => { }, }; const expectedClusters = response.hits.hits.map((hit) => hit._source); - const clusterUuids = expectedClusters.map((cluster) => ({ clusterUuid: cluster.cluster_uuid })); + const clusterUuids = expectedClusters.map((cluster) => cluster.cluster_uuid); const expectedLicenses = { abc: { type: 'basic' }, xyz: { type: 'basic' }, @@ -30,13 +30,7 @@ describe('get_licenses', () => { it('returns clusters', async () => { callWith.withArgs('search').returns(Promise.resolve(response)); - expect( - await getLicenses( - clusterUuids, - { callCluster: callWith } as any, - { maxBucketSize: 1 } as any - ) - ).toStrictEqual(expectedLicenses); + expect(await getLicenses(clusterUuids, callWith, 1)).toStrictEqual(expectedLicenses); }); }); @@ -44,13 +38,7 @@ describe('get_licenses', () => { it('searches for clusters', async () => { callWith.returns(response); - expect( - await fetchLicenses( - callWith, - clusterUuids.map(({ clusterUuid }) => clusterUuid), - { maxBucketSize: 1 } as any - ) - ).toStrictEqual(response); + expect(await fetchLicenses(callWith, clusterUuids, 1)).toStrictEqual(response); }); }); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts index a8b68929e84b..7b1b877c5127 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.ts @@ -5,26 +5,21 @@ */ import { SearchResponse } from 'elasticsearch'; -import { - ESLicense, - LicenseGetter, - StatsCollectionConfig, -} from 'src/plugins/telemetry_collection_manager/server'; +import { ESLicense } from 'src/plugins/telemetry_collection_manager/server'; +import { LegacyAPICaller } from 'kibana/server'; import { INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; -import { CustomContext } from './get_all_stats'; /** * Get statistics for all selected Elasticsearch clusters. */ -export const getLicenses: LicenseGetter = async ( - clustersDetails, - { callCluster }, - { maxBucketSize } -) => { - const clusterUuids = clustersDetails.map(({ clusterUuid }) => clusterUuid); +export async function getLicenses( + clusterUuids: string[], + callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates + maxBucketSize: number +): Promise<{ [clusterUuid: string]: ESLicense | undefined }> { const response = await fetchLicenses(callCluster, clusterUuids, maxBucketSize); return handleLicenses(response); -}; +} /** * Fetch the Elasticsearch stats. @@ -36,7 +31,7 @@ export const getLicenses: LicenseGetter = async ( * Returns the response for the aggregations to fetch details for the product. */ export function fetchLicenses( - callCluster: StatsCollectionConfig['callCluster'], + callCluster: LegacyAPICaller, clusterUuids: string[], maxBucketSize: number ) { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/index.ts b/x-pack/plugins/monitoring/server/telemetry_collection/index.ts index 764e080e390c..8627c741c974 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/index.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { registerMonitoringCollection } from './register_monitoring_collection'; +export { registerMonitoringTelemetryCollection } from './register_monitoring_telemetry_collection'; diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts deleted file mode 100644 index 109fefd2eb8d..000000000000 --- a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_collection.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - ILegacyCustomClusterClient, - IClusterClient, - SavedObjectsServiceStart, -} from 'kibana/server'; -import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; -import { getAllStats, CustomContext } from './get_all_stats'; -import { getClusterUuids } from './get_cluster_uuids'; -import { getLicenses } from './get_licenses'; - -export function registerMonitoringCollection({ - telemetryCollectionManager, - esCluster, - esClientGetter, - soServiceGetter, - customContext, -}: { - telemetryCollectionManager: TelemetryCollectionManagerPluginSetup; - esCluster: ILegacyCustomClusterClient; - esClientGetter: () => IClusterClient | undefined; - soServiceGetter: () => SavedObjectsServiceStart | undefined; - customContext: CustomContext; -}) { - telemetryCollectionManager.setCollection({ - esCluster, - esClientGetter, - soServiceGetter, - title: 'monitoring', - priority: 2, - statsGetter: getAllStats, - clusterDetailsGetter: getClusterUuids, - licenseGetter: getLicenses, - customContext, - }); -} diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts new file mode 100644 index 000000000000..91d6c2374acb --- /dev/null +++ b/x-pack/plugins/monitoring/server/telemetry_collection/register_monitoring_telemetry_collection.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ILegacyClusterClient } from 'kibana/server'; +import { UsageStatsPayload } from 'src/plugins/telemetry_collection_manager/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { getAllStats } from './get_all_stats'; +import { getClusterUuids } from './get_cluster_uuids'; +import { getLicenses } from './get_licenses'; + +// TODO: To be removed in https://github.com/elastic/kibana/pull/83546 +interface MonitoringCollectorOptions { + ignoreForInternalUploader: boolean; // Allow the additional property required by bulk_uploader to be filtered out +} + +export function registerMonitoringTelemetryCollection( + usageCollection: UsageCollectionSetup, + legacyEsClient: ILegacyClusterClient, + maxBucketSize: number +) { + const monitoringStatsCollector = usageCollection.makeStatsCollector< + UsageStatsPayload[], + unknown, + true, + MonitoringCollectorOptions + >({ + type: 'monitoringTelemetry', + isReady: () => true, + ignoreForInternalUploader: true, // Used only by monitoring's bulk_uploader to filter out unwanted collectors + extendFetchContext: { kibanaRequest: true }, + fetch: async ({ kibanaRequest }) => { + const timestamp = Date.now(); // Collect the telemetry from the monitoring indices for this moment. + // NOTE: Usually, the monitoring indices index stats for each product every 10s (by default). + // However, some data may be delayed up-to 24h because monitoring only collects extended Kibana stats in that interval + // to avoid overloading of the system when retrieving data from the collectors (that delay is dealt with in the Kibana Stats getter inside the `getAllStats` method). + // By 8.x, we expect to stop collecting the Kibana extended stats and keep only the monitoring-related metrics. + const callCluster = kibanaRequest + ? legacyEsClient.asScoped(kibanaRequest).callAsCurrentUser + : legacyEsClient.callAsInternalUser; + const clusterDetails = await getClusterUuids(callCluster, timestamp, maxBucketSize); + const [licenses, stats] = await Promise.all([ + getLicenses(clusterDetails, callCluster, maxBucketSize), + getAllStats(clusterDetails, callCluster, timestamp, maxBucketSize), + ]); + return stats.map((stat) => { + const license = licenses[stat.cluster_uuid]; + return { + ...(license ? { license } : {}), + ...stat, + collectionSource: 'monitoring', + }; + }); + }, + }); + usageCollection.registerCollector(monitoringStatsCollector); +} diff --git a/x-pack/plugins/monitoring/server/types.ts b/x-pack/plugins/monitoring/server/types.ts index 543a12fb4135..b25daced50b7 100644 --- a/x-pack/plugins/monitoring/server/types.ts +++ b/x-pack/plugins/monitoring/server/types.ts @@ -6,7 +6,6 @@ import { Observable } from 'rxjs'; import { IRouter, ILegacyClusterClient, Logger } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { LicenseFeature, ILicense } from '../../licensing/server'; import { PluginStartContract as ActionsPluginsStartContact } from '../../actions/server'; import { @@ -35,7 +34,6 @@ export interface MonitoringElasticsearchConfig { export interface PluginsSetup { encryptedSavedObjects?: EncryptedSavedObjectsPluginSetup; - telemetryCollectionManager?: TelemetryCollectionManagerPluginSetup; usageCollection?: UsageCollectionSetup; licensing: LicensingPluginSetup; features: FeaturesPluginSetupContract; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap index b9bb206b8056..b68186c0c343 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap @@ -156,3 +156,89 @@ Object { "version": "8.0.0", } `; + +exports[`Telemetry Collection: Get Aggregated Stats X-Pack telemetry with appended Monitoring data 1`] = ` +Object { + "cluster_name": "test", + "cluster_stats": Object { + "nodes": Object { + "usage": Object { + "nodes": Array [ + Object { + "aggregations": Object { + "terms": Object { + "bytes": 2, + }, + }, + "node_id": "some_node_id", + "rest_actions": Object { + "nodes_usage_action": 1, + }, + "since": 1588616945163, + "timestamp": 1588617023177, + }, + ], + }, + }, + }, + "cluster_uuid": "test", + "collection": "local", + "stack_stats": Object { + "data": Array [], + "kibana": Object { + "count": 1, + "great": "googlymoogly", + "indices": 1, + "os": Object { + "platformReleases": Array [ + Object { + "count": 1, + "platformRelease": "iv", + }, + ], + "platforms": Array [ + Object { + "count": 1, + "platform": "rocky", + }, + ], + }, + "plugins": Object { + "clouds": Object { + "chances": 95, + }, + "localization": Object { + "integrities": Object {}, + "labelsCount": 0, + "locale": "en", + }, + "rain": Object { + "chances": 2, + }, + "snow": Object { + "chances": 0, + }, + "sun": Object { + "chances": 5, + }, + }, + "versions": Array [ + Object { + "count": 1, + "version": "8675309", + }, + ], + }, + "xpack": Object {}, + }, + "timestamp": Any, + "version": "8.0.0", +} +`; + +exports[`Telemetry Collection: Get Aggregated Stats X-Pack telemetry with appended Monitoring data 2`] = ` +Object { + "collectionSource": "monitoring", + "timestamp": Any, +} +`; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index a4806cefeef3..5b3f73f206c6 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -48,34 +48,54 @@ const getContext = () => ({ logger: coreMock.createPluginInitializerContext().logger.get('test'), }); -const mockUsageCollection = (kibanaUsage = kibana) => ({ +const mockUsageCollection = (kibanaUsage: Record = kibana) => ({ bulkFetch: () => kibanaUsage, toObject: (data: any) => data, }); +/** + * Instantiate the esClient mock with the common requests + */ +function mockEsClient() { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + // mock for license should return a basic license + esClient.license.get.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: { license: { type: 'basic' } } } + ); + // mock for xpack usage should return an empty object + esClient.xpack.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: {} } + ); + // mock for nodes usage should resolve for this test + esClient.nodes.usage.mockResolvedValue( + // @ts-ignore we only care about the response body + { body: { cluster_name: 'test cluster', nodes: nodesUsage } } + ); + // mock for info should resolve for this test + esClient.info.mockResolvedValue( + // @ts-ignore we only care about the response body + { + body: { + cluster_uuid: 'test', + cluster_name: 'test', + version: { number: '8.0.0' }, + }, + } + ); + + return esClient; +} + describe('Telemetry Collection: Get Aggregated Stats', () => { test('OSS-like telemetry (no license nor X-Pack telemetry)', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const esClient = mockEsClient(); // mock for xpack.usage should throw a 404 for this test esClient.xpack.usage.mockRejectedValue(new Error('Not Found')); // mock for license should throw a 404 for this test esClient.license.get.mockRejectedValue(new Error('Not Found')); - // mock for nodes usage should resolve for this test - esClient.nodes.usage.mockResolvedValue( - // @ts-ignore we only care about the response body - { body: { cluster_name: 'test cluster', nodes: nodesUsage } } - ); - // mock for info should resolve for this test - esClient.info.mockResolvedValue( - // @ts-ignore we only care about the response body - { - body: { - cluster_uuid: 'test', - cluster_name: 'test', - version: { number: '8.0.0' }, - }, - } - ); + const usageCollection = mockUsageCollection(); const context = getContext(); @@ -95,32 +115,7 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { }); test('X-Pack telemetry (license + X-Pack)', async () => { - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - // mock for license should return a basic license - esClient.license.get.mockResolvedValue( - // @ts-ignore we only care about the response body - { body: { license: { type: 'basic' } } } - ); - // mock for xpack usage should return an empty object - esClient.xpack.usage.mockResolvedValue( - // @ts-ignore we only care about the response body - { body: {} } - ); - // mock for nodes usage should return the cluster name and nodes usage - esClient.nodes.usage.mockResolvedValue( - // @ts-ignore we only care about the response body - { body: { cluster_name: 'test cluster', nodes: nodesUsage } } - ); - esClient.info.mockResolvedValue( - // @ts-ignore we only care about the response body - { - body: { - cluster_uuid: 'test', - cluster_name: 'test', - version: { number: '8.0.0' }, - }, - } - ); + const esClient = mockEsClient(); const usageCollection = mockUsageCollection(); const context = getContext(); @@ -138,4 +133,29 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { }); }); }); + + test('X-Pack telemetry with appended Monitoring data', async () => { + const esClient = mockEsClient(); + const usageCollection = mockUsageCollection({ + ...kibana, + monitoringTelemetry: [ + { collectionSource: 'monitoring', timestamp: new Date().toISOString() }, + ], + }); + const context = getContext(); + + const stats = await getStatsWithXpack( + [{ clusterUuid: '1234' }], + { + esClient, + usageCollection, + } as any, + context + ); + stats.forEach((entry, index) => { + expect(entry).toMatchSnapshot({ + timestamp: expect.any(String), + }); + }); + }); }); diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts index 87e3d0a9613d..c0e55274b08d 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.ts @@ -21,14 +21,23 @@ export const getStatsWithXpack: StatsGetter<{}, TelemetryAggregatedStats> = asyn const clustersLocalStats = await getLocalStats(clustersDetails, config, context); const xpack = await getXPackUsage(esClient).catch(() => undefined); // We want to still report something (and do not lose the license) even when this method fails. - return clustersLocalStats.map((localStats) => { - if (xpack) { - return { - ...localStats, - stack_stats: { ...localStats.stack_stats, xpack }, - }; - } + return clustersLocalStats + .map((localStats) => { + if (xpack) { + return { + ...localStats, + stack_stats: { ...localStats.stack_stats, xpack }, + }; + } - return localStats; - }); + return localStats; + }) + .reduce((acc, stats) => { + // Concatenate the telemetry reported via monitoring as additional payloads instead of reporting it inside of stack_stats.kibana.plugins.monitoringTelemetry + const monitoringTelemetry = stats.stack_stats.kibana?.plugins?.monitoringTelemetry; + if (monitoringTelemetry) { + delete stats.stack_stats.kibana!.plugins.monitoringTelemetry; + } + return [...acc, stats, ...(monitoringTelemetry || [])]; + }, [] as TelemetryAggregatedStats[]); }; diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.js b/x-pack/test/api_integration/apis/telemetry/telemetry.js index b21ca27167bd..d0b7b2bbbb7d 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.js +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.js @@ -5,48 +5,108 @@ */ import expect from '@kbn/expect'; +import moment from 'moment'; import multiClusterFixture from './fixtures/multicluster'; import basicClusterFixture from './fixtures/basiccluster'; +/** + * Update the .monitoring-* documents loaded via the archiver to the recent `timestamp` + * @param esSupertest The client to send requests to ES + * @param fromTimestamp The lower timestamp limit to query the documents from + * @param toTimestamp The upper timestamp limit to query the documents from + * @param timestamp The new timestamp to be set + */ +function updateMonitoringDates(esSupertest, fromTimestamp, toTimestamp, timestamp) { + return Promise.all([ + esSupertest + .post('/.monitoring-es-*/_update_by_query?refresh=true') + .send({ + query: { + range: { + timestamp: { + format: 'epoch_millis', + gte: moment(fromTimestamp).valueOf(), + lte: moment(toTimestamp).valueOf(), + }, + }, + }, + script: { + source: `ctx._source.timestamp='${timestamp}'`, + lang: 'painless', + }, + }) + .expect(200), + esSupertest + .post('/.monitoring-kibana-*/_update_by_query?refresh=true') + .send({ + query: { + range: { + timestamp: { + format: 'epoch_millis', + gte: moment(fromTimestamp).valueOf(), + lte: moment(toTimestamp).valueOf(), + }, + }, + }, + script: { + source: `ctx._source.timestamp='${timestamp}'`, + lang: 'painless', + }, + }) + .expect(200), + ]); +} + export default function ({ getService }) { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); + const esSupertest = getService('esSupertest'); describe('/api/telemetry/v2/clusters/_stats', () => { - it('should load multiple trial-license clusters', async () => { + const timestamp = new Date().toISOString(); + describe('monitoring/multicluster', () => { const archive = 'monitoring/multicluster'; - const timestamp = '2017-08-16T00:00:00Z'; - - await esArchiver.load(archive); - - const { body } = await supertest - .post('/api/telemetry/v2/clusters/_stats') - .set('kbn-xsrf', 'xxx') - .send({ timestamp, unencrypted: true }) - .expect(200); - - expect(body).length(3); - expect(body).to.eql(multiClusterFixture); + const fromTimestamp = '2017-08-15T21:00:00.000Z'; + const toTimestamp = '2017-08-16T00:00:00.000Z'; + before(async () => { + await esArchiver.load(archive); + await updateMonitoringDates(esSupertest, fromTimestamp, toTimestamp, timestamp); + }); + after(() => esArchiver.unload(archive)); + it('should load multiple trial-license clusters', async () => { + const { body } = await supertest + .post('/api/telemetry/v2/clusters/_stats') + .set('kbn-xsrf', 'xxx') + .send({ timestamp, unencrypted: true }) + .expect(200); - await esArchiver.unload(archive); + expect(body).length(4); + const [localXPack, ...monitoring] = body; + expect(localXPack.collectionSource).to.eql('local_xpack'); + expect(monitoring).to.eql(multiClusterFixture.map((item) => ({ ...item, timestamp }))); + }); }); describe('with basic cluster and reporting and canvas usage info', () => { - it('should load non-expiring basic cluster', async () => { - const archive = 'monitoring/basic_6.3.x'; - const timestamp = '2018-07-23T22:13:00Z'; - + const archive = 'monitoring/basic_6.3.x'; + const fromTimestamp = '2018-07-23T22:54:59.087Z'; + const toTimestamp = '2018-07-23T22:55:05.933Z'; + before(async () => { await esArchiver.load(archive); - + await updateMonitoringDates(esSupertest, fromTimestamp, toTimestamp, timestamp); + }); + after(() => esArchiver.unload(archive)); + it('should load non-expiring basic cluster', async () => { const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') .send({ timestamp, unencrypted: true }) .expect(200); - expect(body).to.eql(basicClusterFixture); - - await esArchiver.unload(archive); + expect(body).length(2); + const [localXPack, ...monitoring] = body; + expect(localXPack.collectionSource).to.eql('local_xpack'); + expect(monitoring).to.eql(basicClusterFixture.map((item) => ({ ...item, timestamp }))); }); }); });