From 44f594ee7b366e6eb304a008d1ed2bc0302bd1f6 Mon Sep 17 00:00:00 2001 From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com> Date: Mon, 14 Sep 2020 13:36:42 +0300 Subject: [PATCH] [Visualizations] Simplify visualization telemetry (#77145) * Move visualizations telemetry into visualizations plugin * Remove x-pack oss_telemetry plugin * Add unit tests * Update docs * Revert not related changes Co-authored-by: Elastic Machine --- docs/developer/plugin-list.asciidoc | 4 - src/plugins/visualizations/server/plugin.ts | 11 +- .../usage_collector/get_past_days.test.ts | 35 +++ .../server/usage_collector/get_past_days.ts | 25 +++ .../get_usage_collector.test.ts | 195 ++++++++++++++++ .../usage_collector/get_usage_collector.ts | 107 +++++++++ .../server/usage_collector/index.ts | 31 +++ x-pack/plugins/oss_telemetry/constants.ts | 9 - x-pack/plugins/oss_telemetry/kibana.json | 9 - x-pack/plugins/oss_telemetry/server/index.ts | 12 - .../server/lib/collectors/index.ts | 16 -- .../get_usage_collector.test.ts | 69 ------ .../visualizations/get_usage_collector.ts | 42 ---- .../register_usage_collector.ts | 17 -- .../server/lib/get_next_midnight.test.ts | 16 -- .../server/lib/get_next_midnight.ts | 12 - .../server/lib/get_past_days.test.ts | 22 -- .../oss_telemetry/server/lib/get_past_days.ts | 11 - .../oss_telemetry/server/lib/tasks/index.ts | 73 ------ .../tasks/visualizations/task_runner.test.ts | 211 ------------------ .../lib/tasks/visualizations/task_runner.ts | 114 ---------- x-pack/plugins/oss_telemetry/server/plugin.ts | 53 ----- .../oss_telemetry/server/test_utils/index.ts | 90 -------- 23 files changed, 403 insertions(+), 781 deletions(-) create mode 100644 src/plugins/visualizations/server/usage_collector/get_past_days.test.ts create mode 100644 src/plugins/visualizations/server/usage_collector/get_past_days.ts create mode 100644 src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts create mode 100644 src/plugins/visualizations/server/usage_collector/get_usage_collector.ts create mode 100644 src/plugins/visualizations/server/usage_collector/index.ts delete mode 100644 x-pack/plugins/oss_telemetry/constants.ts delete mode 100644 x-pack/plugins/oss_telemetry/kibana.json delete mode 100644 x-pack/plugins/oss_telemetry/server/index.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/get_past_days.test.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/get_past_days.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/plugin.ts delete mode 100644 x-pack/plugins/oss_telemetry/server/test_utils/index.ts diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 88f29fa7e3cbe..275fdf8fb69ad 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -416,10 +416,6 @@ using the CURL scripts in the scripts folder. |This plugin provides shared components and services for use across observability solutions, as well as the observability landing page UI. -|{kib-repo}blob/{branch}/x-pack/plugins/oss_telemetry[ossTelemetry] -|WARNING: Missing README. - - |{kib-repo}blob/{branch}/x-pack/plugins/painless_lab[painlessLab] |WARNING: Missing README. diff --git a/src/plugins/visualizations/server/plugin.ts b/src/plugins/visualizations/server/plugin.ts index 993612d22ebfd..7502968a33654 100644 --- a/src/plugins/visualizations/server/plugin.ts +++ b/src/plugins/visualizations/server/plugin.ts @@ -19,6 +19,8 @@ import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; +import { Observable } from 'rxjs'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { PluginInitializerContext, CoreSetup, @@ -32,16 +34,19 @@ import { VISUALIZE_ENABLE_LABS_SETTING } from '../common/constants'; import { visualizationSavedObjectType } from './saved_objects'; import { VisualizationsPluginSetup, VisualizationsPluginStart } from './types'; +import { registerVisualizationsCollector } from './usage_collector'; export class VisualizationsPlugin implements Plugin { private readonly logger: Logger; + private readonly config: Observable<{ kibana: { index: string } }>; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); + this.config = initializerContext.config.legacy.globalConfig$; } - public setup(core: CoreSetup) { + public setup(core: CoreSetup, plugins: { usageCollection?: UsageCollectionSetup }) { this.logger.debug('visualizations: Setup'); core.savedObjects.registerType(visualizationSavedObjectType); @@ -61,6 +66,10 @@ export class VisualizationsPlugin }, }); + if (plugins.usageCollection) { + registerVisualizationsCollector(plugins.usageCollection, this.config); + } + return {}; } diff --git a/src/plugins/visualizations/server/usage_collector/get_past_days.test.ts b/src/plugins/visualizations/server/usage_collector/get_past_days.test.ts new file mode 100644 index 0000000000000..7ef3009de9e5c --- /dev/null +++ b/src/plugins/visualizations/server/usage_collector/get_past_days.test.ts @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment'; +import { getPastDays } from './get_past_days'; + +describe('getPastDays', () => { + test('Returns 2 days that have passed from the current date', () => { + const pastDate = moment().subtract(2, 'days').startOf('day').toString(); + + expect(getPastDays(pastDate)).toEqual(2); + }); + + test('Returns 30 days that have passed from the current date', () => { + const pastDate = moment().subtract(30, 'days').startOf('day').toString(); + + expect(getPastDays(pastDate)).toEqual(30); + }); +}); diff --git a/src/plugins/visualizations/server/usage_collector/get_past_days.ts b/src/plugins/visualizations/server/usage_collector/get_past_days.ts new file mode 100644 index 0000000000000..5fa68d80de111 --- /dev/null +++ b/src/plugins/visualizations/server/usage_collector/get_past_days.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const getPastDays = (dateString: string): number => { + const date = new Date(dateString); + const today = new Date(); + const diff = Math.abs(date.getTime() - today.getTime()); + return Math.trunc(diff / (1000 * 60 * 60 * 24)); +}; diff --git a/src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts b/src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts new file mode 100644 index 0000000000000..4a8e4b70ae070 --- /dev/null +++ b/src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment'; +import { of } from 'rxjs'; + +import { LegacyAPICaller } from 'src/core/server'; +import { getUsageCollector } from './get_usage_collector'; + +const defaultMockSavedObjects = [ + { + _id: 'visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "shell_beads"}' }, + updated_at: moment().subtract(7, 'days').startOf('day').toString(), + }, + }, +]; + +const enlargedMockSavedObjects = [ + // default space + { + _id: 'visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + updated_at: moment().subtract(7, 'days').startOf('day').toString(), + }, + }, + { + _id: 'visualization:coolviz-456', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "printing_press"}' }, + updated_at: moment().subtract(20, 'days').startOf('day').toString(), + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "floppy_disk"}' }, + updated_at: moment().subtract(2, 'months').startOf('day').toString(), + }, + }, + // meat space + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + updated_at: moment().subtract(89, 'days').startOf('day').toString(), + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cuneiform"}' }, + updated_at: moment().subtract(5, 'months').startOf('day').toString(), + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cuneiform"}' }, + updated_at: moment().subtract(2, 'days').startOf('day').toString(), + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "floppy_disk"}' }, + updated_at: moment().subtract(7, 'days').startOf('day').toString(), + }, + }, + // cyber space + { + _id: 'cyber:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "floppy_disk"}' }, + updated_at: moment().subtract(7, 'months').startOf('day').toString(), + }, + }, + { + _id: 'cyber:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "floppy_disk"}' }, + updated_at: moment().subtract(3, 'days').startOf('day').toString(), + }, + }, + { + _id: 'cyber:visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + updated_at: moment().subtract(15, 'days').startOf('day').toString(), + }, + }, +]; + +describe('Visualizations usage collector', () => { + const configMock = of({ kibana: { index: '' } }); + const usageCollector = getUsageCollector(configMock); + const getMockCallCluster = (hits: unknown[]) => + (() => Promise.resolve({ hits: { hits } }) as unknown) as LegacyAPICaller; + + test('Should fit the shape', () => { + expect(usageCollector.type).toBe('visualization_types'); + expect(usageCollector.isReady()).toBe(true); + expect(usageCollector.fetch).toEqual(expect.any(Function)); + }); + + test('Summarizes visualizations response data', async () => { + const result = await usageCollector.fetch(getMockCallCluster(defaultMockSavedObjects)); + + expect(result).toMatchObject({ + shell_beads: { + spaces_avg: 1, + spaces_max: 1, + spaces_min: 1, + total: 1, + saved_7_days_total: 1, + saved_30_days_total: 1, + saved_90_days_total: 1, + }, + }); + }); + + test('Summarizes visualizations response data per Space', async () => { + const expectedStats = { + cave_painting: { + total: 3, + spaces_min: 1, + spaces_max: 1, + spaces_avg: 1, + saved_7_days_total: 1, + saved_30_days_total: 2, + saved_90_days_total: 3, + }, + printing_press: { + total: 1, + spaces_min: 1, + spaces_max: 1, + spaces_avg: 1, + saved_7_days_total: 0, + saved_30_days_total: 1, + saved_90_days_total: 1, + }, + cuneiform: { + total: 2, + spaces_min: 2, + spaces_max: 2, + spaces_avg: 2, + saved_7_days_total: 1, + saved_30_days_total: 1, + saved_90_days_total: 1, + }, + floppy_disk: { + total: 4, + spaces_min: 2, + spaces_max: 2, + spaces_avg: 2, + saved_7_days_total: 2, + saved_30_days_total: 2, + saved_90_days_total: 3, + }, + }; + + const result = await usageCollector.fetch(getMockCallCluster(enlargedMockSavedObjects)); + + expect(result).toMatchObject(expectedStats); + }); +}); diff --git a/src/plugins/visualizations/server/usage_collector/get_usage_collector.ts b/src/plugins/visualizations/server/usage_collector/get_usage_collector.ts new file mode 100644 index 0000000000000..165c3ee649868 --- /dev/null +++ b/src/plugins/visualizations/server/usage_collector/get_usage_collector.ts @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; +import { countBy, get, groupBy, mapValues, max, min, values } from 'lodash'; +import { first } from 'rxjs/operators'; +import { SearchResponse } from 'elasticsearch'; + +import { LegacyAPICaller } from 'src/core/server'; +import { getPastDays } from './get_past_days'; + +const VIS_USAGE_TYPE = 'visualization_types'; + +type ESResponse = SearchResponse<{ visualization: { visState: string } }>; + +interface VisSummary { + type: string; + space: string; + past_days: number; +} + +/* + * Parse the response data into telemetry payload + */ +async function getStats(callCluster: LegacyAPICaller, index: string) { + const searchParams = { + size: 10000, // elasticsearch index.max_result_window default value + index, + ignoreUnavailable: true, + filterPath: [ + 'hits.hits._id', + 'hits.hits._source.visualization', + 'hits.hits._source.updated_at', + ], + body: { + query: { + bool: { filter: { term: { type: 'visualization' } } }, + }, + }, + }; + const esResponse: ESResponse = await callCluster('search', searchParams); + const size = get(esResponse, 'hits.hits.length'); + if (size < 1) { + return; + } + + // `map` to get the raw types + const visSummaries: VisSummary[] = esResponse.hits.hits.map((hit) => { + const spacePhrases = hit._id.split(':'); + const lastUpdated: string = get(hit, '_source.updated_at'); + const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id + const visualization = get(hit, '_source.visualization', { visState: '{}' }); + const visState: { type?: string } = JSON.parse(visualization.visState); + return { + type: visState.type || '_na_', + space, + past_days: getPastDays(lastUpdated), + }; + }); + + // organize stats per type + const visTypes = groupBy(visSummaries, 'type'); + + // get the final result + return mapValues(visTypes, (curr) => { + const total = curr.length; + const spacesBreakdown = countBy(curr, 'space'); + const spaceCounts: number[] = values(spacesBreakdown); + + return { + total, + spaces_min: min(spaceCounts), + spaces_max: max(spaceCounts), + spaces_avg: total / spaceCounts.length, + saved_7_days_total: curr.filter((c) => c.past_days <= 7).length, + saved_30_days_total: curr.filter((c) => c.past_days <= 30).length, + saved_90_days_total: curr.filter((c) => c.past_days <= 90).length, + }; + }); +} + +export function getUsageCollector(config: Observable<{ kibana: { index: string } }>) { + return { + type: VIS_USAGE_TYPE, + isReady: () => true, + fetch: async (callCluster: LegacyAPICaller) => { + const index = (await config.pipe(first()).toPromise()).kibana.index; + return await getStats(callCluster, index); + }, + }; +} diff --git a/src/plugins/visualizations/server/usage_collector/index.ts b/src/plugins/visualizations/server/usage_collector/index.ts new file mode 100644 index 0000000000000..90ee65bb6ad2a --- /dev/null +++ b/src/plugins/visualizations/server/usage_collector/index.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { getUsageCollector } from './get_usage_collector'; + +export function registerVisualizationsCollector( + collectorSet: UsageCollectionSetup, + config: Observable<{ kibana: { index: string } }> +): void { + const collector = collectorSet.makeUsageCollector(getUsageCollector(config)); + collectorSet.registerCollector(collector); +} diff --git a/x-pack/plugins/oss_telemetry/constants.ts b/x-pack/plugins/oss_telemetry/constants.ts deleted file mode 100644 index 1e83bff092f2c..0000000000000 --- a/x-pack/plugins/oss_telemetry/constants.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export const PLUGIN_ID = 'oss_telemetry'; // prefix used for registering properties with services from this plugin -export const VIS_TELEMETRY_TASK = 'vis_telemetry'; // suffix for the _id of our task instance, which must be `get`-able -export const VIS_USAGE_TYPE = 'visualization_types'; // suffix for the properties of data registered with the usage service diff --git a/x-pack/plugins/oss_telemetry/kibana.json b/x-pack/plugins/oss_telemetry/kibana.json deleted file mode 100644 index 0defee0881e0e..0000000000000 --- a/x-pack/plugins/oss_telemetry/kibana.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "id": "ossTelemetry", - "server": true, - "version": "8.0.0", - "kibanaVersion": "kibana", - "requiredPlugins": ["usageCollection", "taskManager"], - "configPath": ["xpack", "oss_telemetry"], - "ui": false -} diff --git a/x-pack/plugins/oss_telemetry/server/index.ts b/x-pack/plugins/oss_telemetry/server/index.ts deleted file mode 100644 index 64527ca6daa7e..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { PluginInitializerContext } from 'src/core/server'; -import { OssTelemetryPlugin } from './plugin'; - -export const plugin = (context: PluginInitializerContext) => new OssTelemetryPlugin(context); - -export * from './plugin'; diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts deleted file mode 100644 index 845e11b80af0e..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { registerVisualizationsCollector } from './visualizations/register_usage_collector'; -import { UsageCollectionSetup } from '../../../../../../src/plugins/usage_collection/server'; -import { TaskManagerStartContract } from '../../../../task_manager/server'; - -export function registerCollectors( - usageCollection: UsageCollectionSetup, - taskManager: Promise -) { - registerVisualizationsCollector(usageCollection, taskManager); -} diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts deleted file mode 100644 index 43114787b40e5..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - getMockTaskFetch, - getMockThrowingTaskFetch, - getMockTaskInstance, -} from '../../../test_utils'; -import { taskManagerMock } from '../../../../../task_manager/server/task_manager.mock'; -import { getUsageCollector } from './get_usage_collector'; - -describe('getVisualizationsCollector#fetch', () => { - test('can return empty stats', async () => { - const { type, fetch } = getUsageCollector( - Promise.resolve(taskManagerMock.start(getMockTaskFetch())) - ); - expect(type).toBe('visualization_types'); - const fetchResult = await fetch(); - expect(fetchResult).toEqual({}); - }); - - test('provides known stats', async () => { - const { type, fetch } = getUsageCollector( - Promise.resolve( - taskManagerMock.start( - getMockTaskFetch([ - getMockTaskInstance({ - state: { - runs: 1, - stats: { comic_books: { total: 16, max: 12, min: 2, avg: 6 } }, - }, - taskType: 'test', - params: {}, - }), - ]) - ) - ) - ); - expect(type).toBe('visualization_types'); - const fetchResult = await fetch(); - expect(fetchResult).toEqual({ comic_books: { avg: 6, max: 12, min: 2, total: 16 } }); - }); - - describe('Error handling', () => { - test('Silently handles Task Manager NotInitialized', async () => { - const { fetch } = getUsageCollector( - Promise.resolve( - taskManagerMock.start( - getMockThrowingTaskFetch( - new Error('NotInitialized taskManager is still waiting for plugins to load') - ) - ) - ) - ); - const result = await fetch(); - expect(result).toBe(undefined); - }); - // In real life, the CollectorSet calls fetch and handles errors - test('defers the errors', async () => { - const { fetch } = getUsageCollector( - Promise.resolve(taskManagerMock.start(getMockThrowingTaskFetch(new Error('BOOM')))) - ); - await expect(fetch()).rejects.toThrowErrorMatchingInlineSnapshot(`"BOOM"`); - }); - }); -}); diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts deleted file mode 100644 index 9828dea4c9393..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { get } from 'lodash'; -import { PLUGIN_ID, VIS_TELEMETRY_TASK, VIS_USAGE_TYPE } from '../../../../constants'; -import { TaskManagerStartContract } from '../../../../../task_manager/server'; - -async function fetch(taskManager: TaskManagerStartContract) { - let docs; - try { - ({ docs } = await taskManager.fetch({ - query: { bool: { filter: { term: { _id: `task:${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } } }, - })); - } catch (err) { - const errMessage = err && err.message ? err.message : err.toString(); - /* - The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) - */ - if (errMessage.includes('NotInitialized')) { - docs = null; - } else { - throw err; - } - } - - return docs; -} - -export function getUsageCollector(taskManager: Promise) { - return { - type: VIS_USAGE_TYPE, - isReady: () => true, - fetch: async () => { - const docs = await fetch(await taskManager); - // get the accumulated state from the recurring task - return get(docs, '[0].state.stats'); - }, - }; -} diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts deleted file mode 100644 index 667e8b9b875fd..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { TaskManagerStartContract } from '../../../../../task_manager/server'; -import { getUsageCollector } from './get_usage_collector'; - -export function registerVisualizationsCollector( - collectorSet: UsageCollectionSetup, - taskManager: Promise -): void { - const collector = collectorSet.makeUsageCollector(getUsageCollector(taskManager)); - collectorSet.registerCollector(collector); -} diff --git a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts deleted file mode 100644 index 3bafb84d61157..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import moment from 'moment'; -import { getNextMidnight } from './get_next_midnight'; - -describe('getNextMidnight', () => { - test('Returns the next time and date of midnight as an iso string', () => { - const nextMidnightMoment = moment().add(1, 'days').startOf('day').toDate(); - - expect(getNextMidnight()).toEqual(nextMidnightMoment); - }); -}); diff --git a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts deleted file mode 100644 index a5ee8d572343c..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export function getNextMidnight() { - const nextMidnight = new Date(); - nextMidnight.setHours(0, 0, 0, 0); - nextMidnight.setDate(nextMidnight.getDate() + 1); - return nextMidnight; -} diff --git a/x-pack/plugins/oss_telemetry/server/lib/get_past_days.test.ts b/x-pack/plugins/oss_telemetry/server/lib/get_past_days.test.ts deleted file mode 100644 index 28909779343a5..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/get_past_days.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import moment from 'moment'; -import { getPastDays } from './get_past_days'; - -describe('getPastDays', () => { - test('Returns 2 days that have passed from the current date', () => { - const pastDate = moment().subtract(2, 'days').startOf('day').toString(); - - expect(getPastDays(pastDate)).toEqual(2); - }); - - test('Returns 30 days that have passed from the current date', () => { - const pastDate = moment().subtract(30, 'days').startOf('day').toString(); - - expect(getPastDays(pastDate)).toEqual(30); - }); -}); diff --git a/x-pack/plugins/oss_telemetry/server/lib/get_past_days.ts b/x-pack/plugins/oss_telemetry/server/lib/get_past_days.ts deleted file mode 100644 index 4f25ef147ad43..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/get_past_days.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -export const getPastDays = (dateString: string): number => { - const date = new Date(dateString); - const today = new Date(); - const diff = Math.abs(date.getTime() - today.getTime()); - return Math.trunc(diff / (1000 * 60 * 60 * 24)); -}; diff --git a/x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts deleted file mode 100644 index 415aeb2791d9e..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable } from 'rxjs'; -import { CoreSetup, Logger } from 'kibana/server'; -import { PLUGIN_ID, VIS_TELEMETRY_TASK } from '../../../constants'; -import { visualizationsTaskRunner } from './visualizations/task_runner'; -import { - TaskInstance, - TaskManagerStartContract, - TaskManagerSetupContract, -} from '../../../../task_manager/server'; - -export function registerTasks({ - taskManager, - logger, - getStartServices, - config, -}: { - taskManager?: TaskManagerSetupContract; - logger: Logger; - getStartServices: CoreSetup['getStartServices']; - config: Observable<{ kibana: { index: string } }>; -}) { - if (!taskManager) { - logger.debug('Task manager is not available'); - return; - } - - const esClientPromise = getStartServices().then( - ([{ elasticsearch }]) => elasticsearch.legacy.client - ); - - taskManager.registerTaskDefinitions({ - [VIS_TELEMETRY_TASK]: { - title: 'X-Pack telemetry calculator for Visualizations', - type: VIS_TELEMETRY_TASK, - createTaskRunner({ taskInstance }: { taskInstance: TaskInstance }) { - return { - run: visualizationsTaskRunner(taskInstance, config, esClientPromise), - cancel: async () => {}, - }; - }, - }, - }); -} - -export async function scheduleTasks({ - taskManager, - logger, -}: { - taskManager?: TaskManagerStartContract; - logger: Logger; -}) { - if (!taskManager) { - logger.debug('Task manager is not available'); - return; - } - - try { - await taskManager.ensureScheduled({ - id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}`, - taskType: VIS_TELEMETRY_TASK, - state: { stats: {}, runs: 0 }, - params: {}, - }); - } catch (e) { - logger.debug(`Error scheduling task, received ${e.message}`); - } -} diff --git a/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts deleted file mode 100644 index c064f39f4bc6a..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - getMockCallWithInternal, - getMockConfig, - getMockEs, - getMockTaskInstance, -} from '../../../test_utils'; -import { visualizationsTaskRunner } from './task_runner'; -import { TaskInstance } from '../../../../../task_manager/server'; -import { getNextMidnight } from '../../get_next_midnight'; -import moment from 'moment'; - -describe('visualizationsTaskRunner', () => { - let mockTaskInstance: TaskInstance; - beforeEach(() => { - mockTaskInstance = getMockTaskInstance(); - }); - - describe('Error handling', () => { - test('catches its own errors', async () => { - const mockCallWithInternal = () => Promise.reject(new Error('Things did not go well!')); - - const runner = visualizationsTaskRunner( - mockTaskInstance, - getMockConfig(), - getMockEs(mockCallWithInternal) - ); - const result = await runner(); - expect(result).toMatchObject({ - error: 'Things did not go well!', - state: { - runs: 1, - stats: undefined, - }, - }); - }); - }); - - test('Summarizes visualization response data', async () => { - const runner = visualizationsTaskRunner(mockTaskInstance, getMockConfig(), getMockEs()); - const result = await runner(); - - expect(result).toMatchObject({ - error: undefined, - runAt: getNextMidnight(), - state: { - runs: 1, - stats: { - shell_beads: { - spaces_avg: 1, - spaces_max: 1, - spaces_min: 1, - total: 1, - saved_7_days_total: 1, - saved_30_days_total: 1, - saved_90_days_total: 1, - }, - }, - }, - }); - }); - - test('Summarizes visualization response data per Space', async () => { - const mockCallWithInternal = getMockCallWithInternal([ - // default space - { - _id: 'visualization:coolviz-123', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "cave_painting"}' }, - updated_at: moment().subtract(7, 'days').startOf('day').toString(), - }, - }, - { - _id: 'visualization:coolviz-456', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "printing_press"}' }, - updated_at: moment().subtract(20, 'days').startOf('day').toString(), - }, - }, - { - _id: 'meat:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "floppy_disk"}' }, - updated_at: moment().subtract(2, 'months').startOf('day').toString(), - }, - }, - // meat space - { - _id: 'meat:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "cave_painting"}' }, - updated_at: moment().subtract(89, 'days').startOf('day').toString(), - }, - }, - { - _id: 'meat:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "cuneiform"}' }, - updated_at: moment().subtract(5, 'months').startOf('day').toString(), - }, - }, - { - _id: 'meat:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "cuneiform"}' }, - updated_at: moment().subtract(2, 'days').startOf('day').toString(), - }, - }, - { - _id: 'meat:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "floppy_disk"}' }, - updated_at: moment().subtract(7, 'days').startOf('day').toString(), - }, - }, - // cyber space - { - _id: 'cyber:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "floppy_disk"}' }, - updated_at: moment().subtract(7, 'months').startOf('day').toString(), - }, - }, - { - _id: 'cyber:visualization:coolviz-789', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "floppy_disk"}' }, - updated_at: moment().subtract(3, 'days').startOf('day').toString(), - }, - }, - { - _id: 'cyber:visualization:coolviz-123', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "cave_painting"}' }, - updated_at: moment().subtract(15, 'days').startOf('day').toString(), - }, - }, - ]); - - const expectedStats = { - cave_painting: { - total: 3, - spaces_min: 1, - spaces_max: 1, - spaces_avg: 1, - saved_7_days_total: 1, - saved_30_days_total: 2, - saved_90_days_total: 3, - }, - printing_press: { - total: 1, - spaces_min: 1, - spaces_max: 1, - spaces_avg: 1, - saved_7_days_total: 0, - saved_30_days_total: 1, - saved_90_days_total: 1, - }, - cuneiform: { - total: 2, - spaces_min: 2, - spaces_max: 2, - spaces_avg: 2, - saved_7_days_total: 1, - saved_30_days_total: 1, - saved_90_days_total: 1, - }, - floppy_disk: { - total: 4, - spaces_min: 2, - spaces_max: 2, - spaces_avg: 2, - saved_7_days_total: 2, - saved_30_days_total: 2, - saved_90_days_total: 3, - }, - }; - - const runner = visualizationsTaskRunner( - mockTaskInstance, - getMockConfig(), - getMockEs(mockCallWithInternal) - ); - const result = await runner(); - - expect(result).toMatchObject({ - error: undefined, - state: { - runs: 1, - stats: expectedStats, - }, - }); - - expect(result.state.stats).toMatchObject(expectedStats); - }); -}); diff --git a/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts deleted file mode 100644 index 27913fafe3257..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable } from 'rxjs'; -import _, { countBy, groupBy, mapValues } from 'lodash'; -import { first } from 'rxjs/operators'; - -import { LegacyAPICaller, ILegacyClusterClient } from 'src/core/server'; -import { getNextMidnight } from '../../get_next_midnight'; -import { getPastDays } from '../../get_past_days'; -import { TaskInstance } from '../../../../../task_manager/server'; -import { ESSearchHit } from '../../../../../apm/typings/elasticsearch'; - -interface VisSummary { - type: string; - space: string; - past_days: number; -} - -/* - * Parse the response data into telemetry payload - */ -async function getStats(callCluster: LegacyAPICaller, index: string) { - const searchParams = { - size: 10000, // elasticsearch index.max_result_window default value - index, - ignoreUnavailable: true, - filterPath: [ - 'hits.hits._id', - 'hits.hits._source.visualization', - 'hits.hits._source.updated_at', - ], - body: { - query: { - bool: { filter: { term: { type: 'visualization' } } }, - }, - }, - }; - const esResponse = await callCluster('search', searchParams); - const size = _.get(esResponse, 'hits.hits.length') as number; - if (size < 1) { - return; - } - - // `map` to get the raw types - const visSummaries: VisSummary[] = esResponse.hits.hits.map( - (hit: ESSearchHit<{ visState: string }>) => { - const spacePhrases: string[] = hit._id.split(':'); - const lastUpdated: string = _.get(hit, '_source.updated_at'); - const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id - const visualization = _.get(hit, '_source.visualization', { visState: '{}' }); - const visState: { type?: string } = JSON.parse(visualization.visState); - return { - type: visState.type || '_na_', - space, - past_days: getPastDays(lastUpdated), - }; - } - ); - - // organize stats per type - const visTypes = groupBy(visSummaries, 'type'); - - // get the final result - return mapValues(visTypes, (curr) => { - const total = curr.length; - const spacesBreakdown = countBy(curr, 'space'); - const spaceCounts: number[] = _.values(spacesBreakdown); - - return { - total, - spaces_min: _.min(spaceCounts), - spaces_max: _.max(spaceCounts), - spaces_avg: total / spaceCounts.length, - saved_7_days_total: curr.filter((c) => c.past_days <= 7).length, - saved_30_days_total: curr.filter((c) => c.past_days <= 30).length, - saved_90_days_total: curr.filter((c) => c.past_days <= 90).length, - }; - }); -} - -export function visualizationsTaskRunner( - taskInstance: TaskInstance, - config: Observable<{ kibana: { index: string } }>, - esClientPromise: Promise -) { - return async () => { - let stats; - let error; - - try { - const index = (await config.pipe(first()).toPromise()).kibana.index; - stats = await getStats((await esClientPromise).callAsInternalUser, index); - } catch (err) { - if (err.constructor === Error) { - error = err.message; - } else { - error = err; - } - } - - return { - runAt: getNextMidnight(), - state: { - runs: taskInstance.state.runs + 1, - stats, - }, - error, - }; - }; -} diff --git a/x-pack/plugins/oss_telemetry/server/plugin.ts b/x-pack/plugins/oss_telemetry/server/plugin.ts deleted file mode 100644 index 6a447da66952a..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/plugin.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Observable } from 'rxjs'; -import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from 'kibana/server'; -import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; -import { registerCollectors } from './lib/collectors'; -import { registerTasks, scheduleTasks } from './lib/tasks'; -import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; - -export interface OssTelemetrySetupDependencies { - usageCollection: UsageCollectionSetup; - taskManager: TaskManagerSetupContract; -} -export interface OssTelemetryStartDependencies { - taskManager: TaskManagerStartContract; -} - -export class OssTelemetryPlugin implements Plugin { - private readonly logger: Logger; - private readonly config: Observable<{ kibana: { index: string } }>; - - constructor(initializerContext: PluginInitializerContext) { - this.logger = initializerContext.logger.get('oss_telemetry'); - this.config = initializerContext.config.legacy.globalConfig$; - } - - public setup( - core: CoreSetup, - deps: OssTelemetrySetupDependencies - ) { - registerTasks({ - taskManager: deps.taskManager, - logger: this.logger, - getStartServices: core.getStartServices, - config: this.config, - }); - registerCollectors( - deps.usageCollection, - core.getStartServices().then(([_, { taskManager }]) => taskManager) - ); - } - - public start(core: CoreStart, deps: OssTelemetryStartDependencies) { - scheduleTasks({ - taskManager: deps.taskManager, - logger: this.logger, - }); - } -} diff --git a/x-pack/plugins/oss_telemetry/server/test_utils/index.ts b/x-pack/plugins/oss_telemetry/server/test_utils/index.ts deleted file mode 100644 index 9201899d5a161..0000000000000 --- a/x-pack/plugins/oss_telemetry/server/test_utils/index.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { LegacyAPICaller } from 'kibana/server'; - -import { of } from 'rxjs'; -import moment from 'moment'; -import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; -import { - ConcreteTaskInstance, - TaskStatus, - TaskManagerStartContract, -} from '../../../task_manager/server'; - -export const getMockTaskInstance = ( - overrides: Partial = {} -): ConcreteTaskInstance => ({ - state: { runs: 0, stats: {} }, - taskType: 'test', - params: {}, - id: '', - scheduledAt: new Date(), - attempts: 1, - status: TaskStatus.Idle, - runAt: new Date(), - startedAt: null, - retryAt: null, - ownerId: null, - ...overrides, -}); - -const defaultMockSavedObjects = [ - { - _id: 'visualization:coolviz-123', - _source: { - type: 'visualization', - visualization: { visState: '{"type": "shell_beads"}' }, - updated_at: moment().subtract(7, 'days').startOf('day').toString(), - }, - }, -]; - -const defaultMockTaskDocs = [getMockTaskInstance()]; - -export const getMockEs = async ( - mockCallWithInternal: LegacyAPICaller = getMockCallWithInternal() -) => { - const client = elasticsearchServiceMock.createLegacyClusterClient(); - (client.callAsInternalUser as any) = mockCallWithInternal; - return client; -}; - -export const getMockCallWithInternal = ( - hits: unknown[] = defaultMockSavedObjects -): LegacyAPICaller => { - return ((() => { - return Promise.resolve({ hits: { hits } }); - }) as unknown) as LegacyAPICaller; -}; - -export const getMockTaskFetch = ( - docs: ConcreteTaskInstance[] = defaultMockTaskDocs -): Partial> => { - return { - fetch: jest.fn((fetchOpts) => { - return Promise.resolve({ docs, searchAfter: [] }); - }), - } as Partial>; -}; - -export const getMockThrowingTaskFetch = ( - throws: Error -): Partial> => { - return { - fetch: jest.fn((fetchOpts) => { - throw throws; - }), - } as Partial>; -}; - -export const getMockConfig = () => { - return of({ kibana: { index: '' } }); -}; - -export const getCluster = () => ({ - callWithInternalUser: getMockCallWithInternal(), -});