From 047a9fdb0d90d3c3c8133ff9a240cbad479ac961 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 18 Mar 2019 09:19:44 -0700 Subject: [PATCH] [Telemetry] Sample Data (#33166) * [Telemetry] Sample Data * keeper.ts * telemetry collector + wip fetch * unit test for collector fetch * update some file and directory names * fix server mixin * fix identifier * fix stub code * config key guard * filter_path * unused params * get counts * isSampleDataSetInstalled * take out unnecessary comments --- src/legacy/core_plugins/kibana/index.js | 3 + src/legacy/core_plugins/kibana/mappings.json | 10 + .../server/sample_data/routes/install.js | 4 + .../server/sample_data/routes/uninstall.js | 4 + .../server/sample_data/sample_data_mixin.js | 5 + .../server/sample_data/usage/collector.ts | 41 ++++ .../sample_data/usage/collector_fetch.test.ts | 180 ++++++++++++++++++ .../sample_data/usage/collector_fetch.ts | 132 +++++++++++++ src/legacy/server/sample_data/usage/index.ts | 21 ++ src/legacy/server/sample_data/usage/usage.ts | 59 ++++++ 10 files changed, 459 insertions(+) create mode 100644 src/legacy/server/sample_data/usage/collector.ts create mode 100644 src/legacy/server/sample_data/usage/collector_fetch.test.ts create mode 100644 src/legacy/server/sample_data/usage/collector_fetch.ts create mode 100644 src/legacy/server/sample_data/usage/index.ts create mode 100644 src/legacy/server/sample_data/usage/usage.ts diff --git a/src/legacy/core_plugins/kibana/index.js b/src/legacy/core_plugins/kibana/index.js index a6c7262368ff..b2a9678fe235 100644 --- a/src/legacy/core_plugins/kibana/index.js +++ b/src/legacy/core_plugins/kibana/index.js @@ -132,6 +132,9 @@ export default function (kibana) { 'kql-telemetry': { isNamespaceAgnostic: true, }, + 'sample-data-telemetry': { + isNamespaceAgnostic: true, + }, }, injectDefaultVars(server, options) { diff --git a/src/legacy/core_plugins/kibana/mappings.json b/src/legacy/core_plugins/kibana/mappings.json index 243d49c448c6..efde24d92c73 100644 --- a/src/legacy/core_plugins/kibana/mappings.json +++ b/src/legacy/core_plugins/kibana/mappings.json @@ -183,5 +183,15 @@ "type": "long" } } + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } } } diff --git a/src/legacy/server/sample_data/routes/install.js b/src/legacy/server/sample_data/routes/install.js index 35900458fab1..b3703e327a84 100644 --- a/src/legacy/server/sample_data/routes/install.js +++ b/src/legacy/server/sample_data/routes/install.js @@ -19,6 +19,7 @@ import Boom from 'boom'; import Joi from 'joi'; +import { usage } from '../usage'; import { loadData } from './lib/load_data'; import { createIndexName } from './lib/create_index_name'; import { @@ -171,6 +172,9 @@ export const createInstallRoute = () => ({ .code(403); } + // track the usage operation in a non-blocking way + usage(request).addInstall(params.id); + return h.response({ elasticsearchIndicesCreated: counts, kibanaSavedObjectsLoaded: sampleDataset.savedObjects.length, diff --git a/src/legacy/server/sample_data/routes/uninstall.js b/src/legacy/server/sample_data/routes/uninstall.js index b98dfacc74ee..6177c0379cd6 100644 --- a/src/legacy/server/sample_data/routes/uninstall.js +++ b/src/legacy/server/sample_data/routes/uninstall.js @@ -19,6 +19,7 @@ import _ from 'lodash'; import Joi from 'joi'; +import { usage } from '../usage'; import { createIndexName } from './lib/create_index_name'; export const createUninstallRoute = () => ({ @@ -70,6 +71,9 @@ export const createUninstallRoute = () => ({ } } + // track the usage operation in a non-blocking way + usage(request).addUninstall(params.id); + return {}; }, }, diff --git a/src/legacy/server/sample_data/sample_data_mixin.js b/src/legacy/server/sample_data/sample_data_mixin.js index b3d0abcbcd73..8a7b48a5bfc8 100644 --- a/src/legacy/server/sample_data/sample_data_mixin.js +++ b/src/legacy/server/sample_data/sample_data_mixin.js @@ -29,6 +29,9 @@ import { logsSpecProvider, ecommerceSpecProvider } from './data_sets'; +import { + makeSampleDataUsageCollector +} from './usage'; export function sampleDataMixin(kbnServer, server) { server.route(createListRoute()); @@ -94,4 +97,6 @@ export function sampleDataMixin(kbnServer, server) { server.registerSampleDataset(flightsSpecProvider); server.registerSampleDataset(logsSpecProvider); server.registerSampleDataset(ecommerceSpecProvider); + + makeSampleDataUsageCollector(server); } diff --git a/src/legacy/server/sample_data/usage/collector.ts b/src/legacy/server/sample_data/usage/collector.ts new file mode 100644 index 000000000000..4ed7487807ee --- /dev/null +++ b/src/legacy/server/sample_data/usage/collector.ts @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as Hapi from 'hapi'; +import { fetchProvider } from './collector_fetch'; + +interface KbnServer extends Hapi.Server { + usage: any; +} + +export function makeSampleDataUsageCollector(server: KbnServer) { + let index: string; + try { + index = server.config().get('kibana.index'); + } catch (err) { + return; // kibana plugin is not enabled (test environment) + } + + server.usage.collectorSet.register( + server.usage.collectorSet.makeUsageCollector({ + type: 'sample-data', + fetch: fetchProvider(index), + }) + ); +} diff --git a/src/legacy/server/sample_data/usage/collector_fetch.test.ts b/src/legacy/server/sample_data/usage/collector_fetch.test.ts new file mode 100644 index 000000000000..736e79015af9 --- /dev/null +++ b/src/legacy/server/sample_data/usage/collector_fetch.test.ts @@ -0,0 +1,180 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import sinon from 'sinon'; +import { fetchProvider } from './collector_fetch'; + +describe('Sample Data Fetch', () => { + let callClusterMock: sinon.SinonStub; + + beforeEach(() => { + callClusterMock = sinon.stub(); + }); + + test('uninitialized .kibana', async () => { + const fetch = fetchProvider('index'); + const telemetry = await fetch(callClusterMock); + + expect(telemetry).toMatchInlineSnapshot(`undefined`); + }); + + test('installed data set', async () => { + const fetch = fetchProvider('index'); + callClusterMock.returns({ + hits: { + hits: [ + { + _id: 'sample-data-telemetry:test1', + _source: { + updated_at: '2019-03-13T22:02:09Z', + 'sample-data-telemetry': { installCount: 1 }, + }, + }, + ], + }, + }); + const telemetry = await fetch(callClusterMock); + + expect(telemetry).toMatchInlineSnapshot(` +Object { + "installed": Array [ + "test1", + ], + "last_install_date": "2019-03-13T22:02:09.000Z", + "last_install_set": "test1", + "last_uninstall_date": null, + "last_uninstall_set": null, + "uninstalled": Array [], +} +`); + }); + + test('multiple installed data sets', async () => { + const fetch = fetchProvider('index'); + callClusterMock.returns({ + hits: { + hits: [ + { + _id: 'sample-data-telemetry:test1', + _source: { + updated_at: '2019-03-13T22:02:09Z', + 'sample-data-telemetry': { installCount: 1 }, + }, + }, + { + _id: 'sample-data-telemetry:test2', + _source: { + updated_at: '2019-03-13T22:13:17Z', + 'sample-data-telemetry': { installCount: 1 }, + }, + }, + ], + }, + }); + const telemetry = await fetch(callClusterMock); + + expect(telemetry).toMatchInlineSnapshot(` +Object { + "installed": Array [ + "test1", + "test2", + ], + "last_install_date": "2019-03-13T22:13:17.000Z", + "last_install_set": "test2", + "last_uninstall_date": null, + "last_uninstall_set": null, + "uninstalled": Array [], +} +`); + }); + + test('installed data set, missing counts', async () => { + const fetch = fetchProvider('index'); + callClusterMock.returns({ + hits: { + hits: [ + { + _id: 'sample-data-telemetry:test1', + _source: { updated_at: '2019-03-13T22:02:09Z', 'sample-data-telemetry': {} }, + }, + ], + }, + }); + const telemetry = await fetch(callClusterMock); + + expect(telemetry).toMatchInlineSnapshot(` +Object { + "installed": Array [], + "last_install_date": null, + "last_install_set": null, + "last_uninstall_date": null, + "last_uninstall_set": null, + "uninstalled": Array [], +} +`); + }); + + test('installed and uninstalled data sets', async () => { + const fetch = fetchProvider('index'); + callClusterMock.returns({ + hits: { + hits: [ + { + _id: 'sample-data-telemetry:test0', + _source: { + updated_at: '2019-03-13T22:29:32Z', + 'sample-data-telemetry': { installCount: 4, unInstallCount: 4 }, + }, + }, + { + _id: 'sample-data-telemetry:test1', + _source: { + updated_at: '2019-03-13T22:02:09Z', + 'sample-data-telemetry': { installCount: 1 }, + }, + }, + { + _id: 'sample-data-telemetry:test2', + _source: { + updated_at: '2019-03-13T22:13:17Z', + 'sample-data-telemetry': { installCount: 1 }, + }, + }, + ], + }, + }); + const telemetry = await fetch(callClusterMock); + + expect(telemetry).toMatchInlineSnapshot(` +Object { + "installed": Array [ + "test1", + "test2", + ], + "last_install_date": "2019-03-13T22:13:17.000Z", + "last_install_set": "test2", + "last_uninstall_date": "2019-03-13T22:29:32.000Z", + "last_uninstall_set": "test0", + "uninstalled": Array [ + "test0", + ], +} +`); + }); +}); diff --git a/src/legacy/server/sample_data/usage/collector_fetch.ts b/src/legacy/server/sample_data/usage/collector_fetch.ts new file mode 100644 index 000000000000..4c7316c85301 --- /dev/null +++ b/src/legacy/server/sample_data/usage/collector_fetch.ts @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { get } from 'lodash'; +import moment from 'moment'; + +interface SearchHit { + _id: string; + _source: { + 'sample-data-telemetry': { + installCount?: number; + unInstallCount?: number; + }; + updated_at: Date; + }; +} + +interface TelemetryResponse { + installed: string[]; + uninstalled: string[]; + last_install_date: moment.Moment | null; + last_install_set: string | null; + last_uninstall_date: moment.Moment | null; + last_uninstall_set: string | null; +} + +export function fetchProvider(index: string) { + return async (callCluster: any) => { + const response = await callCluster('search', { + index, + body: { + query: { term: { type: { value: 'sample-data-telemetry' } } }, + _source: { includes: ['sample-data-telemetry', 'type', 'updated_at'] }, + }, + filter_path: 'hits.hits._id,hits.hits._source', + ignore: [404], + }); + + const getLast = ( + dataSet: string, + dataDate: moment.Moment, + accumSet: string | null, + accumDate: moment.Moment | null + ) => { + let lastDate = accumDate; + let lastSet = accumSet; + + if (!accumDate || (accumDate && dataDate > accumDate)) { + // if a max date has not been accumulated yet, or if the current date is the new max + lastDate = dataDate; + lastSet = dataSet; + } + + return { lastDate, lastSet }; + }; + + const initial: TelemetryResponse = { + installed: [], + uninstalled: [], + last_install_date: null, + last_install_set: null, + last_uninstall_date: null, + last_uninstall_set: null, + }; + + const hits: any[] = get(response, 'hits.hits', []); + if (hits == null || hits.length === 0) { + return; + } + + return hits.reduce((telemetry: TelemetryResponse, hit: SearchHit) => { + const { installCount = 0, unInstallCount = 0 } = hit._source['sample-data-telemetry'] || { + installCount: 0, + unInstallCount: 0, + }; + + if (installCount === 0 && unInstallCount === 0) { + return telemetry; + } + + const isSampleDataSetInstalled = installCount - unInstallCount > 0; + const dataSet = hit._id.replace('sample-data-telemetry:', ''); // sample-data-telemetry:ecommerce => ecommerce + const dataDate = moment.utc(hit._source.updated_at); + + if (isSampleDataSetInstalled) { + const { lastDate, lastSet } = getLast( + dataSet, + dataDate, + telemetry.last_install_set, + telemetry.last_install_date + ); + + return { + ...telemetry, + installed: telemetry.installed.concat(dataSet), + last_install_date: lastDate, + last_install_set: lastSet, + }; + } else { + const { lastDate, lastSet } = getLast( + dataSet, + dataDate, + telemetry.last_uninstall_set, + telemetry.last_uninstall_date + ); + + return { + ...telemetry, + uninstalled: telemetry.uninstalled.concat(dataSet), + last_uninstall_date: lastDate, + last_uninstall_set: lastSet, + }; + } + }, initial); + }; +} diff --git a/src/legacy/server/sample_data/usage/index.ts b/src/legacy/server/sample_data/usage/index.ts new file mode 100644 index 000000000000..a98931758370 --- /dev/null +++ b/src/legacy/server/sample_data/usage/index.ts @@ -0,0 +1,21 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { makeSampleDataUsageCollector } from './collector'; +export { usage } from './usage'; diff --git a/src/legacy/server/sample_data/usage/usage.ts b/src/legacy/server/sample_data/usage/usage.ts new file mode 100644 index 000000000000..0fb17128b102 --- /dev/null +++ b/src/legacy/server/sample_data/usage/usage.ts @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import * as Hapi from 'hapi'; + +const SAVED_OBJECT_ID = 'sample-data-telemetry'; + +export function usage(request: Hapi.Request) { + const { server } = request; + + const handleIncrementError = (err: Error) => { + if (err != null) { + server.log(['debug', 'sample_data', 'telemetry'], err.stack); + } + server.log( + ['warning', 'sample_data', 'telemetry'], + `saved objects repository incrementCounter encountered an error: ${err}` + ); + }; + + const { + savedObjects: { getSavedObjectsRepository }, + } = server; + const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin'); + const internalRepository = getSavedObjectsRepository(callWithInternalUser); + + return { + addInstall: async (dataSet: string) => { + try { + internalRepository.incrementCounter(SAVED_OBJECT_ID, dataSet, `installCount`); + } catch (err) { + handleIncrementError(err); + } + }, + addUninstall: async (dataSet: string) => { + try { + internalRepository.incrementCounter(SAVED_OBJECT_ID, dataSet, `unInstallCount`); + } catch (err) { + handleIncrementError(err); + } + }, + }; +}