[Maps] Enh/gis telemetry

x-pack/plugins/maps/index.js

@@ -11,11 +11,12 @@ import mappings from './mappings.json';
 import { checkLicense } from './check_license';
 import { watchStatusAndLicenseToInitialize } from
   '../../server/lib/watch_status_and_license_to_initialize';
+import { initTelemetryCollection } from './server/maps_telemetry';
 
 export function maps(kibana) {
 
   return new kibana.Plugin({
-    require: ['kibana', 'elasticsearch', 'xpack_main', 'tile_map'],
+    require: ['kibana', 'elasticsearch', 'xpack_main', 'tile_map', 'task_manager'],
     id: 'maps',
     configPrefix: 'xpack.maps',
     publicDir: resolve(__dirname, 'public'),
@@ -32,6 +33,11 @@ export function maps(kibana) {
       ],
       home: ['plugins/maps/register_feature'],
       styleSheetPaths: `${__dirname}/public/index.scss`,
+      savedObjectSchemas: {
+        'maps-telemetry': {
+          isNamespaceAgnostic: true
+        }
+      },
       mappings
     },
     config(Joi) {
@@ -43,30 +49,32 @@ export function maps(kibana) {
     init(server) {
       const gisEnabled = server.config().get('xpack.maps.enabled');
 
-      if (gisEnabled) {
-        const thisPlugin = this;
-        const xpackMainPlugin = server.plugins.xpack_main;
-        let routesInitialized = false;
-
-        watchStatusAndLicenseToInitialize(xpackMainPlugin, thisPlugin,
-          async license => {
-            if (license && license.gis && !routesInitialized) {
-              routesInitialized = true;
-              initRoutes(server, license.uid);
-            }
-          });
+      if (!gisEnabled) {
+        server.log(['info', 'maps'], 'Maps app disabled by configuration');
+        return;
+      }
 
-        xpackMainPlugin.info
-          .feature(thisPlugin.id)
-          .registerLicenseCheckResultsGenerator(checkLicense);
+      const thisPlugin = this;
+      const xpackMainPlugin = server.plugins.xpack_main;
+      let routesInitialized = false;
 
-        server.addSavedObjectsToSampleDataset('logs', webLogsSavedObjects);
-        server.injectUiAppVars('maps', async () => {
-          return await server.getInjectedUiAppVars('kibana');
+      watchStatusAndLicenseToInitialize(xpackMainPlugin, thisPlugin,
+        async license => {
+          if (license && license.gis && !routesInitialized) {
+            routesInitialized = true;
+            initRoutes(server, license.uid);
+            initTelemetryCollection(server);
+          }
         });
-      } else {
-        server.log(['info', 'maps'], 'Maps app disabled by configuration');
-      }
+
+      xpackMainPlugin.info
+        .feature(thisPlugin.id)
+        .registerLicenseCheckResultsGenerator(checkLicense);
+
+      server.addSavedObjectsToSampleDataset('logs', webLogsSavedObjects);
+      server.injectUiAppVars('maps', async () => {
+        return await server.getInjectedUiAppVars('kibana');
+      });
     }
   });
-}
+}

x-pack/plugins/maps/mappings.json

@@ -23,5 +23,50 @@
         "type": "text"
       }
     }
+  },
+  "maps-telemetry": {
+    "properties": {
+      "mapsTotalCount": {
+        "type": "long"
+      },
+      "attributesPerMap": {
+        "properties": {
+          "dataSourcesCount": {
+            "properties": {
+              "min": {
+                "type": "long"
+              },
+              "max": {
+                "type": "long"
+              },
+              "avg": {
+                "type": "long"
+              }
+            }
+          },
+          "layersCount": {
+            "properties": {
+              "min": {
+                "type": "long"
+              },
+              "max": {
+                "type": "long"
+              },
+              "avg": {
+                "type": "long"
+              }
+            }
+          },
+          "layerTypesCount": {
+            "dynamic": "true",
+            "properties": {}
+          },
+          "emsVectorLayersCount": {
+            "dynamic": "true",
+            "properties": {}
+          }
+        }
+      }
+    }
   }
-}
+}

x-pack/plugins/maps/server/maps_telemetry/index.js

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { initTelemetryCollection } from './maps_usage_collector';

x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.js

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+
+function getSavedObjectsClient(server, callCluster) {
+  const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects;
+  const internalRepository = getSavedObjectsRepository(callCluster);
+  return new SavedObjectsClient(internalRepository);
+}
+
+function getUniqueLayerCounts(layerCountsList, mapsCount) {
+  const uniqueLayerTypes = _.uniq(_.flatten(
+    layerCountsList.map(lTypes => Object.keys(lTypes))));
+
+  return uniqueLayerTypes.reduce((accu, type) => {
+    const typeCounts = layerCountsList.reduce((accu, tCounts) => {
+      tCounts[type] && accu.push(tCounts[type]);
+      return accu;
+    }, []);
+    const typeCountsSum = _.sum(typeCounts);
+    accu[type] = {
+      min: typeCounts.length ? _.min(typeCounts) : 0,
+      max: typeCounts.length ? _.max(typeCounts) : 0,
+      avg: typeCountsSum ? typeCountsSum / mapsCount : 0
+    };
+    return accu;
+  }, {});
+}
+
+export function buildMapsTelemetry(savedObjects) {
+  const savedMaps = savedObjects
+    .map(savedMapObject =>
+      JSON.parse(savedMapObject.attributes.layerListJSON));
+  const mapsCount = savedMaps.length;
+
+  const dataSourcesCount = savedMaps.map(lList => {
+    const sourceIdList = lList.map(layer => layer.sourceDescriptor.id);
+    return _.uniq(sourceIdList).length;
+  });
+
+  const layersCount = savedMaps.map(lList => lList.length);
+  const layerTypesCount = savedMaps.map(lList => _.countBy(lList, 'type'));
+
+  // Count of EMS Vector layers used
+  const emsLayersCount = savedMaps.map(lList => _(lList)
+    .countBy(layer => {
+      const isEmsFile = _.get(layer, 'sourceDescriptor.type') === 'EMS_FILE';
+      return isEmsFile && _.get(layer, 'sourceDescriptor.id');
+    })
+    .pick((val, key) => key !== 'false')
+    .value());
+
+  const dataSourcesCountSum = _.sum(dataSourcesCount);
+  const layersCountSum = _.sum(layersCount);
+  const mapsTelem = {
+    // Total count of maps
+    mapsTotalCount: mapsCount,
+    attributesPerMap: {
+      // Count of data sources per map
+      dataSourcesCount: {
+        min: dataSourcesCount.length ? _.min(dataSourcesCount) : 0,
+        max: dataSourcesCount.length ? _.max(dataSourcesCount) : 0,
+        avg: dataSourcesCountSum ? layersCountSum / mapsCount : 0
+      },
+      // Total count of layers per map
+      layersCount: {
+        min: layersCount.length ? _.min(layersCount) : 0,
+        max: layersCount.length ? _.max(layersCount) : 0,
+        avg: layersCountSum ? layersCountSum / mapsCount : 0
+      },
+      // Count of layers by type
+      layerTypesCount: {
+        ...getUniqueLayerCounts(layerTypesCount, mapsCount)
+      },
+      // Count of layer by EMS region
+      emsVectorLayersCount: {
+        ...getUniqueLayerCounts(emsLayersCount, mapsCount)
+      }
+    }
+  };
+  return mapsTelem;
+}
+
+async function getSavedObjects(savedObjectsClient) {
+  const gisMapsSavedObject = await savedObjectsClient.find({
+    type: 'gis-map'
+  });
+  return _.get(gisMapsSavedObject, 'saved_objects');
+}
+
+export async function getMapsTelemetry(server, callCluster) {
+  const savedObjectsClient = getSavedObjectsClient(server, callCluster);
+  const savedObjects = await getSavedObjects(savedObjectsClient);
+  const mapsTelemetry = buildMapsTelemetry(savedObjects);
+
+  return await savedObjectsClient.create('maps-telemetry',
+    mapsTelemetry, {
+      id: 'maps-telemetry',
+      overwrite: true,
+    });
+}

x-pack/plugins/maps/server/maps_telemetry/maps_telemetry.test.js

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as savedObjectsPayload from
+  './test_resources/sample_saved_objects.json';
+import { buildMapsTelemetry } from './maps_telemetry';
+
+describe('buildMapsTelemetry', () => {
+
+  test('returns zeroed telemetry data when there are no saved objects',
+    async () => {
+
+      const gisMaps = [];
+      const result = buildMapsTelemetry(gisMaps);
+
+      expect(result).toMatchObject({
+        "attributesPerMap": {
+          "dataSourcesCount": {
+            "avg": 0,
+            "max": 0,
+            "min": 0
+          },
+          "emsVectorLayersCount": {},
+          "layerTypesCount": {},
+          "layersCount": {
+            "avg": 0,
+            "max": 0,
+            "min": 0
+          }
+        },
+        "mapsTotalCount": 0
+      });
+    });
+
+  test('returns expected telemetry data from saved objects', async () => {
+
+    const gisMaps = savedObjectsPayload.saved_objects;
+    const result = buildMapsTelemetry(gisMaps);
+
+    expect(result).toMatchObject({
+      "attributesPerMap": {
+        "dataSourcesCount": {
+          "avg": 2.6666666666666665,
+          "max": 3,
+          "min": 2
+        },
+        "emsVectorLayersCount": {
+          "canada_provinces": {
+            "avg": 0.3333333333333333,
+            "max": 1,
+            "min": 1
+          },
+          "france_departments": {
+            "avg": 0.3333333333333333,
+            "max": 1,
+            "min": 1
+          },
+          "italy_provinces": {
+            "avg": 0.3333333333333333,
+            "max": 1,
+            "min": 1
+          }
+        },
+        "layerTypesCount": {
+          "TILE": {
+            "avg": 1,
+            "max": 1,
+            "min": 1
+          },
+          "VECTOR": {
+            "avg": 1.6666666666666667,
+            "max": 2,
+            "min": 1
+          }
+        },
+        "layersCount": {
+          "avg": 2.6666666666666665,
+          "max": 3,
+          "min": 2
+        }
+      },
+      "mapsTotalCount": 3
+    });
+  });
+});

x-pack/plugins/maps/server/maps_telemetry/maps_usage_collector.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import { TASK_ID, scheduleTask, registerMapsTelemetryTask } from './telemetry_task';
+
+export function initTelemetryCollection(server) {
+  const { taskManager } = server;
+
+  registerMapsTelemetryTask(taskManager);
+  scheduleTask(server, taskManager);
+  registerMapsUsageCollector(server);
+}
+
+export function buildCollectorObj(server) {
+  return {
+    type: 'maps',
+    fetch: async () => {
+      let docs;
+      try {
+        ({ docs } = await server.taskManager.fetch({
+          query: {
+            bool: {
+              filter: {
+                term: {
+                  _id: TASK_ID
+                }
+              }
+            }
+          }
+        }));
+      } catch (err) {
+        const errMessage = err && err.message ? err.message : err.toString();
+        /*
+         * The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager
+         * has to wait for all plugins to initialize first.
+         * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error)
+         */
+        if (errMessage.indexOf('NotInitialized') >= 0) {
+          docs = {};
+        } else {
+          throw err;
+        }
+      }
+
+      return _.get(docs, '[0].state.stats');
+    },
+  };
+}
+
+export function registerMapsUsageCollector(server) {
+  const { usage } = server;
+  const collectorObj = buildCollectorObj(server);
+  const mapsUsageCollector = usage.collectorSet.makeUsageCollector(collectorObj);
+  usage.collectorSet.register(mapsUsageCollector);
+}