Add server OS information to telemetry stats (#23793) (#24316)

* Add server OS data to monitoring collector and telemetry * Fixup naming * Fix functional tests
elastic · Oct 20, 2018 · da13a58 · da13a58
1 parent bfc838b
commit da13a58
Show file tree

Hide file tree

Showing 10 changed files with 209 additions and 7 deletions.
diff --git a/package.json b/package.json
@@ -130,6 +130,7 @@
     "extract-text-webpack-plugin": "3.0.1",
     "file-loader": "1.1.4",
     "font-awesome": "4.4.0",
+    "getos": "^3.1.0",
     "glob": "^7.1.2",
     "glob-all": "^3.1.0",
     "good-squeeze": "2.1.0",

diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
@@ -56,8 +56,8 @@ export function getOpsStatsCollector(server) {
   return collectorSet.makeStatsCollector({
     type: KIBANA_STATS_TYPE_MONITORING,
     init: start,
-    fetch: () => {
-      return buffer.flush();
+    fetch: async () => {
+      return await buffer.flush();
     }
   });
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/get_os_info.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/get_os_info.js
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import os from 'os';
+import getos from 'getos';
+import { promisify } from 'util';
+
+/**
+ * Returns an object of OS information/
+ */
+export async function getOSInfo() {
+  const osInfo = {
+    platform: os.platform(),
+    // Include the platform name in the release to avoid grouping unrelated platforms together.
+    // release 1.0 across windows, linux, and darwin don't mean anything useful.
+    platformRelease: `${os.platform()}-${os.release()}`
+  };
+
+  // Get distribution information for linux
+  if (os.platform() === 'linux') {
+    try {
+      const distro = await promisify(getos)();
+      osInfo.distro = distro.dist;
+      // Include distro name in release for same reason as above.
+      osInfo.distroRelease = `${distro.dist}-${distro.release}`;
+    } catch (e) {
+      // ignore errors
+    }
+  }
+
+  return osInfo;
+}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/get_os_info.test.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/get_os_info.test.js
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+jest.mock('os', () => ({
+  platform: jest.fn(),
+  release: jest.fn(),
+}));
+jest.mock('getos');
+
+import os from 'os';
+import getos from 'getos';
+
+import { getOSInfo } from './get_os_info';
+
+describe('getOSInfo', () => {
+  it('returns basic OS info on non-linux', async () => {
+    os.platform.mockImplementation(() => 'darwin');
+    os.release.mockImplementation(() => '1.0.0');
+
+    const osInfo = await getOSInfo();
+
+    expect(osInfo).toEqual({
+      platform: 'darwin',
+      platformRelease: 'darwin-1.0.0',
+    });
+  });
+
+  it('returns basic OS info and distro info on linux', async () => {
+    os.platform.mockImplementation(() => 'linux');
+    os.release.mockImplementation(() => '4.9.93-linuxkit-aufs');
+
+    // Mock getos response
+    getos.mockImplementation((cb) => cb(null, {
+      os: 'linux',
+      dist: 'Ubuntu Linux',
+      codename: 'precise',
+      release: '12.04'
+    }));
+
+    const osInfo = await getOSInfo();
+
+    expect(osInfo).toEqual({
+      platform: 'linux',
+      platformRelease: 'linux-4.9.93-linuxkit-aufs',
+      // linux distro info
+      distro: 'Ubuntu Linux',
+      distroRelease: 'Ubuntu Linux-12.04',
+    });
+  });
+});
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js
@@ -6,6 +6,7 @@
 
 import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG } from '../../../../common/constants';
 import { EventRoller } from './event_roller';
+import { getOSInfo } from './get_os_info';
 import { CloudDetector } from '../../../cloud';
 
 /**
@@ -26,16 +27,24 @@ export function opsBuffer(server) {
       server.log(['debug', LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG], 'Received Kibana Ops event data');
     },
 
-    flush() {
+    async flush() {
       let cloud; // a property that will be left out of the result if the details are undefined
       const cloudDetails = cloudDetector.getCloudDetails();
       if (cloudDetails != null) {
         cloud = { cloud: cloudDetails };
       }
 
+      const eventRollup = eventRoller.flush();
+      if (eventRollup && eventRollup.os) {
+        eventRollup.os = {
+          ...eventRollup.os,
+          ...(await getOSInfo())
+        };
+      }
+
       return {
         ...cloud,
-        ...eventRoller.flush()
+        ...eventRollup
       };
     }
   };

diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_all_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_all_stats.js
@@ -127,6 +127,10 @@ describe('get_all_stats', () => {
                 kibana_stats: {
                   kibana: {
                     version: '1.2.3-alpha1'
+                  },
+                  os: {
+                    platform: 'win',
+                    platformRelease: 'win-10.0'
                   }
                 }
               }
@@ -159,13 +163,25 @@ describe('get_all_stats', () => {
               versions: [
                 { version: '1.2.3-alpha1', count: 1 }
               ],
+              os: {
+                platforms: [{ platform: 'win', count: 1 }],
+                platformReleases: [{ platformRelease: 'win-10.0', count: 1 }],
+                distros: [],
+                distroReleases: [],
+              },
               cloud: undefined
             },
             logstash: {
               count: 1,
               versions: [
                 { version: '2.3.4-beta2', count: 1 }
               ],
+              os: {
+                platforms: [],
+                platformReleases: [],
+                distros: [],
+                distroReleases: [],
+              },
               cloud: undefined
             }
           }

diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_high_level_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/__tests__/get_high_level_stats.js
@@ -30,6 +30,12 @@ describe('get_high_level_stats', () => {
             [`${product}_stats`]: {
               [`${product}`]: {
                 version: '1.2.3-alpha1'
+              },
+              os: {
+                platform: 'linux',
+                platformRelease: 'linux-4.0',
+                distro: 'Ubuntu Linux',
+                distroRelease: 'Ubuntu Linux-14.04'
               }
             }
           }
@@ -40,6 +46,12 @@ describe('get_high_level_stats', () => {
             [`${product}_stats`]: {
               [`${product}`]: {
                 version: '1.2.3-alpha1'
+              },
+              os: {
+                platform: 'linux',
+                platformRelease: 'linux-4.0',
+                distro: 'Ubuntu Linux',
+                distroRelease: 'Ubuntu Linux-14.04'
               }
             }
           }
@@ -50,6 +62,12 @@ describe('get_high_level_stats', () => {
             [`${product}_stats`]: {
               [`${product}`]: {
                 version: '2.3.4-rc1'
+              },
+              os: {
+                platform: 'linux',
+                platformRelease: 'linux-4.0',
+                distro: 'Ubuntu Linux',
+                distroRelease: 'Ubuntu Linux-14.04'
               }
             }
           }
@@ -60,11 +78,15 @@ describe('get_high_level_stats', () => {
             [`${product}_stats`]: {
               [`${product}`]: {
                 version: '2.3.4'
+              },
+              os: {
+                platform: 'darwin',
+                platformRelease: 'darwin-18.0'
               }
             }
           }
         },
-        // no version
+        // no version or os
         {
           _source: {
             cluster_uuid: 'b'
@@ -155,6 +177,12 @@ describe('get_high_level_stats', () => {
       versions: [
         { version: '1.2.3-alpha1', count: 2 }
       ],
+      os: {
+        platforms: [{ platform: 'linux', count: 2 }],
+        platformReleases: [{ platformRelease: 'linux-4.0', count: 2 }],
+        distros: [{ distro: 'Ubuntu Linux', count: 2 }],
+        distroReleases: [{ distroRelease: 'Ubuntu Linux-14.04', count: 2 }],
+      },
       cloud: undefined
     },
     b: {
@@ -163,13 +191,31 @@ describe('get_high_level_stats', () => {
         { version: '2.3.4-rc1', count: 1 },
         { version: '2.3.4', count: 1 }
       ],
+      os: {
+        platformReleases: [
+          { platformRelease: 'linux-4.0', count: 1 },
+          { platformRelease: 'darwin-18.0', count: 1 },
+        ],
+        platforms: [
+          { platform: 'linux', count: 1 },
+          { platform: 'darwin', count: 1 },
+        ],
+        distros: [{ distro: 'Ubuntu Linux', count: 1 }],
+        distroReleases: [{ distroRelease: 'Ubuntu Linux-14.04', count: 1 }],
+      },
       cloud: undefined
     },
     c: {
       count: 4,
       versions: [
         { version: '5.6.1', count: 4 }
       ],
+      os: {
+        platforms: [],
+        platformReleases: [],
+        distros: [],
+        distroReleases: [],
+      },
       cloud: [
         {
           name: cloudName,

diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_high_level_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/monitoring/get_high_level_stats.js
@@ -79,13 +79,23 @@ function groupInstancesByCluster(instances, product) {
     const clusterUuid = get(instance, '_source.cluster_uuid');
     const version = get(instance, `_source.${product}_stats.${product}.version`);
     const cloud = get(instance, `_source.${product}_stats.cloud`);
+    const os = get(instance, `_source.${product}_stats.os`);
 
-    // put the instance into the right cluster map
     if (clusterUuid) {
       let cluster = clusterMap.get(clusterUuid);
 
       if (!cluster) {
-        cluster = { count: 0, versions: new Map(), cloudMap: new Map() };
+        cluster = {
+          count: 0,
+          versions: new Map(),
+          cloudMap: new Map(),
+          os: {
+            platforms: new Map(),
+            platformReleases: new Map(),
+            distros: new Map(),
+            distroReleases: new Map(),
+          }
+        };
         clusterMap.set(clusterUuid, cluster);
       }
 
@@ -94,6 +104,13 @@ function groupInstancesByCluster(instances, product) {
 
       incrementByKey(cluster.versions, version);
       reduceCloudForCluster(cluster.cloudMap, cloud);
+
+      if (os) {
+        incrementByKey(cluster.os.platforms, os.platform);
+        incrementByKey(cluster.os.platformReleases, os.platformRelease);
+        incrementByKey(cluster.os.distros, os.distro);
+        incrementByKey(cluster.os.distroReleases, os.distroRelease);
+      }
     }
   });
 
@@ -159,6 +176,7 @@ export function fetchHighLevelStats(server, callCluster, clusterUuids, start, en
     filterPath: [
       'hits.hits._source.cluster_uuid',
       `hits.hits._source.${product}_stats.${product}.version`,
+      `hits.hits._source.${product}_stats.os`,
       `hits.hits._source.${product}_stats.usage`,
       // we don't want metadata
       `hits.hits._source.${product}_stats.cloud.name`,
@@ -221,6 +239,12 @@ export function handleHighLevelStatsResponse(response, product) {
       count: cluster.count,
       // remap the versions into something more digestable that won't blowup mappings:
       versions: mapToList(cluster.versions, 'version'),
+      os: {
+        platforms: mapToList(cluster.os.platforms, 'platform'),
+        platformReleases: mapToList(cluster.os.platformReleases, 'platformRelease'),
+        distros: mapToList(cluster.os.distros, 'distro'),
+        distroReleases: mapToList(cluster.os.distroReleases, 'distroRelease'),
+      },
       cloud: clouds.length > 0 ? clouds : undefined
     };
   }

diff --git a/x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/basiccluster.json b/x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/basiccluster.json
@@ -180,6 +180,12 @@
         "index_pattern": {
           "total": 1
         },
+        "os": {
+          "distros": [],
+          "distroReleases": [],
+          "platforms": [],
+          "platformReleases": []
+        },
         "indices": 1,
         "search": {
           "total": 0

diff --git a/x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/multicluster.json b/x-pack/test/api_integration/apis/xpack_main/telemetry/fixtures/multicluster.json
@@ -744,6 +744,12 @@
           "total": 0
         },
         "indices": 1,
+        "os": {
+          "distros": [],
+          "distroReleases": [],
+          "platforms": [],
+          "platformReleases": []
+        },
         "search": {
           "total": 0
         },
@@ -759,6 +765,12 @@
       },
       "logstash": {
         "count": 1,
+        "os": {
+          "distros": [],
+          "distroReleases": [],
+          "platforms": [],
+          "platformReleases": []
+        },
         "versions": [
           {
             "count": 1,