[Cases] Adding mime type telemetry (#154679)

This PR adds telemetry for collecting statistics for the mimetypes of the files uploaded to cases. It only looks for the top 20 mime types using a terms aggregation. It does this across all cases and broken down by owner. On the telemetry server we'll need to create a single document per entry in the array here so we can do filtering for the visualizations. Example ``` "topMimeTypes": [ { "count": 2, "name": "image/png" }, { "count": 1, "name": "application/json" }, { "count": 1, "name": "text/plain" } ] ``` <details><summary>Example telemetry result</summary> "cases": { "cases": { "all": { "total": 1, "daily": 1, "weekly": 1, "monthly": 1, "status": { "open": 1, "inProgress": 0, "closed": 0 }, "syncAlertsOn": 1, "syncAlertsOff": 0, "totalUsers": 1, "totalParticipants": 1, "totalTags": 1, "totalWithAlerts": 0, "totalWithConnectors": 0, "latestDates": { "createdAt": "2023-04-11T16:07:00.565Z", "updatedAt": "2023-04-11T16:07:17.712Z", "closedAt": "" }, "assignees": { "total": 0, "totalWithZero": 1, "totalWithAtLeastOne": 0 }, "attachmentFramework": { "externalAttachments": [ { "type": ".files", "average": 4, "maxOnACase": 4, "total": 4 } ], "persistableAttachments": [], "files": { "averageSize": 3, "average": 4, "maxOnACase": 4, "total": 4, "topMimeTypes": [ { "count": 2, "name": "image/png" }, { "count": 1, "name": "application/json" }, { "count": 1, "name": "text/plain" } ] } } }, "sec": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "attachmentFramework": { "externalAttachments": [], "persistableAttachments": [], "files": { "averageSize": 0, "average": 0, "maxOnACase": 0, "total": 0, "topMimeTypes": [] } }, "assignees": { "total": 0, "totalWithZero": 0, "totalWithAtLeastOne": 0 } }, "obs": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "attachmentFramework": { "externalAttachments": [], "persistableAttachments": [], "files": { "averageSize": 0, "average": 0, "maxOnACase": 0, "total": 0, "topMimeTypes": [] } }, "assignees": { "total": 0, "totalWithZero": 0, "totalWithAtLeastOne": 0 } }, "main": { "total": 1, "daily": 1, "weekly": 1, "monthly": 1, "attachmentFramework": { "externalAttachments": [ { "type": ".files", "average": 4, "maxOnACase": 4, "total": 4 } ], "persistableAttachments": [], "files": { "averageSize": 3, "average": 4, "maxOnACase": 4, "total": 4, "topMimeTypes": [ { "count": 2, "name": "image/png" }, { "count": 1, "name": "application/json" }, { "count": 1, "name": "text/plain" } ] } }, "assignees": { "total": 0, "totalWithZero": 1, "totalWithAtLeastOne": 0 } } }, "userActions": { "all": { "total": 5, "daily": 5, "weekly": 5, "monthly": 5, "maxOnACase": 5 } }, "comments": { "all": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "maxOnACase": 0 } }, "alerts": { "all": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "maxOnACase": 0 } }, "connectors": { "all": { "all": { "totalAttached": 0 }, "itsm": { "totalAttached": 0 }, "sir": { "totalAttached": 0 }, "jira": { "totalAttached": 0 }, "resilient": { "totalAttached": 0 }, "swimlane": { "totalAttached": 0 }, "maxAttachedToACase": 0 } }, "pushes": { "all": { "total": 0, "maxOnACase": 0 } }, "configuration": { "all": { "closure": { "manually": 0, "automatic": 0 } } } } </details> ## Testing To test modify this file: https://github.com/elastic/kibana/blob/main/x-pack/plugins/cases/server/telemetry/schedule_telemetry_task.ts With: ``` export const scheduleCasesTelemetryTask = ( taskManager: TaskManagerStartContract, logger: Logger ) => { (async () => { await taskManager .ensureScheduled({ id: CASES_TELEMETRY_TASK_NAME, taskType: CASES_TELEMETRY_TASK_NAME, schedule: { interval: `${MINUTES_ON_HALF_DAY}m`, }, scope: ['cases'], params: {}, state: {}, }) .catch((err) => { logger.debug( `Error scheduling cases task with ID ${CASES_TELEMETRY_TASK_NAME} and type ${CASES_TELEMETRY_TASK_NAME}. Received ${err.message}` ); }); await taskManager.runSoon(CASES_TELEMETRY_TASK_NAME); })(); }; ``` This will cause the telemetry to be sent as soon as the server is restarted. To generate files and attachments to add stats to the telemetry I created this python script: https://github.com/elastic/cases-files-generator To retrieve the telemetry: ``` POST http://localhost:5601/api/telemetry/v2/clusters/_stats { "refreshCache": true, "unencrypted": true } ```
elastic · Apr 13, 2023 · fe9985b · fe9985b
1 parent 07a7e76
commit fe9985b
Show file tree

Hide file tree

Showing 7 changed files with 385 additions and 45 deletions.
diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts
@@ -12,7 +12,7 @@ import type {
   AttachmentAggregationResult,
   AttachmentFrameworkAggsResult,
   CaseAggregationResult,
-  FileAttachmentAggregationResult,
+  FileAttachmentAggregationResults,
 } from '../types';
 import { getCasesTelemetryData } from './cases';
 
@@ -187,17 +187,65 @@ describe('getCasesTelemetryData', () => {
         ...attachmentFramework,
       };
 
-      const filesRes: FileAttachmentAggregationResult = {
+      const filesRes: FileAttachmentAggregationResults = {
         securitySolution: {
-          averageSize: 500,
+          averageSize: { value: 500 },
+          topMimeTypes: {
+            buckets: [
+              {
+                doc_count: 5,
+                key: 'image/png',
+              },
+              {
+                doc_count: 1,
+                key: 'application/json',
+              },
+            ],
+          },
         },
         observability: {
-          averageSize: 500,
+          averageSize: { value: 500 },
+          topMimeTypes: {
+            buckets: [
+              {
+                doc_count: 5,
+                key: 'image/png',
+              },
+              {
+                doc_count: 1,
+                key: 'application/json',
+              },
+            ],
+          },
         },
         cases: {
-          averageSize: 500,
+          averageSize: { value: 500 },
+          topMimeTypes: {
+            buckets: [
+              {
+                doc_count: 5,
+                key: 'image/png',
+              },
+              {
+                doc_count: 1,
+                key: 'application/json',
+              },
+            ],
+          },
+        },
+        averageSize: { value: 500 },
+        topMimeTypes: {
+          buckets: [
+            {
+              doc_count: 5,
+              key: 'image/png',
+            },
+            {
+              doc_count: 1,
+              key: 'application/json',
+            },
+          ],
         },
-        averageSize: 500,
       };
 
       mockFind(caseAggsResult);
@@ -259,6 +307,16 @@ describe('getCasesTelemetryData', () => {
               average,
               maxOnACase: 10,
               total,
+              topMimeTypes: [
+                {
+                  count: 5,
+                  name: 'image/png',
+                },
+                {
+                  count: 1,
+                  name: 'application/json',
+                },
+              ],
             },
           },
         };
@@ -644,6 +702,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+                    "size": 10,
                   },
                 },
                 "persistableReferenceTypes": Object {
@@ -677,6 +736,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+                    "size": 10,
                   },
                 },
               },
@@ -717,6 +777,7 @@ describe('getCasesTelemetryData', () => {
               },
               "terms": Object {
                 "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+                "size": 10,
               },
             },
             "observability": Object {
@@ -752,6 +813,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+                    "size": 10,
                   },
                 },
                 "persistableReferenceTypes": Object {
@@ -785,6 +847,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+                    "size": 10,
                   },
                 },
               },
@@ -830,6 +893,7 @@ describe('getCasesTelemetryData', () => {
               },
               "terms": Object {
                 "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+                "size": 10,
               },
             },
             "securitySolution": Object {
@@ -865,6 +929,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+                    "size": 10,
                   },
                 },
                 "persistableReferenceTypes": Object {
@@ -898,6 +963,7 @@ describe('getCasesTelemetryData', () => {
                   },
                   "terms": Object {
                     "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+                    "size": 10,
                   },
                 },
               },
@@ -1031,6 +1097,12 @@ describe('getCasesTelemetryData', () => {
                     "field": "file.attributes.size",
                   },
                 },
+                "topMimeTypes": Object {
+                  "terms": Object {
+                    "field": "file.attributes.mime_type",
+                    "size": 20,
+                  },
+                },
               },
               "filter": Object {
                 "term": Object {
@@ -1045,6 +1117,12 @@ describe('getCasesTelemetryData', () => {
                     "field": "file.attributes.size",
                   },
                 },
+                "topMimeTypes": Object {
+                  "terms": Object {
+                    "field": "file.attributes.mime_type",
+                    "size": 20,
+                  },
+                },
               },
               "filter": Object {
                 "term": Object {
@@ -1059,20 +1137,32 @@ describe('getCasesTelemetryData', () => {
                     "field": "file.attributes.size",
                   },
                 },
+                "topMimeTypes": Object {
+                  "terms": Object {
+                    "field": "file.attributes.mime_type",
+                    "size": 20,
+                  },
+                },
               },
               "filter": Object {
                 "term": Object {
                   "file.attributes.Meta.owner": "securitySolution",
                 },
               },
             },
+            "topMimeTypes": Object {
+              "terms": Object {
+                "field": "file.attributes.mime_type",
+                "size": 20,
+              },
+            },
           },
           "filter": Object {
             "arguments": Array [
               Object {
                 "isQuoted": false,
                 "type": "literal",
-                "value": "file.attributes.Meta.caseId",
+                "value": "file.attributes.Meta.caseIds",
               },
               Object {
                 "type": "wildcard",

diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.ts
@@ -23,7 +23,7 @@ import type {
   LatestDates,
   CaseAggregationResult,
   AttachmentAggregationResult,
-  FileAttachmentAggregationResult,
+  FileAttachmentAggregationResults,
 } from '../types';
 import {
   findValueInBuckets,
@@ -234,6 +234,7 @@ const getCommentsSavedObjectTelemetry = async (
     externalReferenceTypes: {
       terms: {
         field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.externalReferenceAttachmentTypeId`,
+        size: 10,
       },
       aggs: {
         ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
@@ -242,6 +243,7 @@ const getCommentsSavedObjectTelemetry = async (
     persistableReferenceTypes: {
       terms: {
         field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.persistableStateAttachmentTypeId`,
+        size: 10,
       },
       aggs: {
         ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
@@ -284,7 +286,7 @@ const getCommentsSavedObjectTelemetry = async (
 
 const getFilesTelemetry = async (
   savedObjectsClient: ISavedObjectsRepository
-): Promise<SavedObjectsFindResponse<unknown, FileAttachmentAggregationResult>> => {
+): Promise<SavedObjectsFindResponse<unknown, FileAttachmentAggregationResults>> => {
   const averageSize = () => ({
     averageSize: {
       avg: {
@@ -293,6 +295,15 @@ const getFilesTelemetry = async (
     },
   });
 
+  const top20MimeTypes = () => ({
+    topMimeTypes: {
+      terms: {
+        field: `${FILE_SO_TYPE}.attributes.mime_type`,
+        size: 20,
+      },
+    },
+  });
+
   const filesByOwnerAggregationQuery = OWNERS.reduce(
     (aggQuery, owner) => ({
       ...aggQuery,
@@ -304,20 +315,21 @@ const getFilesTelemetry = async (
         },
         aggs: {
           ...averageSize(),
+          ...top20MimeTypes(),
         },
       },
     }),
     {}
   );
 
-  const filterCaseIdExists = fromKueryExpression(`${FILE_SO_TYPE}.attributes.Meta.caseId: *`);
+  const filterCaseIdExists = fromKueryExpression(`${FILE_SO_TYPE}.attributes.Meta.caseIds: *`);
 
-  return savedObjectsClient.find<unknown, FileAttachmentAggregationResult>({
+  return savedObjectsClient.find<unknown, FileAttachmentAggregationResults>({
     page: 0,
     perPage: 0,
     type: FILE_SO_TYPE,
     filter: filterCaseIdExists,
-    aggs: { ...filesByOwnerAggregationQuery, ...averageSize() },
+    aggs: { ...filesByOwnerAggregationQuery, ...averageSize(), ...top20MimeTypes() },
   });
 };