improvement: track the command run result and duration (#3837)

* improvement: track the command run result and duration * chore: refactor analytics naming and types
garden-io · Mar 15, 2023 · a86d967 · a86d967
1 parent dd1b599
commit a86d967
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 24 deletions.
diff --git a/core/src/analytics/analytics-types.ts b/core/src/analytics/analytics-types.ts
@@ -6,10 +6,12 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-export enum AnalyticsType {
-  COMMAND = "Run Command",
-  CALL_API = "Call API",
-  MODULE_CONFIG_ERROR = "Module Configuration Error",
-  PROJECT_CONFIG_ERROR = "Project Configuration Error",
-  VALIDATION_ERROR = "Validation Error",
-}
+export type AnalyticsEventType =
+  | "Run Command"
+  | "Command Result"
+  | "Call API"
+  | "Module Configuration Error"
+  | "Project Configuration Error"
+  | "Validation Error"
+
+export type AnalyticsCommandResult = "failure" | "success"
diff --git a/core/src/analytics/analytics.ts b/core/src/analytics/analytics.ts
@@ -11,17 +11,18 @@ import { platform, release } from "os"
 import ci = require("ci-info")
 import { isEmpty, uniq } from "lodash"
 import { globalConfigKeys, AnalyticsGlobalConfig } from "../config-store"
-import { getPackageVersion, uuidv4, sleep } from "../util/util"
+import { getPackageVersion, uuidv4, sleep, getDurationMsec } from "../util/util"
 import { SEGMENT_PROD_API_KEY, SEGMENT_DEV_API_KEY, gardenEnv } from "../constants"
 import { LogEntry } from "../logger/log-entry"
 import hasha = require("hasha")
 import { Garden } from "../garden"
-import { AnalyticsType } from "./analytics-types"
+import { AnalyticsCommandResult, AnalyticsEventType } from "./analytics-types"
 import dedent from "dedent"
 import { getGitHubUrl } from "../docs/common"
 import { Profile } from "../util/profiling"
 import { ModuleConfig } from "../config/module"
 import { UserResult } from "@garden-io/platform-api-types"
+import { GardenBaseError } from "../exceptions"
 
 const API_KEY = process.env.ANALYTICS_DEV ? SEGMENT_DEV_API_KEY : SEGMENT_PROD_API_KEY
 const CI_USER = "ci-user"
@@ -111,43 +112,53 @@ interface PropertiesBase {
 }
 
 interface EventBase {
-  type: AnalyticsType
+  type: AnalyticsEventType
   properties: PropertiesBase
 }
 
 interface CommandEvent extends EventBase {
-  type: AnalyticsType.COMMAND
+  type: "Run Command"
   properties: PropertiesBase & {
     name: string
   }
 }
 
 interface ApiEvent extends EventBase {
-  type: AnalyticsType.CALL_API
+  type: "Call API"
   properties: PropertiesBase & {
     path: string
     command: string
     name: string
   }
 }
 
+interface CommandResultEvent extends EventBase {
+  type: "Command Result"
+  properties: PropertiesBase & {
+    name: string
+    durationMsec: number
+    result: AnalyticsCommandResult
+    errors: string[] // list of GardenBaseError types
+  }
+}
+
 interface ConfigErrorEvent extends EventBase {
-  type: AnalyticsType.MODULE_CONFIG_ERROR
+  type: "Module Configuration Error"
   properties: PropertiesBase & {
     moduleName: string
     moduleType: string
   }
 }
 
 interface ProjectErrorEvent extends EventBase {
-  type: AnalyticsType.PROJECT_CONFIG_ERROR
+  type: "Project Configuration Error"
   properties: PropertiesBase & {
     fields: Array<string>
   }
 }
 
 interface ValidationErrorEvent extends EventBase {
-  type: AnalyticsType.VALIDATION_ERROR
+  type: "Validation Error"
   properties: PropertiesBase & {
     fields: Array<string>
   }
@@ -173,12 +184,18 @@ interface ApiRequestBody {
   command: string
 }
 
-type AnalyticsEvent = CommandEvent | ApiEvent | ConfigErrorEvent | ProjectErrorEvent | ValidationErrorEvent
+type AnalyticsEvent =
+  | CommandEvent
+  | CommandResultEvent
+  | ApiEvent
+  | ConfigErrorEvent
+  | ProjectErrorEvent
+  | ValidationErrorEvent
 
 export interface SegmentEvent {
   userId?: string
   anonymousId?: string
-  event: AnalyticsType
+  event: AnalyticsEventType
   properties: AnalyticsEvent["properties"]
 }
 
@@ -517,9 +534,29 @@ export class AnalyticsHandler {
    */
   trackCommand(commandName: string) {
     return this.track({
-      type: AnalyticsType.COMMAND,
+      type: "Run Command",
+      properties: {
+        name: commandName,
+        ...this.getBasicAnalyticsProperties(),
+      },
+    })
+  }
+
+  /**
+   * Track a command result.
+   */
+  trackCommandResult(commandName: string, errors: GardenBaseError[], startTime: Date) {
+    const result: AnalyticsCommandResult = errors.length > 0 ? "failure" : "success"
+
+    const durationMsec = getDurationMsec(startTime, new Date())
+
+    return this.track({
+      type: "Command Result",
       properties: {
         name: commandName,
+        durationMsec,
+        result,
+        errors: errors.map((e) => e.type),
         ...this.getBasicAnalyticsProperties(),
       },
     })
@@ -539,7 +576,7 @@ export class AnalyticsHandler {
     }
 
     return this.track({
-      type: AnalyticsType.CALL_API,
+      type: "Call API",
       properties,
     })
   }
@@ -550,7 +587,7 @@ export class AnalyticsHandler {
   trackModuleConfigError(name: string, moduleType: string) {
     const moduleName = hasha(name, { algorithm: "sha256" })
     return this.track({
-      type: AnalyticsType.MODULE_CONFIG_ERROR,
+      type: "Module Configuration Error",
       properties: {
         ...this.getBasicAnalyticsProperties(),
         moduleName,
@@ -564,7 +601,7 @@ export class AnalyticsHandler {
    */
   trackProjectConfigError(fields: Array<string>) {
     return this.track({
-      type: AnalyticsType.PROJECT_CONFIG_ERROR,
+      type: "Project Configuration Error",
       properties: {
         ...this.getBasicAnalyticsProperties(),
         fields,
@@ -577,7 +614,7 @@ export class AnalyticsHandler {
    */
   trackConfigValidationError(fields: Array<string>) {
     return this.track({
-      type: AnalyticsType.VALIDATION_ERROR,
+      type: "Validation Error",
       properties: {
         ...this.getBasicAnalyticsProperties(),
         fields,

diff --git a/core/src/cli/cli.ts b/core/src/cli/cli.ts
@@ -690,6 +690,8 @@ ${renderCommands(commands)}
 
     this.processRecord = processRecord!
 
+    const commandStartTime = new Date()
+
     try {
       const runResults = await this.runCommand({ command, parsedArgs, parsedOpts, processRecord, workingDir })
       commandResult = runResults.result
@@ -700,13 +702,15 @@ ${renderCommands(commands)}
 
     errors.push(...(commandResult.errors || []))
 
+    const gardenErrors: GardenBaseError[] = errors.map(toGardenError)
+
+    analytics?.trackCommandResult(command.getFullName(), gardenErrors, commandStartTime)
+
     // Flushes the Analytics events queue in case there are some remaining events.
     if (analytics) {
       await analytics.flush()
     }
 
-    const gardenErrors: GardenBaseError[] = errors.map(toGardenError)
-
     // --output option set
     if (argv.output) {
       const renderer = OUTPUT_RENDERERS[argv.output]!