chore: 0.13 analytics improvements

core/src/analytics/analytics.ts

@@ -24,6 +24,8 @@ import { ModuleConfig } from "../config/module"
 import { UserResult } from "@garden-io/platform-api-types"
 import { uuidv4 } from "../util/random"
 import { GardenBaseError } from "../exceptions"
+import { ActionConfigMap } from "../actions/types"
+import { actionKinds } from "../actions/types"
 
 const API_KEY = process.env.ANALYTICS_DEV ? SEGMENT_DEV_API_KEY : SEGMENT_PROD_API_KEY
 const CI_USER = "ci-user"
@@ -92,6 +94,11 @@ interface ProjectMetadata {
   servicesCount: number
   testsCount: number
   moduleTypes: string[]
+  actionsCount: number
+  buildActionCount: number
+  testActionCount: number
+  deployActionCount: number
+  runActionCount: number
 }
 
 interface PropertiesBase {
@@ -144,6 +151,7 @@ interface CommandResultEvent extends EventBase {
     durationMsec: number
     result: AnalyticsCommandResult
     errors: string[] // list of GardenBaseError types
+    exitCode?: number
   }
 }
 
@@ -249,6 +257,7 @@ export class AnalyticsHandler {
     analyticsConfig,
     anonymousUserId,
     moduleConfigs,
+    actionConfigs,
     cloudUser,
     isEnabled,
     ciInfo,
@@ -258,6 +267,7 @@ export class AnalyticsHandler {
     analyticsConfig: AnalyticsGlobalConfig
     anonymousUserId: string
     moduleConfigs: ModuleConfig[]
+    actionConfigs: ActionConfigMap
     isEnabled: boolean
     cloudUser?: UserResult
     ciInfo: CiInfo
@@ -273,12 +283,30 @@ export class AnalyticsHandler {
     this.pendingEvents = new Map()
 
     this.analyticsConfig = analyticsConfig
+
+    let actionsCount = 0
+    const countByActionKind: { [key: string]: number } = {}
+
+    for (const kind of actionKinds) {
+      countByActionKind[kind] = 0
+
+      for (const name in actionConfigs[kind]) {
+        countByActionKind[kind] = countByActionKind[kind] + 1
+        actionsCount++
+      }
+    }
+
     this.projectMetadata = {
       modulesCount: moduleConfigs.length,
       moduleTypes: uniq(moduleConfigs.map((c) => c.type)),
       tasksCount: countActions(moduleConfigs, "tasks"),
       servicesCount: countActions(moduleConfigs, "services"),
       testsCount: countActions(moduleConfigs, "tests"),
+      actionsCount,
+      buildActionCount: countByActionKind["Build"],
+      testActionCount: countByActionKind["Test"],
+      deployActionCount: countByActionKind["Deploy"],
+      runActionCount: countByActionKind["Run"],
     }
     this.systemConfig = {
       platform: platform(),
@@ -381,6 +409,7 @@ export class AnalyticsHandler {
     const currentAnalyticsConfig = await garden.globalConfigStore.get("analytics")
     const isFirstRun = !currentAnalyticsConfig.firstRunAt
     const moduleConfigs = await garden.getRawModuleConfigs()
+    const actionConfigs = await garden.getRawActionConfigs()
 
     let cloudUser: UserResult | undefined
     if (garden.cloudApi) {
@@ -434,6 +463,7 @@ export class AnalyticsHandler {
       log,
       analyticsConfig,
       moduleConfigs,
+      actionConfigs,
       cloudUser,
       isEnabled,
       ciInfo,
@@ -561,7 +591,7 @@ export class AnalyticsHandler {
   /**
    * Track a command result.
    */
-  trackCommandResult(commandName: string, errors: GardenBaseError[], startTime: Date) {
+  trackCommandResult(commandName: string, errors: GardenBaseError[], startTime: Date, exitCode?: number) {
     const result: AnalyticsCommandResult = errors.length > 0 ? "failure" : "success"
 
     const durationMsec = getDurationMsec(startTime, new Date())
@@ -573,6 +603,7 @@ export class AnalyticsHandler {
         durationMsec,
         result,
         errors: errors.map((e) => e.type),
+        exitCode,
         ...this.getBasicAnalyticsProperties(),
       },
     })

core/src/cli/cli.ts

@@ -339,7 +339,8 @@ ${renderCommands(commands)}
 
     let garden: Garden
     let result: CommandResult<any> = {}
-    let analytics: AnalyticsHandler
+    let analytics: AnalyticsHandler | undefined = undefined
+    let commandStartTime: Date | undefined = undefined
 
     const prepareParams = {
       log,
@@ -472,6 +473,8 @@ ${renderCommands(commands)}
 
         await checkForStaticDir()
 
+        commandStartTime = new Date()
+
         // Check if the command is protected and ask for confirmation to proceed if production flag is "true".
         if (await command.isAllowedToRun(garden, log, parsedOpts)) {
           // TODO: enforce that commands always output DeepPrimitiveMap
@@ -491,6 +494,10 @@ ${renderCommands(commands)}
           result = {}
         }
 
+        // Track the result of the command run
+        const allErrors = result.errors || []
+        analytics.trackCommandResult(command.getFullName(), allErrors, commandStartTime, result.exitCode)
+
         // This is a little trick to do a round trip in the event loop, which may be necessary for event handlers to
         // fire, which may be needed to e.g. capture monitors added in event handlers
         await waitForOutputFlush()
@@ -514,6 +521,12 @@ ${renderCommands(commands)}
             parsedOpts.format
           )
         }
+
+        analytics?.trackCommandResult(command.getFullName(), [err], commandStartTime || new Date(), result.exitCode)
+
+        // flush analytics early since when we throw the instance is not returned
+        await analytics?.flush()
+
         throw err
       } finally {
         if (!result.restartRequired) {
@@ -664,8 +677,6 @@ ${renderCommands(commands)}
 
     this.processRecord = processRecord!
 
-    const commandStartTime = new Date()
-
     try {
       const runResults = await this.runCommand({ command, parsedArgs, parsedOpts, processRecord, workingDir, log })
       commandResult = runResults.result
@@ -678,12 +689,8 @@ ${renderCommands(commands)}
 
     const gardenErrors: GardenBaseError[] = errors.map(toGardenError)
 
-    analytics?.trackCommandResult(command.getFullName(), gardenErrors, commandStartTime)
-
     // Flushes the Analytics events queue in case there are some remaining events.
-    if (analytics) {
-      await analytics.flush()
-    }
+    await analytics?.flush()
 
     // --output option set
     if (argv.output) {

core/test/unit/src/analytics/analytics.ts

@@ -423,6 +423,11 @@ describe("AnalyticsHandler", () => {
             tasksCount: 4,
             servicesCount: 3,
             testsCount: 5,
+            actionsCount: 0,
+            buildActionCount: 0,
+            testActionCount: 0,
+            deployActionCount: 0,
+            runActionCount: 0,
           },
         },
       })
@@ -462,6 +467,11 @@ describe("AnalyticsHandler", () => {
             tasksCount: 4,
             servicesCount: 3,
             testsCount: 5,
+            actionsCount: 0,
+            buildActionCount: 0,
+            testActionCount: 0,
+            deployActionCount: 0,
+            runActionCount: 0,
           },
         },
       })
@@ -518,6 +528,11 @@ describe("AnalyticsHandler", () => {
             tasksCount: 0,
             servicesCount: 0,
             testsCount: 0,
+            actionsCount: 0,
+            buildActionCount: 0,
+            testActionCount: 0,
+            deployActionCount: 0,
+            runActionCount: 0,
           },
         },
       })
@@ -562,6 +577,60 @@ describe("AnalyticsHandler", () => {
             tasksCount: 0,
             servicesCount: 0,
             testsCount: 0,
+            actionsCount: 0,
+            buildActionCount: 0,
+            testActionCount: 0,
+            deployActionCount: 0,
+            runActionCount: 0,
+          },
+        },
+      })
+    })
+    it("should have counts for action kinds", async () => {
+      scope.post(`/v1/batch`).reply(200)
+
+      const root = getDataDir("test-projects", "config-templates")
+      garden = await makeTestGarden(root)
+      garden.vcsInfo.originUrl = remoteOriginUrl
+
+      await garden.globalConfigStore.set("analytics", basicConfig)
+      const now = freezeTime()
+      analytics = await AnalyticsHandler.factory({ garden, log: garden.log, ciInfo })
+
+      const event = analytics.trackCommand("testCommand")
+
+      expect(event).to.eql({
+        type: "Run Command",
+        properties: {
+          name: "testCommand",
+          projectId: AnalyticsHandler.hash(remoteOriginUrl),
+          projectIdV2: AnalyticsHandler.hashV2(remoteOriginUrl),
+          projectName: AnalyticsHandler.hash("config-templates"),
+          projectNameV2: AnalyticsHandler.hashV2("config-templates"),
+          enterpriseDomain: AnalyticsHandler.hash(DEFAULT_GARDEN_CLOUD_DOMAIN),
+          enterpriseDomainV2: AnalyticsHandler.hashV2(DEFAULT_GARDEN_CLOUD_DOMAIN),
+          enterpriseProjectId: undefined,
+          enterpriseProjectIdV2: undefined,
+          isLoggedIn: false,
+          customer: undefined,
+          ciName: analytics["ciName"],
+          system: analytics["systemConfig"],
+          isCI: analytics["isCI"],
+          sessionId: analytics["sessionId"],
+          firstRunAt: basicConfig.firstRunAt,
+          latestRunAt: now,
+          isRecurringUser: false,
+          projectMetadata: {
+            modulesCount: 0,
+            moduleTypes: [],
+            tasksCount: 0,
+            servicesCount: 0,
+            testsCount: 0,
+            actionsCount: 3,
+            buildActionCount: 1,
+            testActionCount: 1,
+            deployActionCount: 1,
+            runActionCount: 0,
           },
         },
       })

core/test/unit/src/cli/analytics.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import nock from "nock"
+import { expect } from "chai"
+
+import { GardenCli } from "../../../../src/cli/cli"
+import { GlobalConfigStore } from "../../../../src/config-store/global"
+import { TestGarden, enableAnalytics, makeTestGardenA } from "../../../helpers"
+import { Command } from "../../../../src/commands/base"
+import { isEqual } from "lodash"
+import { getRootLogger, RootLogger } from "../../../../src/logger/logger"
+
+// TODO: These tests are skipped because they fail repeatedly in CI, but works fine locally
+describe("cli analytics", () => {
+  let cli: GardenCli
+  const globalConfigStore = new GlobalConfigStore()
+  const log = getRootLogger().createLog()
+
+  beforeEach(async () => {
+    cli = new GardenCli()
+    garden = await makeTestGardenA()
+    resetAnalyticsConfig = await enableAnalytics(garden)
+  })
+
+  afterEach(async () => {
+    if (cli.processRecord && cli.processRecord.pid) {
+      await globalConfigStore.delete("activeProcesses", String(cli.processRecord.pid))
+    }
+
+    await resetAnalyticsConfig()
+    nock.cleanAll()
+  })
+
+  let garden: TestGarden
+  let resetAnalyticsConfig: Function
+
+  class TestCommand extends Command {
+    name = "test-command"
+    help = "hilfe!"
+    noProject = true
+
+    printHeader() {}
+
+    async action({ args }) {
+      return { result: { args } }
+    }
+  }
+
+  it.skip("should access the version check service", async () => {
+    const scope = nock("https://get.garden.io")
+    scope.get("/version").query(true).reply(200)
+
+    const command = new TestCommand()
+    cli.addCommand(command)
+
+    await cli.run({ args: ["test-command"], exitOnError: false })
+
+    expect(scope.done()).to.not.throw
+  })
+
+  it.skip("should wait for queued analytic events to flush", async () => {
+    const scope = nock("https://api.segment.io")
+
+    // Each command run result in two events:
+    // 'Run Command' and 'Command Result'
+    scope
+      .post(`/v1/batch`, (body) => {
+        const events = body.batch.map((event: any) => ({
+          event: event.event,
+          type: event.type,
+          name: event.properties.name,
+        }))
+
+        return isEqual(events, [
+          {
+            event: "Run Command",
+            type: "track",
+            name: "test-command",
+          },
+          {
+            event: "Command Result",
+            type: "track",
+            name: "test-command",
+          },
+        ])
+      })
+      .reply(200)
+
+    const command = new TestCommand()
+    cli.addCommand(command)
+
+    await cli.run({ args: ["test-command"], exitOnError: false })
+
+    expect(scope.done()).to.not.throw
+  })
+})