feat(logs): allow filtering log lines by tag

The new `--tag` option for the `garden logs` command can be used to show
only log lines that match a given tag, e.g. `--tag 'container=foo'`.

If multiple filters are specified in a single tag option (e.g.
`--tag 'container=foo,someOtherTag=bar'`), they must all be matched.

If multiple `--tag` options are provided (e.g.
`--tag 'container=api' --tag 'container=frontend'`), they will be OR-ed
together (i.e. if any of them match, the log line will be included).

Glob-style wildcards, can also be used (e.g.
`--tag 'container=prefix-*'`).

core/src/cli/params.ts

@@ -290,6 +290,23 @@ export class BooleanParameter extends Parameter<boolean> {
   }
 }
 
+/**
+ * Similar to `StringsOption`, but doesn't split individual option values on `,`
+ */
+export class TagsOption extends Parameter<string[] | undefined> {
+  type = "array:tag"
+  schema = joi.array().items(joi.string())
+
+  coerce(input?: string | string[]): string[] {
+    if (!input) {
+      return []
+    } else if (!isArray(input)) {
+      input = [input]
+    }
+    return input
+  }
+}
+
 export class EnvironmentOption extends StringParameter {
   type = "string"
   schema = joi.environment()

core/src/commands/logs.ts

@@ -6,21 +6,24 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+import dotenv = require("dotenv")
 import { Command, CommandResult, CommandParams, PrepareParams } from "./base"
 import chalk from "chalk"
-import { sortBy } from "lodash"
+import { every, some, sortBy } from "lodash"
 import { ServiceLogEntry } from "../types/plugin/service/getServiceLogs"
 import Bluebird = require("bluebird")
 import { GardenService } from "../types/service"
 import Stream from "ts-stream"
 import { LoggerType, logLevelMap, LogLevel, parseLogLevel } from "../logger/logger"
-import { StringsParameter, BooleanParameter, IntegerParameter, DurationParameter } from "../cli/params"
+import { StringsParameter, BooleanParameter, IntegerParameter, DurationParameter, TagsOption } from "../cli/params"
 import { printHeader, renderDivider } from "../logger/util"
 import stripAnsi = require("strip-ansi")
 import hasAnsi = require("has-ansi")
-import { dedent } from "../util/string"
+import { dedent, deline } from "../util/string"
 import { padSection } from "../logger/renderers"
 import { PluginEventBroker } from "../plugin-context"
+import { ParameterError } from "../exceptions"
+import { isMatch } from "micromatch"
 
 const logsArgs = {
   services: new StringsParameter({
@@ -31,26 +34,40 @@ const logsArgs = {
 }
 
 const logsOpts = {
+  "tag": new TagsOption({
+    help: deline`
+      Only show log lines that match the given tag, e.g. \`--tag 'container=foo'\`. If you specify multiple filters
+      in a single tag option (e.g. \`--tag 'container=foo,someOtherTag=bar'\`), they must all be matched. If you
+      provide multiple \`--tag\` options (e.g. \`--tag 'container=api' --tag 'container=frontend'\`), they will be OR-ed
+      together (i.e. if any of them match, the log line will be included). You can specify glob-style wildcards, e.g.
+      \`--tag 'container=prefix-*'\`.
+    `,
+  }),
   "follow": new BooleanParameter({
     help: "Continuously stream new logs from the service(s).",
     alias: "f",
   }),
   "tail": new IntegerParameter({
-    help: dedent`
+    help: deline`
       Number of lines to show for each service. Defaults to showing all log lines (up to a certain limit). Takes precedence over
       the \`--since\` flag if both are set. Note that we don't recommend using a large value here when in follow mode.
     `,
     alias: "t",
   }),
+  // DEPRECATED, remove in 0.13 in favor of --show-tags
   "show-container": new BooleanParameter({
     help: "Show the name of the container with log output. May not apply to all providers",
     defaultValue: false,
   }),
+  "show-tags": new BooleanParameter({
+    help: "Show any tags attached to each log line. May not apply to all providers",
+    defaultValue: false,
+  }),
   "timestamps": new BooleanParameter({
     help: "Show timestamps with log output.",
   }),
   "since": new DurationParameter({
-    help: dedent`
+    help: deline`
       Only show logs newer than a relative duration like 5s, 2m, or 3h. Defaults to \`"1m"\` when \`--follow\` is true
       unless \`--tail\` is set. Note that we don't recommend using a large value here when in follow mode.
     `,
@@ -70,6 +87,10 @@ type Opts = typeof logsOpts
 
 export const colors = ["green", "cyan", "magenta", "yellow", "blueBright", "red"]
 
+type LogsTagFilter = [string, string]
+type LogsTagAndFilter = LogsTagFilter[]
+type LogsTagOrFilter = LogsTagAndFilter[]
+
 /**
  * Skip empty entries.
  */
@@ -88,12 +109,13 @@ export class LogsCommand extends Command<Args, Opts> {
 
     Examples:
 
-        garden logs                       # interleaves color-coded logs from all services (up to a certain limit)
-        garden logs --since 2d            # interleaves color-coded logs from all services from the last 2 days
-        garden logs --tail 100            # interleaves the last 100 log lines from all services
-        garden logs service-a,service-b   # interleaves color-coded logs for service-a and service-b
-        garden logs --follow              # keeps running and streams all incoming logs to the console
-        garden logs --original-color      # interleaves logs from all services and prints the original output color
+        garden logs                            # interleaves color-coded logs from all services (up to a certain limit)
+        garden logs --since 2d                 # interleaves color-coded logs from all services from the last 2 days
+        garden logs --tail 100                 # interleaves the last 100 log lines from all services
+        garden logs service-a,service-b        # interleaves color-coded logs for service-a and service-b
+        garden logs --follow                   # keeps running and streams all incoming logs to the console
+        garden logs --tag container=service-a  # only shows logs from containers with names matching the pattern
+        garden logs --original-color           # interleaves logs from all services and prints the original output color
   `
 
   arguments = logsArgs
@@ -118,14 +140,17 @@ export class LogsCommand extends Command<Args, Opts> {
   }
 
   async action({ garden, log, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<ServiceLogEntry[]>> {
-    const { follow, timestamps } = opts
+    const { follow, timestamps, tag } = opts
     let tail = opts.tail as number | undefined
     let since = opts.since as string | undefined
     const originalColor = opts["original-color"]
     const showContainer = opts["show-container"]
+    const showTags = opts["show-tags"]
     const hideService = opts["hide-service"]
     const logLevel = parseLogLevel(opts["log-level"])
 
+    let tagFilters: LogsTagOrFilter | undefined = undefined
+
     if (tail) {
       // Tail takes precedence over since...
       since = undefined
@@ -134,6 +159,23 @@ export class LogsCommand extends Command<Args, Opts> {
       since = "1m"
     }
 
+    if (tag && tag.length > 0) {
+      const parameterErrorMsg = `Unable to parse the given --tag flags. Format should be key=value.`
+      try {
+        tagFilters = tag.map((tagGroup: string) => {
+          return tagGroup.split(",").map((t: string) => {
+            const parsed = Object.entries(dotenv.parse(t))[0]
+            if (!parsed) {
+              throw new ParameterError(parameterErrorMsg, { tags: tag })
+            }
+            return parsed
+          })
+        })
+      } catch {
+        throw new ParameterError(parameterErrorMsg, { tags: tag })
+      }
+    }
+
     const graph = await garden.getConfigGraph({ log, emit: false })
     const allServices = graph.getServices()
     const services = args.services ? allServices.filter((s) => args.services?.includes(s.name)) : allServices
@@ -170,6 +212,19 @@ export class LogsCommand extends Command<Args, Opts> {
       return acc
     }, {})
 
+    const matchTagFilters = (entry: ServiceLogEntry): boolean => {
+      if (!tagFilters) {
+        return true
+      }
+      // We OR together the filter results of each tag option instance.
+      return some(tagFilters, (andFilter: LogsTagAndFilter) => {
+        // We AND together the filter results within a given tag option instance.
+        return every(andFilter, ([key, value]: LogsTagFilter) => {
+          return isMatch(entry.tags?.[key] || "", value)
+        })
+      })
+    }
+
     const formatEntry = (entry: ServiceLogEntry) => {
       const style = chalk[colorMap[entry.serviceName]]
       const sectionStyle = style.bold
@@ -178,6 +233,7 @@ export class LogsCommand extends Command<Args, Opts> {
 
       let timestamp: string | undefined
       let container: string | undefined
+      let tags: string | undefined
 
       if (timestamps && entry.timestamp) {
         timestamp = "                        "
@@ -186,8 +242,15 @@ export class LogsCommand extends Command<Args, Opts> {
         } catch {}
       }
 
-      if (showContainer && entry.containerName) {
-        container = entry.containerName
+      // DEPRECATED, remove in 0.13 in favor of --show-tags
+      if (showContainer && entry.tags?.container) {
+        container = entry.tags.container
+      }
+
+      if (showTags && entry.tags) {
+        tags = Object.entries(entry.tags)
+          .map(([k, v]) => `${k}=${v}`)
+          .join(" ")
       }
 
       if (entryLevel <= logLevel) {
@@ -205,6 +268,9 @@ export class LogsCommand extends Command<Args, Opts> {
       if (timestamp) {
         out += `${chalk.gray(timestamp)} → `
       }
+      if (tags) {
+        out += chalk.gray("[" + tags + "] ")
+      }
       if (originalColor) {
         // If the line doesn't have ansi encoding, we color it white to prevent logger from applying styles.
         out += hasAnsi(serviceLog) ? serviceLog : chalk.white(serviceLog)
@@ -221,6 +287,11 @@ export class LogsCommand extends Command<Args, Opts> {
         return
       }
 
+      // Match against all of the specified filters, if any
+      if (!matchTagFilters(entry)) {
+        return
+      }
+
       if (follow) {
         const levelStr = logLevelMap[entry.level || LogLevel.info] || "info"
         const msg = formatEntry(entry)

core/src/plugins/kubernetes/logs.ts

@@ -115,9 +115,9 @@ async function readLogs({
       try {
         const [timestampStr, msg] = splitFirst(line, " ")
         const timestamp = moment(timestampStr).toDate()
-        return { ...res, timestamp, msg }
+        return makeServiceLogEntry({ ...res, timestamp, msg })
       } catch {
-        return { ...res, msg: line }
+        return makeServiceLogEntry({ ...res, msg: line })
       }
     })
   })
@@ -430,13 +430,39 @@ export class K8sLogFollower {
     level?: LogLevel
     timestamp?: Date
   }) {
-    void this.stream.write({
-      serviceName: this.service.name,
-      timestamp,
-      msg,
-      containerName,
-      level,
-    })
+    void this.stream.write(
+      makeServiceLogEntry({
+        serviceName: this.service.name,
+        timestamp,
+        msg,
+        level,
+        containerName,
+      })
+    )
+  }
+}
+
+function makeServiceLogEntry({
+  serviceName,
+  msg,
+  containerName,
+  level,
+  timestamp,
+}: {
+  serviceName: string
+  msg: string
+  containerName?: string
+  level?: LogLevel
+  timestamp?: Date
+}): ServiceLogEntry {
+  return {
+    serviceName,
+    timestamp,
+    msg,
+    level,
+    tags: {
+      container: containerName || "",
+    },
   }
 }
 

core/src/types/plugin/service/getServiceLogs.ts

@@ -11,9 +11,8 @@ import { PluginServiceActionParamsBase, serviceActionParamsSchema } from "../bas
 import { dedent } from "../../../util/string"
 import { GardenModule } from "../../module"
 import { runtimeContextSchema } from "../../../runtime-context"
-import { joi } from "../../../config/common"
+import { joi, joiStringMap } from "../../../config/common"
 import { LogLevel } from "../../../logger/logger"
-import { string } from "@hapi/joi"
 
 export interface GetServiceLogsParams<M extends GardenModule = GardenModule, S extends GardenModule = GardenModule>
   extends PluginServiceActionParamsBase<M, S> {
@@ -29,7 +28,7 @@ export interface ServiceLogEntry {
   timestamp?: Date
   msg: string
   level?: LogLevel
-  containerName?: string
+  tags?: { [key: string]: string }
 }
 
 export const serviceLogEntrySchema = () =>
@@ -39,13 +38,13 @@ export const serviceLogEntrySchema = () =>
       serviceName: joi.string().required().description("The name of the service the log entry originated from."),
       timestamp: joi.date().required().description("The time when the log entry was generated by the service."),
       msg: joi.string().required().description("The content of the log entry."),
-      containerName: string().description("The name of the container the service runs, if appicable."),
       level: joi.string().description(
         dedent`
           The log level of the entry. The 'info' level should be reserved for logs from the service proper.
           Other levels can be used to print warnings or debug information from the plugin.
         `
       ),
+      tags: joiStringMap(joi.string()).description("Tags used for later filtering in the logs command."),
     })
     .description("A log entry returned by a getServiceLogs action handler.")