Prometheus support (#450)

Co-authored-by: nils måsén <[email protected]> Co-authored-by: MihailITPlace <[email protected]> Co-authored-by: Sebastiaan Tammer <[email protected]>
containrrr · Jan 6, 2021 · d7d5b25 · d7d5b25
1 parent 35490c8
commit d7d5b25
Show file tree

Hide file tree

Showing 23 changed files with 812 additions and 99 deletions.
diff --git a/cmd/root.go b/cmd/root.go
@@ -1,12 +1,16 @@
 package cmd
 
 import (
+	metrics2 "github.com/containrrr/watchtower/pkg/metrics"
 	"os"
 	"os/signal"
 	"strconv"
 	"syscall"
 	"time"
 
+	"github.com/containrrr/watchtower/pkg/api/metrics"
+	"github.com/containrrr/watchtower/pkg/api/update"
+
 	"github.com/containrrr/watchtower/internal/actions"
 	"github.com/containrrr/watchtower/internal/flags"
 	"github.com/containrrr/watchtower/pkg/api"
@@ -144,7 +148,10 @@ func PreRun(cmd *cobra.Command, args []string) {
 func Run(c *cobra.Command, names []string) {
 	filter := filters.BuildFilter(names, enableLabel, scope)
 	runOnce, _ := c.PersistentFlags().GetBool("run-once")
-	httpAPI, _ := c.PersistentFlags().GetBool("http-api")
+	enableUpdateAPI, _ := c.PersistentFlags().GetBool("http-api-update")
+	enableMetricsAPI, _ := c.PersistentFlags().GetBool("http-api-metrics")
+
+	apiToken, _ := c.PersistentFlags().GetString("http-api-token")
 
 	if runOnce {
 		if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage {
@@ -160,17 +167,20 @@ func Run(c *cobra.Command, names []string) {
 		log.Fatal(err)
 	}
 
-	if httpAPI {
-		apiToken, _ := c.PersistentFlags().GetString("http-api-token")
+	httpAPI := api.New(apiToken)
 
-		if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil {
-			log.Fatal(err)
-			os.Exit(1)
-		}
+	if enableUpdateAPI {
+		updateHandler := update.New(func() { runUpdatesWithNotifications(filter) })
+		httpAPI.RegisterFunc(updateHandler.Path, updateHandler.Handle)
+	}
 
-		api.WaitForHTTPUpdates()
+	if enableMetricsAPI {
+		metricsHandler := metrics.New()
+		httpAPI.RegisterHandler(metricsHandler.Path, metricsHandler.Handle)
 	}
 
+	httpAPI.Start(enableUpdateAPI)
+
 	if err := runUpgradesOnSchedule(c, filter); err != nil {
 		log.Error(err)
 	}
@@ -189,8 +199,11 @@ func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error {
 			select {
 			case v := <-tryLockSem:
 				defer func() { tryLockSem <- v }()
-				runUpdatesWithNotifications(filter)
+				metric := runUpdatesWithNotifications(filter)
+				metrics2.RegisterScan(metric)
 			default:
+				// Update was skipped
+				metrics2.RegisterScan(nil)
 				log.Debug("Skipped another update already running.")
 			}
 
@@ -222,7 +235,8 @@ func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error {
 	return nil
 }
 
-func runUpdatesWithNotifications(filter t.Filter) {
+func runUpdatesWithNotifications(filter t.Filter) *metrics2.Metric {
+
 	notifier.StartNotification()
 	updateParams := t.UpdateParams{
 		Filter:         filter,
@@ -233,9 +247,10 @@ func runUpdatesWithNotifications(filter t.Filter) {
 		LifecycleHooks: lifecycleHooks,
 		RollingRestart: rollingRestart,
 	}
-	err := actions.Update(client, updateParams)
+	metrics, err := actions.Update(client, updateParams)
 	if err != nil {
 		log.Println(err)
 	}
 	notifier.SendNotification()
+	return metrics
 }
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,43 @@
+version: '3.7'
+
+services:
+  watchtower:
+    container_name: watchtower
+    build:
+      context: ./
+      dockerfile: dockerfiles/Dockerfile.dev-self-contained
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    ports:
+      - 8080:8080
+    command: --interval 10 --http-api-metrics --http-api-token demotoken --debug prometheus grafana parent child
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+      - prometheus:/prometheus/
+    ports:
+      - 9090:9090
+  grafana:
+    container_name: grafana
+    image: grafana/grafana
+    ports:
+      - 3000:3000
+    environment:
+      GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource
+    volumes:
+      - grafana:/var/lib/grafana
+      - ./grafana:/etc/grafana/provisioning
+  parent:
+    image: nginx
+    container_name: parent
+  child:
+    image: nginx:alpine
+    labels:
+      com.centurylinklabs.watchtower.depends-on: parent
+    container_name: child
+
+volumes:
+  prometheus: {}
+  grafana: {}
diff --git a/docs/arguments.md b/docs/arguments.md
@@ -164,7 +164,7 @@ Environment Variable: WATCHTOWER_LABEL_ENABLE
 ## Without updating containers
 Will only monitor for new images, send notifications and invoke the [pre-check/post-check hooks](https://containrrr.dev/watchtower/lifecycle-hooks/), but will **not** update the containers.
 
-> ### ⚠️ Please note
+> **⚠️ Please note**
 >
 > Due to Docker API limitations the latest image will still be pulled from the registry.
 
@@ -238,9 +238,7 @@ Sets an authentication token to HTTP API requests.
 Environment Variable: WATCHTOWER_HTTP_API_TOKEN
                 Type: String
              Default: -
-```
-
-## Filter by scope
+```## Filter by scope
 Update containers that have a `com.centurylinklabs.watchtower.scope` label set with the same value as the given argument. This enables [running multiple instances](https://containrrr.github.io/watchtower/running-multiple-instances).
 
 ```
@@ -250,6 +248,16 @@ Environment Variable: WATCHTOWER_SCOPE
              Default: -
 ``` 
 
+## HTTP API Metrics
+Enables a metrics endpoint, exposing prometheus metrics via HTTP. See [Metrics](metrics.md) for details.  
+
+```
+            Argument: --http-api-metrics
+Environment Variable: WATCHTOWER_HTTP_API_METRICS
+                Type: Boolean
+             Default: false
+```
+
 ## Scheduling
 [Cron expression](https://pkg.go.dev/github.com/robfig/[email protected]?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression
 can be defined, but not both. An example: `--schedule "0 0 4 * * *"`

diff --git a/docs/assets/grafana-dashboard.png b/docs/assets/grafana-dashboard.png
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -0,0 +1,26 @@
+> **⚠️ Experimental feature**
+> 
+> This feature was added in v1.0.4 and is still considered experimental.
+> If you notice any strange behavior, please raise a ticket in the repository issues.
+
+Metrics can be used to track how Watchtower behaves over time.
+
+To use this feature, you have to set an [API token](arguments.md#http-api-token) and [enable the metrics API](arguments.md#http-api-metrics),
+as well as creating a port mapping for your container for port `8080`.
+
+## Available Metrics 
+
+| Name                            | Type    | Description                                                                 |
+| ------------------------------- | ------- | --------------------------------------------------------------------------- |
+| `watchtower_containers_scanned` | Gauge   | Number of containers scanned for changes by watchtower during the last scan |
+| `watchtower_containers_updated` | Gauge   | Number of containers updated by watchtower during the last scan             |
+| `watchtower_containers_failed`  | Gauge   | Number of containers where update failed during the last scan               |
+| `watchtower_scans_total`        | Counter | Number of scans since the watchtower started                                |
+| `watchtower_scans_skipped`      | Counter | Number of skipped scans since watchtower started                            |
+
+## Demo
+
+The repository contains a demo with prometheus and grafana, available through `docker-compose.yml`. This demo
+is preconfigured with a dashboard, which will look something like this:
+
+![grafana metrics](assets/grafana-dashboard.png)
diff --git a/docs/notifications.md b/docs/notifications.md
@@ -1,4 +1,3 @@
-
 # Notifications
 
 Watchtower can send notifications when containers are updated. Notifications are sent via hooks in the logging system, [logrus](http://github.com/sirupsen/logrus).
@@ -12,14 +11,14 @@ The types of notifications to send are set by passing a comma-separated list of
 
 > There is currently a [bug](https://github.com/spf13/viper/issues/380) in Viper, which prevents comma-separated slices to be used when using the environment variable. A workaround is available where we instead put quotes around the environment variable value and replace the commas with spaces, as `WATCHTOWER_NOTIFICATIONS="slack msteams"`
 
-> If you're a `docker-compose` user, make sure to specify environment variables' values in your `.yml` file without double quotes (`"`). 
+> If you're a `docker-compose` user, make sure to specify environment variables' values in your `.yml` file without double quotes (`"`).
 >
 > This prevents unexpected errors when watchtower starts.
 
 ## Settings
 
 - `--notifications-level` (env. `WATCHTOWER_NOTIFICATIONS_LEVEL`): Controls the log level which is used for the notifications. If omitted, the default log level is `info`. Possible values are: `panic`, `fatal`, `error`, `warn`, `info`, `debug` or `trace`.
-- Watchtower will post a notification every time it is started. This behavior [can be changed](https://containrrr.github.io/watchtower/arguments/#without_sending_a_startup_message) with an argument. 
+- Watchtower will post a notification every time it is started. This behavior [can be changed](https://containrrr.github.io/watchtower/arguments/#without_sending_a_startup_message) with an argument.
 
 ## Available services
 
@@ -47,7 +46,7 @@ docker run -d \
   -e [email protected] \
   -e [email protected] \
   -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER=smtp.gmail.com \
-  -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT=587 \  
+  -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT=587 \
   -e [email protected] \
   -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PASSWORD=app_password \
   -e WATCHTOWER_NOTIFICATION_EMAIL_DELAY=2 \
@@ -56,19 +55,19 @@ docker run -d \
 
 The previous example assumes, that you already have an SMTP server up and running you can connect to. If you don't or you want to bring up watchtower with your own simple SMTP relay the following `docker-compose.yml` might be a good start for you.
 
-The following example assumes, that your domain is called `your-domain.com` and that you are going to use a certificate valid for `smtp.your-domain.com`. This hostname has to be used as `WATCHTOWER_NOTIFICATION_EMAIL_SERVER` otherwise the TLS connection is going to fail with  `Failed to send notification email` or `connect: connection refused`. We also have to add a network for this setup in order to add an alias to it. If you also want to enable DKIM or other features on the SMTP server, you will find more information at [freinet/postfix-relay](https://hub.docker.com/r/freinet/postfix-relay).
+The following example assumes, that your domain is called `your-domain.com` and that you are going to use a certificate valid for `smtp.your-domain.com`. This hostname has to be used as `WATCHTOWER_NOTIFICATION_EMAIL_SERVER` otherwise the TLS connection is going to fail with `Failed to send notification email` or `connect: connection refused`. We also have to add a network for this setup in order to add an alias to it. If you also want to enable DKIM or other features on the SMTP server, you will find more information at [freinet/postfix-relay](https://hub.docker.com/r/freinet/postfix-relay).
 
 Example including an SMTP relay:
 
 ```yaml
 ---
-version: "3.8"
+version: '3.8'
 services:
   watchtower:
     image: containrrr/watchtower:latest
     container_name: watchtower
     environment:
-      WATCHTOWER_MONITOR_ONLY: "true"
+      WATCHTOWER_MONITOR_ONLY: 'true'
       WATCHTOWER_NOTIFICATIONS: email
       WATCHTOWER_NOTIFICATION_EMAIL_FROM: [email protected]
       WATCHTOWER_NOTIFICATION_EMAIL_TO: [email protected]
@@ -90,9 +89,9 @@ services:
       - 25
     environment:
       MAILNAME: somename.your-domain.com
-      TLS_KEY: "/etc/ssl/domains/your-domain.com/your-domain.com.key"
-      TLS_CRT: "/etc/ssl/domains/your-domain.com/your-domain.com.crt"
-      TLS_CA: "/etc/ssl/domains/your-domain.com/intermediate.crt"
+      TLS_KEY: '/etc/ssl/domains/your-domain.com/your-domain.com.key'
+      TLS_CRT: '/etc/ssl/domains/your-domain.com/your-domain.com.crt'
+      TLS_CA: '/etc/ssl/domains/your-domain.com/intermediate.crt'
     volumes:
       - /etc/ssl/domains/your-domain.com/:/etc/ssl/domains/your-domain.com/:ro
     networks:
@@ -172,7 +171,7 @@ docker run -d \
 
 `-e WATCHTOWER_NOTIFICATION_GOTIFY_TOKEN` or `--notification-gotify-token` can also reference a file, in which case the contents of the file are used.
 
-If you want to disable TLS verification for the Gotify instance, you can use either `-e WATCHTOWER_NOTIFICATION_GOTIFY_TLS_SKIP_VERIFY=true` or `--notification-gotify-tls-skip-verify`. 
+If you want to disable TLS verification for the Gotify instance, you can use either `-e WATCHTOWER_NOTIFICATION_GOTIFY_TLS_SKIP_VERIFY=true` or `--notification-gotify-tls-skip-verify`.
 
 ### [containrrr/shoutrrr](https://github.com/containrrr/shoutrrr)
 

diff --git a/go.mod b/go.mod
@@ -46,6 +46,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/pkg/errors v0.8.1 // indirect
+	github.com/prometheus/client_golang v0.9.3
 	github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967
 	github.com/sirupsen/logrus v1.4.1
 	github.com/spf13/cobra v0.0.3