add docker hub ratelimit metric

.github/workflows/build-docker-sha.yaml

@@ -29,7 +29,7 @@ jobs:
       - uses: docker/metadata-action@v5
         id: meta
         with:
-          images: ghcr.io/whywaita/myshoes
+          images: ghcr.io/${{ github.repository_owner }}/myshoes
           tags: |
             type=sha
       - name: Build container image

docs/01_01_for_admin_setup.md

@@ -106,6 +106,17 @@ A config variables can set from environment values.
   - set linux username that executes runner. you need to set exist user.
     - DO NOT set root. It can't run GitHub Actions runner in root permission.
     - Example: `ubuntu`
+- `PROVIDE_DOCKER_HUB_METRICS`
+  - default: `false`
+  - set `true` if you want to provide rate-limit metrics for Docker Hub.
+  - If you're not anonymous user, you need to set `DOCKER_HUB_USERNAME` and `DOCKER_HUB_PASSWORD`.
+- `DOCKER_HUB_USERNAME`
+  - default: `` (empty)
+  - set Docker Hub username for pulling Docker image. (Use for provide rate-limit metrics)
+- `DOCKER_HUB_PASSWORD`
+  - default: `` (empty)
+  - set Docker Hub password for pulling Docker image. (Use for provide rate-limit metrics)
+
 
 For tuning values
 

pkg/config/config.go

@@ -27,6 +27,15 @@ type Conf struct {
 
 	GitHubURL     string
 	RunnerVersion string
+
+	DockerHubCredential DockerHubCredential
+	ProvideDockerHubMetrics bool
+}
+
+// DockerHubCredential is type of config value
+type DockerHubCredential struct {
+	Username string
+	Password string
 }
 
 // GitHubApp is type of config value
@@ -54,6 +63,9 @@ const (
 	EnvMaxConcurrencyDeleting    = "MAX_CONCURRENCY_DELETING"
 	EnvGitHubURL                 = "GITHUB_URL"
 	EnvRunnerVersion             = "RUNNER_VERSION"
+	EnvDockerHubUsername         = "DOCKER_HUB_USERNAME"
+	EnvDockerHubPassword         = "DOCKER_HUB_PASSWORD"
+	EnvProvideDockerHubMetrics   = "PROVIDE_DOCKER_HUB_METRICS"
 )
 
 // ModeWebhookType is type value for GitHub webhook

pkg/config/init.go

@@ -75,6 +75,17 @@ func LoadWithDefault() Conf {
 		c.ModeWebhookType = mwt
 	}
 
+	c.ProvideDockerHubMetrics = false
+	if os.Getenv(EnvProvideDockerHubMetrics) == "true" {
+		c.ProvideDockerHubMetrics = true
+	}
+
+	c.DockerHubCredential = DockerHubCredential{}
+	if os.Getenv(EnvDockerHubUsername) != "" && os.Getenv(EnvDockerHubPassword) != "" {
+		c.DockerHubCredential.Username = os.Getenv(EnvDockerHubUsername)
+		c.DockerHubCredential.Password = os.Getenv(EnvDockerHubPassword)
+	}
+
 	c.MaxConnectionsToBackend = 50
 	if os.Getenv(EnvMaxConnectionsToBackend) != "" {
 		numberPB, err := strconv.ParseInt(os.Getenv(EnvMaxConnectionsToBackend), 10, 64)

pkg/docker/ratelimit.go

@@ -0,0 +1,106 @@
+package docker
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang-jwt/jwt/v4"
+	"github.com/whywaita/myshoes/pkg/config"
+)
+
+// RateLimit is Docker Hub API rate limit
+type RateLimit struct {
+	Limit     int
+	Remaining int
+}
+
+type tokenCache struct {
+	expire time.Time
+	token  string
+}
+
+var cacheMap = make(map[int]tokenCache, 1)
+
+func getToken() (string, error) {
+	url := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull"
+	req, err := http.NewRequest("GET", url, nil)
+	if config.Config.DockerHubCredential.Password != "" && config.Config.DockerHubCredential.Username != "" {
+		req.SetBasicAuth(config.Config.DockerHubCredential.Username, config.Config.DockerHubCredential.Password)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("request token: %w", err)
+	}
+	if cache, ok := cacheMap[0]; ok && cache.expire.After(time.Now()) {
+		return cache.token, nil
+	}
+	defer resp.Body.Close()
+	byteArray, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("read body: %w", err)
+	}
+	jsonMap := make(map[string]interface{})
+	if err := json.Unmarshal(byteArray, &jsonMap); err != nil {
+		return "", fmt.Errorf("unmarshal json: %w", err)
+	}
+	tokenString, ok := jsonMap["token"].(string)
+	if !ok {
+		return "", fmt.Errorf("tokenString is not string")
+	}
+	token, _, err := new(jwt.Parser).ParseUnverified(tokenString, jwt.MapClaims{})
+	if err != nil {
+		return "", fmt.Errorf("parse token: %w", err)
+	}
+	exp, ok := token.Claims.(jwt.MapClaims)["exp"].(float64)
+	if !ok {
+		return "", fmt.Errorf("exp is not float64")
+	}
+	cacheMap[0] = tokenCache{
+		expire: time.Unix(int64(exp), 0),
+		token:  tokenString,
+	}
+	return tokenString, nil
+}
+
+// GetRateLimit get Docker Hub API rate limit
+func GetRateLimit() (RateLimit, error) {
+	token, err := getToken()
+	if err != nil {
+		return RateLimit{}, fmt.Errorf("get token: %w", err)
+	}
+	url := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest"
+	req, err := http.NewRequest("HEAD", url, nil)
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return RateLimit{}, fmt.Errorf("get rate limit: %w", err)
+	}
+	defer resp.Body.Close()
+	limitHeader := resp.Header.Get("ratelimit-limit")
+	if limitHeader == "" {
+		return RateLimit{}, fmt.Errorf("not found ratelimit-limit header")
+	}
+	limit, err := strconv.Atoi(strings.Split(limitHeader, ";")[0])
+	if err != nil {
+		return RateLimit{}, fmt.Errorf("parse limit: %w", err)
+	}
+	remainingHeader := resp.Header.Get("ratelimit-remaining")
+	if remainingHeader == "" {
+		return RateLimit{}, fmt.Errorf("not found ratelimit-remaining header")
+
+	}
+	remaining, err := strconv.Atoi(strings.Split(remainingHeader, ";")[0])
+	if err != nil {
+		return RateLimit{}, fmt.Errorf("parse remaining: %w", err)
+	}
+
+	return RateLimit{
+		Limit:     limit,
+		Remaining: remaining,
+	}, nil
+}

pkg/metric/scrape_memory.go

@@ -7,6 +7,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
+	"github.com/whywaita/myshoes/pkg/docker"
 	"github.com/whywaita/myshoes/pkg/gh"
 	"github.com/whywaita/myshoes/pkg/runner"
 	"github.com/whywaita/myshoes/pkg/starter"
@@ -37,14 +38,24 @@ var (
 	)
 	memoryGitHubRateLimitRemaining = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_remaining"),
-		"The number of rate limit remaining",
+		"The number of rate limit remaining in GitHub",
 		[]string{"scope"}, nil,
 	)
 	memoryGitHubRateLimitLimiting = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, memoryName, "github_rate_limit_limiting"),
-		"The number of rate limit max",
+		"The number of rate limit max in GitHub",
 		[]string{"scope"}, nil,
 	)
+	memoryDockerHubRateLimitRemaining = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, memoryName, "dockerhub_rate_limit_remaining"),
+		"The number of rate limit remaining in DockerHub",
+		[]string{}, nil,
+	)
+	memoryDockerHubRateLimitLimiting = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, memoryName, "dockerhub_rate_limit_limiting"),
+		"The number of rate limit max in DockerHub",
+		[]string{}, nil,
+	)
 	memoryRunnerMaxConcurrencyDeleting = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, memoryName, "runner_max_concurrency_deleting"),
 		"The number of max concurrency deleting in runner (Config)",
@@ -81,7 +92,11 @@ func (ScraperMemory) Scrape(ctx context.Context, ds datastore.Datastore, ch chan
 	if err := scrapeRecoveredRuns(ch); err != nil {
 		return fmt.Errorf("failed to scrape recovered runs: %w", err)
 	}
-
+	if config.Config.ProvideDockerHubMetrics {
+		if err := scrapeDockerValues(ch); err != nil {
+			return fmt.Errorf("failed to scrape Docker values: %w", err)
+		}
+	}
 	return nil
 }
 
@@ -142,3 +157,17 @@ func scrapeGitHubValues(ch chan<- prometheus.Metric) error {
 }
 
 var _ Scraper = ScraperMemory{}
+
+func scrapeDockerValues(ch chan<- prometheus.Metric) error {
+	rateLimit, err := docker.GetRateLimit()
+	if err != nil {
+		return fmt.Errorf("failed to get rate limit: %w", err)
+	}
+	ch <- prometheus.MustNewConstMetric(
+		memoryDockerHubRateLimitRemaining, prometheus.GaugeValue, float64(rateLimit.Remaining),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		memoryDockerHubRateLimitLimiting, prometheus.GaugeValue, float64(rateLimit.Limit),
+	)
+	return nil
+}