Merge pull request #1 from wbollock/feat/init_o11y_e2e

feat: init o11y_e2e s/g
linode-obs · Dec 18, 2023 · f3c3ad9 · f3c3ad9
2 parents e36f64c + fe51c5b
commit f3c3ad9
Show file tree

Hide file tree

Showing 8 changed files with 305 additions and 55 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,8 +8,8 @@
 *.so
 *.dylib
 
-# go_project_template binary
-go_project_template
+# o11y_e2e binary
+o11y_e2e
 
 # Test binary, built with `go test -c`
 *.test

diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -52,8 +52,8 @@ dockers:
     - "--label=org.opencontainers.image.title={{.ProjectName}}"
     - "--label=org.opencontainers.image.revision={{.FullCommit}}"
     - "--label=org.opencontainers.image.version={{.Version}}"
-    - "--label=org.opencontainers.image.source=https://github.com/linode-obs/go_project_template"
-    - "--label=org.opencontainers.image.description='go_project_template description TODO'"
+    - "--label=org.opencontainers.image.source=https://github.com/linode-obs/o11y_e2e"
+    - "--label=org.opencontainers.image.description='o11y_e2e description TODO'"
     - "--label=org.opencontainers.image.licenses=MIT"
     - "--platform=linux/amd64"
     dockerfile: "Dockerfile"
@@ -68,8 +68,8 @@ dockers:
     - "--label=org.opencontainers.image.title={{.ProjectName}}"
     - "--label=org.opencontainers.image.revision={{.FullCommit}}"
     - "--label=org.opencontainers.image.version={{.Version}}"
-    - "--label=org.opencontainers.image.source=https://github.com/linode-obs/go_project_template"
-    - "--label=org.opencontainers.image.description='go_project_template description TODO'"
+    - "--label=org.opencontainers.image.source=https://github.com/linode-obs/o11y_e2e"
+    - "--label=org.opencontainers.image.description='o11y_e2e description TODO'"
     - "--label=org.opencontainers.image.licenses=MIT"
     - "--platform=linux/arm64"
     dockerfile: "Dockerfile"
@@ -78,7 +78,7 @@ nfpms:
      formats:
       - deb
       - rpm
-     package_name: "go_project_template"
+     package_name: "o11y_e2e"
      section: "default"
      priority: "extra"
      replaces: []
@@ -88,18 +88,18 @@ nfpms:
      conflicts: []
      maintainer: "Will Bollock <[email protected]>"
      description: |
-       go_project_template description TODO
-     homepage: "https://github.com/linode-obs/go_project_template"
+       o11y_e2e description TODO
+     homepage: "https://github.com/linode-obs/o11y_e2e"
      license: "MIT"
      contents:
      # provided by goreleaser
-     - src: ./dist/go_project_template_linux_amd64_v1/go_project_template
-       dst: /usr/local/bin/go_project_template
-     - src: ./nfpm/systemd/go_project_template.service
-       dst: /etc/systemd/system/go_project_template.service
+     - src: ./dist/o11y_e2e_linux_amd64_v1/o11y_e2e
+       dst: /usr/local/bin/o11y_e2e
+     - src: ./nfpm/systemd/o11y_e2e.service
+       dst: /etc/systemd/system/o11y_e2e.service
        type: config
-     - src: ./nfpm/etc/default/go_project_template
-       dst: /etc/default/go_project_template
+     - src: ./nfpm/etc/default/o11y_e2e
+       dst: /etc/default/o11y_e2e
        type: config
      scripts:
         postinstall: ./nfpm/scripts/postinstall.sh
diff --git a/Dockerfile b/Dockerfile
@@ -2,16 +2,16 @@ ARG ARCH="amd64"
 ARG OS="linux"
 FROM golang:alpine3.18 AS builderimage
 LABEL maintainer="Akamai SRE Observability Team <[email protected]>"
-WORKDIR /go/src/go_project_template
+WORKDIR /go/src/o11y_e2e
 COPY . .
-RUN go build -o go_project_template cmd/main.go
+RUN go build -o o11y_e2e cmd/main.go
 
 ###################################################################
 
 FROM golang:alpine3.18
-COPY --from=builderimage /go/src/go_project_template/go_project_template /app/
+COPY --from=builderimage /go/src/o11y_e2e/o11y_e2e /app/
 WORKDIR /app
 
-EXPOSE      9141
+EXPOSE      9927
 USER        nobody
-ENTRYPOINT  [ "./go_project_template" ]
+ENTRYPOINT  [ "./o11y_e2e" ]
diff --git a/README.md b/README.md
@@ -1,29 +1,62 @@
-This is an opinionated Golang project template skeleton. Replace linode-obs/go_project_template with your project. Sometimes just go_project_template will need to be replaced. Also search TODO.
-
-Make sure to write tests and `pre-commit install`. You'll also want to install [goreleaser](https://goreleaser.com/) and [pre-commit](https://pre-commit.com/). Note that the MIT license is included too.
-
-Beware that:
-
-* The Go version is set at `1.21` in [golang-tests.yaml](.github/workflows/golang-tests.yaml). This could be automated with renovate or dependabot
-* Markdown table of contents be nice to add, easy to create with an [extension](https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one) in your editor of choice
-* Document your CLI values and metrics instrumented
-
-# go_project_template
-
-![Github Release Downloads](https://img.shields.io/github/downloads/linode-obs/go_project_template/total.svg)
-[![license](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/linode-obs/go_project_template/blob/master/LICENSE)
-[![golangci-lint](https://github.com/linode-obs/go_project_template/actions/workflows/golangci-lint.yaml/badge.svg)](https://github.com/linode-obs/go_project_template/actions/workflows/golangci-lint.yaml)
-![Go Report Card](https://goreportcard.com/badge/github.com/linode-obs/go_project_template)
-[![contributions](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat")](https://github.com/linode-obs/go_project_template/issues)
-
-Project description.
-
-We recommend using a standard [go project layout](https://github.com/golang-standards/project-layout) too.
+# o11y_e2e
+
+![Github Release Downloads](https://img.shields.io/github/downloads/linode-obs/o11y_e2e/total.svg)
+[![license](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/linode-obs/o11y_e2e/blob/master/LICENSE)
+[![golangci-lint](https://github.com/linode-obs/o11y_e2e/actions/workflows/golangci-lint.yaml/badge.svg)](https://github.com/linode-obs/o11y_e2e/actions/workflows/golangci-lint.yaml)
+![Go Report Card](https://goreportcard.com/badge/github.com/linode-obs/o11y_e2e)
+[![contributions](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat")](https://github.com/linode-obs/o11y_e2e/issues)
+
+The best way to measure pipeline effectiveness, especially for SLOs, is examining behavior from the end user's perspective. This project aims to simplify that by ingesting a log into systems like Loki and determining how long it takes for that metric to be query-able as an end-user.
+
+This project is inspired by Alex Hidalgo's *[Implementing Service Level Objectives](https://learning.oreilly.com/library/view/implementing-service-level/9781492076803/ch01.html#:-:text=you%20should%20consider%20measuring,reaches%20the%20other%20end)*:
+
+>... even better would be to measure how much time elapses after inserting a particular record into the pipeline before you can retrieve that exact record at the other end
+
+This project generates metrics on an internal interval rather than on-scrape as this type of data is not always going to be a good fit during Prometheus scrape periods (~`2m` is common).
+
+- [o11y\_e2e](#o11y_e2e)
+  - [Example Configuration](#example-configuration)
+  - [Installation](#installation)
+    - [Debian/RPM package](#debianrpm-package)
+    - [Docker](#docker)
+    - [Binary](#binary)
+    - [Source](#source)
+  - [Potential Features](#potential-features)
+    - [Proposed features](#proposed-features)
+    - [Vaguely planned features](#vaguely-planned-features)
+    - [Not planned features](#not-planned-features)
+  - [Releasing](#releasing)
+  - [Contributors](#contributors)
+
+## Example Configuration
+
+```yaml
+o11y_e2e:
+  - name: loki_prod
+      type: loki # only loki is currently supported
+      enabled: true
+      url: "https://my_loki_api_url.com"
+      timeout: "1m" # how long to wait before giving up querying the metric
+      tls_config: # tls.Config standard options
+          insecure_skip_verify: false
+      labels:
+      - cluster: infra-logging-atl1-us-staging
+        dc: atl1
+        environment: production
+      syslog_push: # rather than just generate log to stdout/journalctl, send logs to syslog reciever
+          enabled: true # TODO - document defaults and structure better
+          url: "my_syslog_reciever.com"
+          tls_config:
+            insecure_skip_verify: false
+  - name: loki_prod_region2
+    # etc..
+```
 
-Features:
+These config options will result in a logfmt log line of:
 
-* Feature 1
-* Feature 2
+```console
+creation_time=11:11:11, level=info, application=o11y_e2e, hash=MYHASH1234AFFF, cluster=infra-logging-atl1-us-staging, dc=atl1, environment=production, syslog=true, syslog_url="my_syslog_reciever.com"
+```
 
 ## Installation
 
@@ -32,36 +65,54 @@ Features:
 Substitute `{{ version }}` for your desired release.
 
 ```bash
-wget https://github.com/linode-obs/go_project_template/releases/download/v{{ version }}/go_project_template_{{ version }}_linux_amd64.{deb,rpm}
-{dpkg,rpm} -i go_project_template_{{ version }}_linux_amd64.{deb,rpm}
+wget https://github.com/linode-obs/o11y_e2e/releases/download/v{{ version }}/o11y_e2e_{{ version }}_linux_amd64.{deb,rpm}
+{dpkg,rpm} -i o11y_e2e_{{ version }}_linux_amd64.{deb,rpm}
 ```
 
 ### Docker
 
 ```console
 sudo docker run \
 --privileged \
-ghcr.io/linode-obs/go_project_template
+ghcr.io/linode-obs/o11y_e2e
 ```
 
 ### Binary
 
 ```bash
-wget https://github.com/linode-obs/go_project_template/releases/download/v{{ version }}/go_project_template_{{ version }}_Linux_x86_64.tar.gz
-tar xvf go_project_template_{{ version }}_Linux_x86_64.tar.gz
-./go_project_template/go_project_template
+wget https://github.com/linode-obs/o11y_e2e/releases/download/v{{ version }}/o11y_e2e_{{ version }}_Linux_x86_64.tar.gz
+tar xvf o11y_e2e_{{ version }}_Linux_x86_64.tar.gz
+./o11y_e2e/o11y_e2e
 ```
 
 ### Source
 
 ```bash
-wget https://github.com/linode-obs/go_project_template/archive/refs/tags/v{{ version }}.tar.gz
-tar xvf go_project_template-{{ version }}.tar.gz
-cd ./go_project_template-{{ version }}
-go build go_project_template.go
-./go_project_template.go
+wget https://github.com/linode-obs/o11y_e2e/archive/refs/tags/v{{ version }}.tar.gz
+tar xvf o11y_e2e-{{ version }}.tar.gz
+cd ./o11y_e2e-{{ version }}
+go build o11y_e2e.go
+./o11y_e2e.go
 ```
 
+## Potential Features
+
+### Proposed features
+
+- Loki support
+- Generate a unique log and provide a slew of Prometheus metrics on the latency and durability of that log message at the end of the pipeline
+- Support syslog-style "push" to a pipeline entry point rather than rely on unknown external sending feature like promtail or otelcol agent
+- Support multiple pipelines from one instance of o11y_e2e
+
+### Vaguely planned features
+
+- Expansion into metrics pipelines like Thanos, Mimir, Victoria Metrics
+- Support for elasticsearch or other log aggregators
+
+### Not planned features
+
+- Prometheus multi target exporter pattern as the data generation can be too slow and managing both internal metric generation and on-demand scrapes is painful
+
 ## Releasing
 
 1. Merge commits to main.

diff --git a/cmd/main.go b/cmd/main.go
@@ -0,0 +1,88 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"o11y_e2e/internal/server"
+	"os"
+
+	"log/slog"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+	defaultLogLevel      = "info"
+	defaultListenAddress = "0.0.0.0:9927"
+	defaultMetricsPath   = "/metrics"
+)
+
+var (
+	listenAddress = flag.String("web.listen-address", defaultListenAddress, "Address to listen on for telemetry")
+	showVersion   = flag.Bool("version", false, "show version information")
+	logLevel      = flag.String("log.level", defaultLogLevel,
+		"Minimum Log level [info, debug, warn, error]")
+
+	// Build info for o11y_e2e itself, will be populated by linker during build
+	Version   string
+	BuildDate string
+	Commit    string
+
+	versionInfo = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "o11y_e2e_version_info",
+			Help: "o11y_e2e build information",
+		},
+		[]string{"version", "commit", "builddate"},
+	)
+)
+
+func printVersion() {
+	fmt.Printf("o11y_e2e\n")
+	fmt.Printf("Version:   %s\n", Version)
+	fmt.Printf("BuildDate: %s\n", BuildDate)
+	fmt.Printf("Commit:    %s\n", Commit)
+	fmt.Printf("o11y_e2e pipeline measurement tool\n")
+}
+
+func main() {
+	flag.Parse()
+
+	if *showVersion {
+		printVersion()
+		os.Exit(0)
+	}
+
+	versionInfo.WithLabelValues(Version, Commit, BuildDate).Set(1)
+	prometheus.MustRegister(versionInfo)
+
+	slogLogLevel := new(slog.LevelVar)
+	switch *logLevel {
+	case "debug":
+		slogLogLevel.Set(-4)
+	case "warn":
+		slogLogLevel.Set(4)
+	case "error":
+		slogLogLevel.Set(8)
+	default:
+		slogLogLevel.Set(0)
+	}
+
+	opts := &slog.HandlerOptions{
+		Level: slogLogLevel,
+	}
+
+	handler := slog.NewTextHandler(os.Stdout, opts)
+
+	slog.SetDefault(slog.New(handler))
+
+	http.Handle(defaultMetricsPath, promhttp.Handler())
+	http.Handle("/", server.SetupServer())
+
+	slog.Info("Starting server", "address", *listenAddress)
+	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
+		slog.Error("Failed to start the server", "error", err)
+	}
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,20 @@
+module o11y_e2e
+
+go 1.21.0
+
+require (
+	github.com/prometheus/client_golang v1.17.0
+	github.com/sirupsen/logrus v1.9.3
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+)
diff --git a/go.sum b/go.sum
@@ -0,0 +1,43 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=