Skip to content

Commit

Permalink
Add visualization tool to debug image sources
Browse files Browse the repository at this point in the history
  • Loading branch information
phillebaba committed May 19, 2024
1 parent 3d770a2 commit 3fe6c39
Show file tree
Hide file tree
Showing 7 changed files with 388 additions and 8 deletions.
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ require (
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fsnotify/fsnotify v1.7.1-0.20240403050945-7086bea086b7 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
Expand Down Expand Up @@ -175,11 +175,11 @@ require (
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.18.0 // indirect
gonum.org/v1/gonum v0.13.0 // indirect
gonum.org/v1/gonum v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
Expand Down
11 changes: 6 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -764,8 +764,9 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/fsnotify/fsnotify v1.7.1-0.20240403050945-7086bea086b7 h1:5ZeiG5gIjLqPKLl+f5zv++9ZO2oxA6hmZ3e7G0mMW1M=
github.com/fsnotify/fsnotify v1.7.1-0.20240403050945-7086bea086b7/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
Expand Down Expand Up @@ -1771,8 +1772,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down Expand Up @@ -1896,8 +1897,8 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM=
gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU=
gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY=
Expand Down
50 changes: 50 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net"
"net/http"
"net/http/pprof"
"net/netip"
"net/url"
"os"
"os/signal"
Expand All @@ -29,6 +30,7 @@ import (
"github.com/spegel-org/spegel/pkg/routing"
"github.com/spegel-org/spegel/pkg/state"
"github.com/spegel-org/spegel/pkg/throttle"
"github.com/spegel-org/spegel/pkg/visualize"
)

type ConfigurationCmd struct {
Expand Down Expand Up @@ -65,7 +67,16 @@ type RegistryCmd struct {
ResolveLatestTag bool `arg:"--resolve-latest-tag,env:RESOLVE_LATEST_TAG" default:"true" help:"When true latest tags will be resolved to digests."`
}

type VisualizationCmd struct {
ContainerdRegistryConfigPath string `arg:"--containerd-registry-config-path,env:CONTAINERD_REGISTRY_CONFIG_PATH" default:"/etc/containerd/certs.d" help:"Directory where mirror configuration is written."`
ContainerdSock string `arg:"--containerd-sock,env:CONTAINERD_SOCK" default:"/run/containerd/containerd.sock" help:"Endpoint of containerd service."`
ContainerdNamespace string `arg:"--containerd-namespace,env:CONTAINERD_NAMESPACE" default:"k8s.io" help:"Containerd namespace to fetch images from."`
ContainerdContentPath string `arg:"--containerd-content-path,env:CONTAINERD_CONTENT_PATH" default:"/var/lib/containerd/io.containerd.content.v1.content" help:"Path to Containerd content store"`
Registries []url.URL `arg:"--registries,env:REGISTRIES,required" help:"registries that are configured to be mirrored."`
}

type Arguments struct {
Visualization *VisualizationCmd `arg:"subcommand:visualization"`
Configuration *ConfigurationCmd `arg:"subcommand:configuration"`
Registry *RegistryCmd `arg:"subcommand:registry"`
LogLevel slog.Level `arg:"--log-level,env:LOG_LEVEL" default:"INFO" help:"Minimum log level to output. Value should be DEBUG, INFO, WARN, or ERROR."`
Expand Down Expand Up @@ -96,6 +107,8 @@ func run(ctx context.Context, args *Arguments) error {
ctx, cancel := signal.NotifyContext(ctx, syscall.SIGTERM)
defer cancel()
switch {
case args.Visualization != nil:
return visualizeCommand(ctx, args.Visualization)
case args.Configuration != nil:
return configurationCommand(ctx, args.Configuration)
case args.Registry != nil:
Expand All @@ -105,6 +118,25 @@ func run(ctx context.Context, args *Arguments) error {
}
}

func visualizeCommand(_ context.Context, args *VisualizationCmd) error {
ociClient, err := oci.NewContainerd(args.ContainerdSock, args.ContainerdNamespace, args.ContainerdRegistryConfigPath, args.Registries, oci.WithContentPath(args.ContainerdContentPath))
if err != nil {
return err
}
eventStore := visualize.NewMemoryStore()
eventStore.Record("foobar", netip.MustParseAddr("10.0.0.0"), 200, true)
eventStore.Record("sha256:1f1a2d56de1d604801a9671f301190704c25d604a416f59e03c04f5c6ffee0d6", netip.MustParseAddr("10.0.0.0"), 404, true)
eventStore.Record("serve", netip.MustParseAddr("10.0.0.0"), 404, false)
eventStore.Record("serve", netip.MustParseAddr("10.0.0.1"), 200, false)
vSvr := visualize.NewServer(eventStore, ociClient)
svr := vSvr.Server(":9091")
err = svr.ListenAndServe()
if err != nil {
return err
}
return nil
}

func configurationCommand(ctx context.Context, args *ConfigurationCmd) error {
fs := afero.NewOsFs()
err := oci.AddMirrorConfiguration(ctx, fs, args.ContainerdRegistryConfigPath, args.Registries, args.MirrorRegistries, args.ResolveTags, args.AppendMirrors)
Expand Down Expand Up @@ -188,13 +220,31 @@ func registryCommand(ctx context.Context, args *RegistryCmd) (err error) {
return nil
})

// Visualizer
eventStore := visualize.NewMemoryStore()
vis := visualize.NewServer(eventStore, ociClient)
visSrv := vis.Server(":9091")
g.Go(func() error {
if err := visSrv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
})
g.Go(func() error {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
return visSrv.Shutdown(shutdownCtx)
})

// Registry
registryOpts := []registry.Option{
registry.WithResolveLatestTag(args.ResolveLatestTag),
registry.WithResolveRetries(args.MirrorResolveRetries),
registry.WithResolveTimeout(args.MirrorResolveTimeout),
registry.WithLocalAddress(args.LocalAddr),
registry.WithLogger(log),
registry.WithEventStore(eventStore),
}
if args.BlobSpeed != nil {
registryOpts = append(registryOpts, registry.WithBlobSpeed(*args.BlobSpeed))
Expand Down
40 changes: 40 additions & 0 deletions pkg/registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"net/http"
"net/http/httputil"
"net/netip"
"net/url"
"path"
"strconv"
Expand All @@ -21,6 +22,7 @@ import (
"github.com/spegel-org/spegel/pkg/oci"
"github.com/spegel-org/spegel/pkg/routing"
"github.com/spegel-org/spegel/pkg/throttle"
"github.com/spegel-org/spegel/pkg/visualize"
)

const (
Expand All @@ -37,6 +39,7 @@ type Registry struct {
resolveRetries int
resolveTimeout time.Duration
resolveLatestTag bool
eventStore visualize.EventStore
}

type Option func(*Registry)
Expand Down Expand Up @@ -83,6 +86,12 @@ func WithLogger(log logr.Logger) Option {
}
}

func WithEventStore(eventStore visualize.EventStore) Option {
return func(r *Registry) {
r.eventStore = eventStore
}
}

func NewRegistry(ociClient oci.Client, router routing.Router, opts ...Option) *Registry {
r := &Registry{
ociClient: ociClient,
Expand Down Expand Up @@ -188,6 +197,23 @@ func (r *Registry) registryHandler(rw mux.ResponseWriter, req *http.Request) str
return "mirror"
}

defer func() {
if req.Method != http.MethodGet {
return
}
id := ref.name
if id == "" {
// First 7 character plus sha256: prefix
id = ref.dgst.String()[:14]
}
ip := getClientIP(req)
addr, err := netip.ParseAddr(ip)
if err != nil {
return
}
r.eventStore.Record(id, addr, rw.Status(), false)
}()

// Serve registry endpoints.
switch ref.kind {
case referenceKindManifest:
Expand Down Expand Up @@ -288,6 +314,20 @@ func (r *Registry) handleMirror(rw mux.ResponseWriter, req *http.Request, ref re
return nil
}
proxy.ServeHTTP(rw, req)

// Track image events if enabled
if r.eventStore != nil {
if req.Method != http.MethodGet {
return
}
id := ref.name
if id == "" {
// First 7 character plus sha256: prefix
id = ref.dgst.String()[:14]
}
r.eventStore.Record(id, ipAddr.Addr(), rw.Status(), true)
}

if !succeeded {
break
}
Expand Down
108 changes: 108 additions & 0 deletions pkg/visualize/event.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package visualize

import (
"fmt"
"net/http"
"net/netip"
"strings"
"sync"
)

type EventStore interface {
Record(id string, peer netip.Addr, status int, mirror bool)
Dot(filter []string) string
}

type edge struct {
node string
status int
rootIsSource bool
}

type MemoryStore struct {

Check failure on line 22 in pkg/visualize/event.go

View workflow job for this annotation

GitHub Actions / lint

fieldalignment: struct with 40 pointer bytes could be 16 (govet)
mx sync.RWMutex
nodes map[string]string
edges map[string]edge
}

func NewMemoryStore() *MemoryStore {
return &MemoryStore{
nodes: map[string]string{},
edges: map[string]edge{},
}
}

func (m *MemoryStore) Record(id string, peer netip.Addr, status int, mirror bool) {
m.mx.Lock()
defer m.mx.Unlock()

m.nodes[peer.String()] = peer.String()
m.edges[id] = edge{
node: peer.String(),
status: status,
rootIsSource: mirror,
}
}

func (m *MemoryStore) Dot(filter []string) string {
m.mx.RLock()
defer m.mx.RUnlock()

if len(filter) == 0 {
return dagToDot(m.nodes, m.edges)
}
filteredNodes := map[string]string{}
filteredEdges := map[string]edge{}
for _, v := range filter {
edge, ok := m.edges[v]
if !ok {
continue
}
filteredNodes[edge.node] = edge.node
filteredEdges[v] = edge
}
return dagToDot(filteredNodes, filteredEdges)
}

func dagToDot(nodes map[string]string, edges map[string]edge) string {
dotNodes := []string{}
for _, v := range nodes {
node := fmt.Sprintf(`%q[label="%s"];`, v, v)

Check failure on line 70 in pkg/visualize/event.go

View workflow job for this annotation

GitHub Actions / lint

sprintfQuotedString: use %q instead of "%s" for quoted strings (gocritic)
dotNodes = append(dotNodes, node)
}
dotEdges := []string{}
for _, v := range edges {
color := ""
switch v.status {
case 0:
color = "yellow"
case http.StatusOK:
color = "green"
default:
color = "red"
}

src := "n0"
dest := v.node
if !v.rootIsSource {
src = v.node
dest = "n0"
}

edge := fmt.Sprintf("%q -> %q[color=%s]", src, dest, color)
dotEdges = append(dotEdges, edge)
}

return fmt.Sprintf(`
digraph {
layout="circo";
root="n0";
n0[label="host"]
%s
%s
}
`, strings.Join(dotNodes, "\n\t"), strings.Join(dotEdges, "\n\t"))
}
Loading

0 comments on commit 3fe6c39

Please sign in to comment.