From 0b149e4a22e292e524c90cfe1b7ceab06fe98b93 Mon Sep 17 00:00:00 2001 From: fahed dorgaa Date: Mon, 22 Feb 2021 20:09:04 +0100 Subject: [PATCH] setup stats command Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa fixes Signed-off-by: fahed dorgaa --- README.md | 9 +- cmd/nerdctl/main.go | 1 + cmd/nerdctl/stats.go | 477 +++++++++++++++++++++++++++++++++++ cmd/nerdctl/stats_freebsd.go | 25 ++ cmd/nerdctl/stats_linux.go | 61 +++++ cmd/nerdctl/stats_test.go | 43 ++++ cmd/nerdctl/stats_windows.go | 25 ++ go.mod | 1 + go.sum | 3 +- pkg/eventutil/eventutil.go | 57 +++++ pkg/statsutil/stats.go | 212 ++++++++++++++++ pkg/statsutil/stats_linux.go | 160 ++++++++++++ 12 files changed, 1070 insertions(+), 4 deletions(-) create mode 100644 cmd/nerdctl/stats.go create mode 100644 cmd/nerdctl/stats_freebsd.go create mode 100644 cmd/nerdctl/stats_linux.go create mode 100644 cmd/nerdctl/stats_test.go create mode 100644 cmd/nerdctl/stats_windows.go create mode 100644 pkg/eventutil/eventutil.go create mode 100644 pkg/statsutil/stats.go create mode 100644 pkg/statsutil/stats_linux.go diff --git a/README.md b/README.md index 6ade2040245..c0cf91796bb 100644 --- a/README.md +++ b/README.md @@ -931,6 +931,12 @@ Flags: - :whale: `-f, --format`: Format the output using the given Go template, e.g, `{{json .}}` ## Stats +### :whale: nerdctl stats +Display a live stream of container(s) resource usage statistics. + + +Usage: `nerdctl stats [flags]` + ### :whale: nerdctl top Display the running processes of a container. @@ -1061,9 +1067,6 @@ Container management: - `docker checkpoint *` -Stats: -- `docker stats` - Image: - `docker export` and `docker import` - `docker history` diff --git a/cmd/nerdctl/main.go b/cmd/nerdctl/main.go index f4e1d2d2cee..e82d417e156 100644 --- a/cmd/nerdctl/main.go +++ b/cmd/nerdctl/main.go @@ -221,6 +221,7 @@ func newApp() *cobra.Command { // stats newTopCommand(), + newStatsCommand(), // #region Management newContainerCommand(), diff --git a/cmd/nerdctl/stats.go b/cmd/nerdctl/stats.go new file mode 100644 index 00000000000..f2bae535265 --- /dev/null +++ b/cmd/nerdctl/stats.go @@ -0,0 +1,477 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "text/tabwriter" + "text/template" + "time" + + "github.com/containerd/containerd" + eventstypes "github.com/containerd/containerd/api/events" + "github.com/containerd/containerd/events" + "github.com/containerd/nerdctl/pkg/eventutil" + "github.com/containerd/nerdctl/pkg/formatter" + "github.com/containerd/nerdctl/pkg/idutil/containerwalker" + "github.com/containerd/nerdctl/pkg/infoutil" + "github.com/containerd/nerdctl/pkg/labels" + "github.com/containerd/nerdctl/pkg/rootlessutil" + "github.com/containerd/nerdctl/pkg/statsutil" + "github.com/containerd/typeurl" + "github.com/docker/cli/templates" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +func newStatsCommand() *cobra.Command { + var statsCommand = &cobra.Command{ + Use: "stats", + Short: "Display a live stream of container(s) resource usage statistics.", + RunE: statsAction, + ValidArgsFunction: statsShellComplete, + SilenceUsage: true, + SilenceErrors: true, + } + + addStatsFlags(statsCommand) + + return statsCommand +} + +func addStatsFlags(cmd *cobra.Command) { + cmd.Flags().BoolP("all", "a", false, "Show all containers (default shows just running)") + cmd.Flags().String("format", "", "Pretty-print images using a Go template") + cmd.Flags().Bool("no-stream", false, "Disable streaming stats and only pull the first result") + cmd.Flags().Bool("no-trunc", false, "Do not truncate output") +} + +type stats struct { + mu sync.Mutex + cs []*statsutil.Stats +} + +//add is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/stats_helpers.go#L26-L34 +func (s *stats) add(cs *statsutil.Stats) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.isKnownContainer(cs.Container); !exists { + s.cs = append(s.cs, cs) + return true + } + return false +} + +//remove is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/stats_helpers.go#L36-L42 +func (s *stats) remove(id string) { + s.mu.Lock() + if i, exists := s.isKnownContainer(id); exists { + s.cs = append(s.cs[:i], s.cs[i+1:]...) + } + s.mu.Unlock() +} + +//isKnownContainer is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/stats_helpers.go#L44-L51 +func (s *stats) isKnownContainer(cid string) (int, bool) { + for i, c := range s.cs { + if c.Container == cid { + return i, true + } + } + return -1, false +} + +func statsAction(cmd *cobra.Command, args []string) error { + + // NOTE: rootless container does not rely on cgroupv1. + // more details about possible ways to resolve this concern: #223 + if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" { + return errors.New("stats requires cgroup v2 for rootless containers, see https://rootlesscontaine.rs/getting-started/common/cgroup2/") + } + + showAll := len(args) == 0 + closeChan := make(chan error) + + all, err := cmd.Flags().GetBool("all") + if err != nil { + return err + } + + noStream, err := cmd.Flags().GetBool("no-stream") + if err != nil { + return err + } + + format, err := cmd.Flags().GetString("format") + if err != nil { + return err + } + + noTrunc, err := cmd.Flags().GetBool("no-trunc") + if err != nil { + return err + } + + // waitFirst is a WaitGroup to wait first stat data's reach for each container + waitFirst := &sync.WaitGroup{} + cStats := stats{} + + client, ctx, cancel, err := newClient(cmd) + if err != nil { + return err + } + defer cancel() + + monitorContainerEvents := func(started chan<- struct{}, c chan *events.Envelope) { + eventsClient := client.EventService() + eventsCh, errCh := eventsClient.Subscribe(ctx) + + // Whether we successfully subscribed to eventsCh or not, we can now + // unblock the main goroutine. + close(started) + + for { + select { + case event := <-eventsCh: + c <- event + case err = <-errCh: + closeChan <- err + return + } + } + + } + + // getContainerList get all existing containers (only used when calling `nerdctl stats` without arguments). + getContainerList := func() { + containers, err := client.Containers(ctx) + if err != nil { + closeChan <- err + } + + for _, c := range containers { + cStatus := formatter.ContainerStatus(ctx, c) + if !strings.HasPrefix(cStatus, "Up") { + continue + } + s := statsutil.NewStats(c.ID()) + if cStats.add(s) { + waitFirst.Add(1) + go collect(cmd, s, waitFirst, c.ID(), !noStream) + } + } + } + + if showAll { + started := make(chan struct{}) + var ( + datacc *eventstypes.ContainerCreate + datacd *eventstypes.ContainerDelete + ) + + eh := eventutil.InitEventHandler() + eh.Handle("/containers/create", func(e events.Envelope) { + if all { + if e.Event != nil { + anydata, err := typeurl.UnmarshalAny(e.Event) + if err != nil { + // just skip + return + } + switch v := anydata.(type) { + case *eventstypes.ContainerCreate: + datacc = v + default: + // just skip + return + } + } + s := statsutil.NewStats(datacc.ID) + if cStats.add(s) { + waitFirst.Add(1) + go collect(cmd, s, waitFirst, datacc.ID, !noStream) + } + } + }) + + eh.Handle("/containers/delete", func(e events.Envelope) { + if !all { + if e.Event != nil { + anydata, err := typeurl.UnmarshalAny(e.Event) + if err != nil { + // just skip + return + } + switch v := anydata.(type) { + case *eventstypes.ContainerDelete: + datacd = v + default: + // just skip + return + } + } + cStats.remove(datacd.ID) + } + }) + + eventChan := make(chan *events.Envelope) + + go eh.Watch(eventChan) + go monitorContainerEvents(started, eventChan) + + defer close(eventChan) + <-started + + // Start a goroutine to retrieve the initial list of containers stats. + getContainerList() + + // make sure each container get at least one valid stat data + waitFirst.Wait() + + } else { + walker := &containerwalker.ContainerWalker{ + Client: client, + OnFound: func(ctx context.Context, found containerwalker.Found) error { + s := statsutil.NewStats(found.Container.ID()) + if cStats.add(s) { + waitFirst.Add(1) + go collect(cmd, s, waitFirst, found.Container.ID(), !noStream) + } + return nil + }, + } + + for _, req := range args { + n, err := walker.Walk(ctx, req) + if err != nil { + return err + } else if n == 0 { + return fmt.Errorf("no such container %s", req) + } + } + + // make sure each container get at least one valid stat data + waitFirst.Wait() + + var errs []string + cStats.mu.Lock() + for _, c := range cStats.cs { + if err := c.GetError(); err != nil { + errs = append(errs, err.Error()) + } + } + cStats.mu.Unlock() + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + } + + cleanScreen := func() { + if !noStream { + fmt.Fprint(cmd.OutOrStdout(), "\033[2J") + fmt.Fprint(cmd.OutOrStdout(), "\033[H") + } + } + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for range ticker.C { + cleanScreen() + ccstats := []statsutil.StatsEntry{} + cStats.mu.Lock() + for _, c := range cStats.cs { + ccstats = append(ccstats, c.GetStatistics()) + } + cStats.mu.Unlock() + + w := cmd.OutOrStdout() + var tmpl *template.Template + + switch format { + case "", "table": + w = tabwriter.NewWriter(cmd.OutOrStdout(), 10, 1, 3, ' ', 0) + fmt.Fprintln(w, "CONTAINER ID\tNAME\tCPU %\tMEM USAGE / LIMIT\tMEM %\tNET I/O\tBLOCK I/O\tPIDS") + case "raw": + return errors.New("unsupported format: \"raw\"") + default: + tmpl, err = templates.Parse(format) + if err != nil { + break + } + } + + for _, c := range ccstats { + rc := statsutil.RenderEntry(&c, noTrunc) + if tmpl != nil { + var b bytes.Buffer + if err := tmpl.Execute(&b, rc); err != nil { + break + } + if _, err = fmt.Fprintf(cmd.OutOrStdout(), b.String()+"\n"); err != nil { + break + } + } else { + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + rc.ID, + rc.Name, + rc.CPUPerc, + rc.MemUsage, + rc.MemPerc, + rc.NetIO, + rc.BlockIO, + rc.PIDs, + ); err != nil { + break + } + } + } + if f, ok := w.(Flusher); ok { + f.Flush() + } + + if len(cStats.cs) == 0 && !showAll { + break + } + if noStream { + break + } + select { + case err, ok := <-closeChan: + if ok { + if err != nil { + return err + } + } + default: + // just skip + } + } + + return err +} + +func collect(cmd *cobra.Command, s *statsutil.Stats, waitFirst *sync.WaitGroup, id string, noStream bool) { + + logrus.Debugf("collecting stats for %s", s.Container) + var ( + getFirst bool + u = make(chan error, 1) + ) + + defer func() { + // if error happens and we get nothing of stats, release wait group whatever + if !getFirst { + getFirst = true + waitFirst.Done() + } + }() + + client, ctx, cancel, err := newClient(cmd) + if err != nil { + s.SetError(err) + return + } + defer cancel() + container, err := client.LoadContainer(ctx, id) + if err != nil { + s.SetError(err) + return + } + + go func() { + + previousStats := make(map[string]uint64) + + for { + //task is in the for loop to avoid nil task just after Container creation + task, err := container.Task(ctx, nil) + if err != nil { + u <- err + continue + } + + //labels is in the for loop to avoid nil labels just after Container creation + clabels, err := container.Labels(ctx) + if err != nil { + u <- err + continue + } + + //sleep to create distant CPU readings + time.Sleep(500 * time.Millisecond) + + metric, err := task.Metrics(ctx) + if err != nil { + u <- err + continue + } + anydata, err := typeurl.UnmarshalAny(metric.Data) + if err != nil { + u <- err + continue + } + + statsEntry, err := renderStatsEntry(previousStats, anydata) + if err != nil { + u <- err + continue + } + statsEntry.Name = clabels[labels.Name] + statsEntry.ID = container.ID() + + s.SetStatistics(statsEntry) + u <- nil + } + }() + for { + select { + case <-time.After(6 * time.Second): + // zero out the values if we have not received an update within + // the specified duration. + s.SetErrorAndReset(errors.New("timeout waiting for stats")) + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + case err := <-u: + if err != nil { + s.SetError(err) + continue + } + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + } + } +} + +func statsShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // show running container names + statusFilterFn := func(st containerd.ProcessStatus) bool { + return st == containerd.Running + } + return shellCompleteContainerNames(cmd, statusFilterFn) +} diff --git a/cmd/nerdctl/stats_freebsd.go b/cmd/nerdctl/stats_freebsd.go new file mode 100644 index 00000000000..0a531a40c86 --- /dev/null +++ b/cmd/nerdctl/stats_freebsd.go @@ -0,0 +1,25 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import "github.com/containerd/nerdctl/pkg/statsutil" + +func renderStatsEntry(previousStats map[string]uint64, anydata interface{}) (statsutil.StatsEntry, error) { + + return statsutil.StatsEntry{}, nil + +} diff --git a/cmd/nerdctl/stats_linux.go b/cmd/nerdctl/stats_linux.go new file mode 100644 index 00000000000..5bff76dcdff --- /dev/null +++ b/cmd/nerdctl/stats_linux.go @@ -0,0 +1,61 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + + v1 "github.com/containerd/cgroups/stats/v1" + v2 "github.com/containerd/cgroups/v2/stats" + "github.com/containerd/nerdctl/pkg/statsutil" +) + +var ( + data *v1.Metrics + data2 *v2.Metrics + statsEntry statsutil.StatsEntry +) + +func renderStatsEntry(previousStats map[string]uint64, anydata interface{}) (statsutil.StatsEntry, error) { + + switch v := anydata.(type) { + case *v1.Metrics: + data = v + case *v2.Metrics: + data2 = v + default: + return statsutil.StatsEntry{}, errors.New("cannot convert metric data to cgroups.Metrics") + } + var err error + if data != nil { + statsEntry, err = statsutil.SetCgroupStatsFields(previousStats["CgroupCPU"], previousStats["CgroupSystem"], data) + previousStats["CgroupCPU"] = data.CPU.Usage.Total + previousStats["CgroupSystem"] = data.CPU.Usage.Kernel + if err != nil { + return statsutil.StatsEntry{}, err + } + } else if data2 != nil { + statsEntry, err = statsutil.SetCgroup2StatsFields(previousStats["Cgroup2CPU"], previousStats["Cgroup2System"], data2) + previousStats["Cgroup2CPU"] = data2.CPU.UsageUsec * 1000 + previousStats["Cgroup2System"] = data2.CPU.SystemUsec * 1000 + if err != nil { + return statsutil.StatsEntry{}, err + } + } + + return statsEntry, nil +} diff --git a/cmd/nerdctl/stats_test.go b/cmd/nerdctl/stats_test.go new file mode 100644 index 00000000000..5a60e812b0e --- /dev/null +++ b/cmd/nerdctl/stats_test.go @@ -0,0 +1,43 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/containerd/nerdctl/pkg/infoutil" + "github.com/containerd/nerdctl/pkg/rootlessutil" + "github.com/containerd/nerdctl/pkg/testutil" +) + +func TestStats(t *testing.T) { + // this comment is for `nerdctl ps` but it also valid for `nerdctl stats` : + // https://github.com/containerd/nerdctl/pull/223#issuecomment-851395178 + if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" { + t.Skip("test skipped for rootless containers on cgroup v1") + } + const ( + testContainerName = "nerdctl-test-stats" + ) + + base := testutil.NewBase(t) + defer base.Cmd("rm", "-f", testContainerName).Run() + + base.Cmd("run", "-d", "--name", testContainerName, testutil.AlpineImage, "sleep", "5").AssertOK() + base.Cmd("stats", "--no-stream", testContainerName).AssertOK() + +} diff --git a/cmd/nerdctl/stats_windows.go b/cmd/nerdctl/stats_windows.go new file mode 100644 index 00000000000..0a531a40c86 --- /dev/null +++ b/cmd/nerdctl/stats_windows.go @@ -0,0 +1,25 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import "github.com/containerd/nerdctl/pkg/statsutil" + +func renderStatsEntry(previousStats map[string]uint64, anydata interface{}) (statsutil.StatsEntry, error) { + + return statsutil.StatsEntry{}, nil + +} diff --git a/go.mod b/go.mod index 3d435e4a9fb..76c1e956e14 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/docker/docker v20.10.10+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/fatih/color v1.13.0 github.com/gogo/protobuf v1.3.2 github.com/mattn/go-isatty v0.0.14 diff --git a/go.sum b/go.sum index 83795cf3c12..4c37c7736df 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,9 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= diff --git a/pkg/eventutil/eventutil.go b/pkg/eventutil/eventutil.go new file mode 100644 index 00000000000..a29bca6eeaf --- /dev/null +++ b/pkg/eventutil/eventutil.go @@ -0,0 +1,57 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package eventutil + +import ( + "fmt" + "sync" + + "github.com/containerd/containerd/events" +) + +type eventHandler struct { + handlers map[string]func(events.Envelope) + mu sync.Mutex +} + +// InitEventHandler initializes and returns an eventHandler +func InitEventHandler() *eventHandler { + return &eventHandler{handlers: make(map[string]func(events.Envelope))} +} + +func (w *eventHandler) Handle(action string, h func(events.Envelope)) { + w.mu.Lock() + w.handlers[action] = h + w.mu.Unlock() +} + +// Watch ranges over the passed in event chan and processes the events based on the +// handlers created for a given action. +// To stop watching, close the event chan. +func (w *eventHandler) Watch(c <-chan *events.Envelope) { + for e := range c { + w.mu.Lock() + fmt.Println(e.Topic) + h, exists := w.handlers[e.Topic] + w.mu.Unlock() + if !exists { + continue + } + + go h(*e) + } +} diff --git a/pkg/statsutil/stats.go b/pkg/statsutil/stats.go new file mode 100644 index 00000000000..d010535fab0 --- /dev/null +++ b/pkg/statsutil/stats.go @@ -0,0 +1,212 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package statsutil + +import ( + "fmt" + "sync" + + v1 "github.com/containerd/cgroups/stats/v1" + units "github.com/docker/go-units" +) + +// StatsEntry represents the statistics data collected from a container +type StatsEntry struct { + Container string + Name string + ID string + CPUPercentage float64 + Memory float64 + MemoryLimit float64 + MemoryPercentage float64 + NetworkRx float64 + NetworkTx float64 + BlockRead float64 + BlockWrite float64 + PidsCurrent uint64 + IsInvalid bool +} + +// FormattedStatsEntry represents a formatted StatsEntry +type FormattedStatsEntry struct { + Name string + ID string + CPUPerc string + MemUsage string + MemPerc string + NetIO string + BlockIO string + PIDs string +} + +// Stats represents an entity to store containers statistics synchronously +type Stats struct { + mutex sync.RWMutex + StatsEntry + err error +} + +//NewStats is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L113-L116 +func NewStats(container string) *Stats { + return &Stats{StatsEntry: StatsEntry{Container: container}} +} + +//SetStatistics is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L87-L93 +func (cs *Stats) SetStatistics(s StatsEntry) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + s.Container = cs.Container + cs.StatsEntry = s +} + +//GetStatistics is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L95-L100 +func (cs *Stats) GetStatistics() StatsEntry { + cs.mutex.Lock() + defer cs.mutex.Unlock() + return cs.StatsEntry +} + +//GetError is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L51-L57 +func (cs *Stats) GetError() error { + cs.mutex.Lock() + defer cs.mutex.Unlock() + return cs.err +} + +//SetErrorAndReset is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L59-L75 +func (cs *Stats) SetErrorAndReset(err error) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + cs.CPUPercentage = 0 + cs.Memory = 0 + cs.MemoryPercentage = 0 + cs.MemoryLimit = 0 + cs.NetworkRx = 0 + cs.NetworkTx = 0 + cs.BlockRead = 0 + cs.BlockWrite = 0 + cs.PidsCurrent = 0 + cs.err = err + cs.IsInvalid = true +} + +//SetError is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L77-L85 +func (cs *Stats) SetError(err error) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + cs.err = err + if err != nil { + cs.IsInvalid = true + } +} + +func calculateMemPercent(limit float64, usedNo float64) float64 { + // Limit will never be 0 unless the container is not running and we haven't + // got any data from cgroup + if limit != 0 { + return usedNo / limit * 100.0 + } + return 0 +} + +func calculateNetwork(metrics *v1.Metrics) (float64, float64) { + var rx, tx float64 + + for _, v := range metrics.Network { + rx += float64(v.RxBytes) + tx += float64(v.TxBytes) + } + return rx, tx +} + +// Rendering a FormattedStatsEntry from StatsEntry +func RenderEntry(in *StatsEntry, noTrunc bool) FormattedStatsEntry { + return FormattedStatsEntry{ + Name: in.EntryName(), + ID: in.EntryID(noTrunc), + CPUPerc: in.CPUPerc(), + MemUsage: in.MemUsage(), + MemPerc: in.MemPerc(), + NetIO: in.NetIO(), + BlockIO: in.BlockIO(), + PIDs: in.PIDs(), + } +} + +/* +a set of functions to format container stats +*/ +func (s *StatsEntry) EntryName() string { + if len(s.Name) > 1 { + if len(s.Name) > 12 { + return s.Name[:12] + } + return s.Name + } + return "--" +} + +func (s *StatsEntry) EntryID(noTrunc bool) string { + if !noTrunc { + if len(s.ID) > 12 { + return s.ID[:12] + } + } + return s.ID +} + +func (s *StatsEntry) CPUPerc() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%.2f%%", s.CPUPercentage) +} + +func (s *StatsEntry) MemUsage() string { + if s.IsInvalid { + return fmt.Sprintf("-- / --") + } + return fmt.Sprintf("%s / %s", units.BytesSize(s.Memory), units.BytesSize(s.MemoryLimit)) +} + +func (s *StatsEntry) MemPerc() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%.2f%%", s.MemoryPercentage) +} + +func (s *StatsEntry) NetIO() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%s / %s", units.HumanSizeWithPrecision(s.NetworkRx, 3), units.HumanSizeWithPrecision(s.NetworkTx, 3)) +} + +func (s *StatsEntry) BlockIO() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%s / %s", units.HumanSizeWithPrecision(s.BlockRead, 3), units.HumanSizeWithPrecision(s.BlockWrite, 3)) +} + +func (s *StatsEntry) PIDs() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%d", s.PidsCurrent) +} diff --git a/pkg/statsutil/stats_linux.go b/pkg/statsutil/stats_linux.go new file mode 100644 index 00000000000..c5d3d7995f3 --- /dev/null +++ b/pkg/statsutil/stats_linux.go @@ -0,0 +1,160 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package statsutil + +import ( + "time" + + v1 "github.com/containerd/cgroups/stats/v1" + v2 "github.com/containerd/cgroups/v2/stats" +) + +var ( + memPercent, cpuPercent float64 + blkRead, blkWrite uint64 // Only used on Linux + mem, memLimit float64 + netRx, netTx float64 + pidsStatsCurrent uint64 +) + +func SetCgroupStatsFields(previousCgroupCPU, previousCgroupSystem uint64, data *v1.Metrics) (StatsEntry, error) { + + cpuPercent = calculateCgroupCPUPercent(previousCgroupCPU, previousCgroupSystem, data) + blkRead, blkWrite = calculateCgroupBlockIO(data) + mem = calculateCgroupMemUsage(data) + memLimit = float64(data.Memory.Usage.Limit) + memPercent = calculateMemPercent(memLimit, mem) + pidsStatsCurrent = data.Pids.Current + netRx, netTx = calculateNetwork(data) + + return StatsEntry{ + CPUPercentage: cpuPercent, + Memory: mem, + MemoryPercentage: memPercent, + MemoryLimit: memLimit, + NetworkRx: netRx, + NetworkTx: netTx, + BlockRead: float64(blkRead), + BlockWrite: float64(blkWrite), + PidsCurrent: pidsStatsCurrent, + }, nil + +} + +func SetCgroup2StatsFields(previousCgroup2CPU, previousCgroup2System uint64, metrics *v2.Metrics) (StatsEntry, error) { + + cpuPercent = calculateCgroup2CPUPercent(previousCgroup2CPU, previousCgroup2System, metrics) + blkRead, blkWrite = calculateCgroup2IO(metrics) + mem = calculateCgroup2MemUsage(metrics) + memLimit = float64(metrics.Memory.UsageLimit) + memPercent = calculateMemPercent(memLimit, mem) + pidsStatsCurrent = metrics.Pids.Current + + return StatsEntry{ + CPUPercentage: cpuPercent, + Memory: mem, + MemoryPercentage: memPercent, + MemoryLimit: memLimit, + BlockRead: float64(blkRead), + BlockWrite: float64(blkWrite), + PidsCurrent: pidsStatsCurrent, + }, nil + +} + +func calculateCgroupCPUPercent(previousCPU, previousSystem uint64, metrics *v1.Metrics) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(metrics.CPU.Usage.Total) - float64(previousCPU) + // calculate the change for the entire system between readings + systemDelta = float64(metrics.CPU.Usage.Kernel) - float64(previousSystem) + onlineCPUs = float64(len(metrics.CPU.Usage.PerCPU)) + ) + + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta / systemDelta) * onlineCPUs * 100.0 + } + return cpuPercent +} + +//PercpuUsage is not supported in CgroupV2 +func calculateCgroup2CPUPercent(previousCPU, previousSystem uint64, metrics *v2.Metrics) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(metrics.CPU.UsageUsec*1000) - float64(previousCPU) + // calculate the change for the entire system between readings + systemDelta = float64(metrics.CPU.SystemUsec*1000) - float64(previousSystem) + ) + + u, _ := time.ParseDuration("500ms") + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta + systemDelta) / float64(u.Nanoseconds()) * 100.0 + } + return cpuPercent +} + +func calculateCgroupMemUsage(metrics *v1.Metrics) float64 { + if v := metrics.Memory.TotalInactiveFile; v < metrics.Memory.Usage.Usage { + return float64(metrics.Memory.Usage.Usage - v) + } + return float64(metrics.Memory.Usage.Usage) +} + +func calculateCgroup2MemUsage(metrics *v2.Metrics) float64 { + if v := metrics.Memory.InactiveFile; v < metrics.Memory.Usage { + return float64(metrics.Memory.Usage - v) + } + return float64(metrics.Memory.Usage) +} + +func calculateCgroupBlockIO(metrics *v1.Metrics) (uint64, uint64) { + var blkRead, blkWrite uint64 + for _, bioEntry := range metrics.Blkio.IoServiceBytesRecursive { + if len(bioEntry.Op) == 0 { + continue + } + switch bioEntry.Op[0] { + case 'r', 'R': + blkRead = blkRead + bioEntry.Value + case 'w', 'W': + blkWrite = blkWrite + bioEntry.Value + } + } + return blkRead, blkWrite +} + +func calculateCgroup2IO(metrics *v2.Metrics) (uint64, uint64) { + var ioRead, ioWrite uint64 + + for _, iOEntry := range metrics.Io.Usage { + if iOEntry.Rios == 0 && iOEntry.Wios == 0 { + continue + } + + if iOEntry.Rios != 0 { + ioRead = ioRead + iOEntry.Rbytes + } + + if iOEntry.Wios != 0 { + ioWrite = ioWrite + iOEntry.Wbytes + } + } + + return ioRead, ioWrite +}