From e89233bb137ef7ae39572c7ff68c509a469e6eca Mon Sep 17 00:00:00 2001 From: fahed dorgaa Date: Mon, 22 Feb 2021 20:09:04 +0100 Subject: [PATCH] setup stats command Signed-off-by: fahed dorgaa --- cmd/nerdctl/main.go | 1 + cmd/nerdctl/stats.go | 427 ++++++++++++++++++++++++++++++++++++++ cmd/nerdctl/stats_test.go | 43 ++++ go.mod | 4 + go.sum | 9 +- pkg/statsutil/stats.go | 380 +++++++++++++++++++++++++++++++++ 6 files changed, 861 insertions(+), 3 deletions(-) create mode 100644 cmd/nerdctl/stats.go create mode 100644 cmd/nerdctl/stats_test.go create mode 100644 pkg/statsutil/stats.go diff --git a/cmd/nerdctl/main.go b/cmd/nerdctl/main.go index f4e1d2d2cee..e82d417e156 100644 --- a/cmd/nerdctl/main.go +++ b/cmd/nerdctl/main.go @@ -221,6 +221,7 @@ func newApp() *cobra.Command { // stats newTopCommand(), + newStatsCommand(), // #region Management newContainerCommand(), diff --git a/cmd/nerdctl/stats.go b/cmd/nerdctl/stats.go new file mode 100644 index 00000000000..2fd9cf7d2d1 --- /dev/null +++ b/cmd/nerdctl/stats.go @@ -0,0 +1,427 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "text/tabwriter" + "text/template" + "time" + + wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" + v1 "github.com/containerd/cgroups/stats/v1" + v2 "github.com/containerd/cgroups/v2/stats" + "github.com/containerd/containerd" + "github.com/containerd/nerdctl/pkg/formatter" + "github.com/containerd/nerdctl/pkg/idutil/containerwalker" + "github.com/containerd/nerdctl/pkg/infoutil" + "github.com/containerd/nerdctl/pkg/labels" + "github.com/containerd/nerdctl/pkg/rootlessutil" + statsutil "github.com/containerd/nerdctl/pkg/statsutil" + "github.com/containerd/typeurl" + "github.com/docker/cli/templates" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +func newStatsCommand() *cobra.Command { + var statsCommand = &cobra.Command{ + Use: "stats", + Short: "Display a live stream of container(s) resource usage statistics.", + RunE: statsAction, + ValidArgsFunction: statsShellComplete, + SilenceUsage: true, + SilenceErrors: true, + } + + addStastFlags(statsCommand) + + return statsCommand +} + +func addStastFlags(cmd *cobra.Command) { + cmd.Flags().StringP("all", "a", "", "Show all containers (default shows just running)") + cmd.Flags().String("format", "", "Pretty-print images using a Go template") + cmd.Flags().Bool("no-stream", false, "Disable streaming stats and only pull the first result") + cmd.Flags().Bool("no-trunc", false, "Do not truncate output") +} + +type stats struct { + mu sync.Mutex + cs []*statsutil.Stats +} + +//add is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/stats_helpers.go#L26-L34 +func (s *stats) add(cs *statsutil.Stats) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.isKnownContainer(cs.Container); !exists { + s.cs = append(s.cs, cs) + return true + } + return false +} + +//isKnownContainer is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/stats_helpers.go#L44-L51 +func (s *stats) isKnownContainer(cid string) (int, bool) { + for i, c := range s.cs { + if c.Container == cid { + return i, true + } + } + return -1, false +} + +func statsAction(cmd *cobra.Command, args []string) error { + + // NOTE: rootless container does not rely on cgroupv1. + // more details about possible ways to resolve this concern: #223 + if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" { + return fmt.Errorf("stats requires cgroup v2 for rootless containers, see https://rootlesscontaine.rs/getting-started/common/cgroup2/") + } + + showAll := len(args) == 0 + closeChan := make(chan error) + + noStream, err := cmd.Flags().GetBool("no-stream") + if err != nil { + return err + } + + format, err := cmd.Flags().GetString("format") + if err != nil { + return err + } + + noTrunc, err := cmd.Flags().GetBool("no-trunc") + if err != nil { + return err + } + + // waitFirst is a WaitGroup to wait first stat data's reach for each container + waitFirst := &sync.WaitGroup{} + cStats := stats{} + + client, ctx, cancel, err := newClient(cmd) + if err != nil { + return err + } + defer cancel() + + // getContainerList get all existing containers (only used when calling `nerdctl stats` without arguments). + getContainerList := func() { + containers, err := client.Containers(ctx) + if err != nil { + closeChan <- err + } + + for _, c := range containers { + cStatus := formatter.ContainerStatus(ctx, c) + if !strings.HasPrefix(cStatus, "Up") { + continue + } + s := statsutil.NewStats(c.ID()) + if cStats.add(s) { + waitFirst.Add(1) + go collect(cmd, s, waitFirst, c.ID(), !noStream) + } + } + } + + if showAll { + // TODO: watches for container creation and removal (dynamic stats) + // Start a goroutine to retrieve the initial list of containers stats. + getContainerList() + + // make sure each container get at least one valid stat data + waitFirst.Wait() + + } else { + walker := &containerwalker.ContainerWalker{ + Client: client, + OnFound: func(ctx context.Context, found containerwalker.Found) error { + s := statsutil.NewStats(found.Container.ID()) + if cStats.add(s) { + waitFirst.Add(1) + go collect(cmd, s, waitFirst, found.Container.ID(), !noStream) + } + return nil + }, + } + + for _, req := range args { + n, err := walker.Walk(ctx, req) + if err != nil { + return err + } else if n == 0 { + return fmt.Errorf("no such container %s", req) + } + } + + // make sure each container get at least one valid stat data + waitFirst.Wait() + + var errs []string + cStats.mu.Lock() + for _, c := range cStats.cs { + if err := c.GetError(); err != nil { + errs = append(errs, err.Error()) + } + } + cStats.mu.Unlock() + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + } + + cleanScreen := func() { + if !noStream { + fmt.Fprint(cmd.OutOrStdout(), "\033[2J") + fmt.Fprint(cmd.OutOrStdout(), "\033[H") + } + } + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for range ticker.C { + cleanScreen() + ccstats := []statsutil.StatsEntry{} + cStats.mu.Lock() + for _, c := range cStats.cs { + ccstats = append(ccstats, c.GetStatistics()) + } + cStats.mu.Unlock() + + w := cmd.OutOrStdout() + var tmpl *template.Template + + switch format { + case "", "table": + w = tabwriter.NewWriter(cmd.OutOrStdout(), 10, 1, 3, ' ', 0) + fmt.Fprintln(w, "CONTAINER ID\tNAME\tCPU %\tMEM USAGE / LIMIT\tMEM %\tNET I/O\tBLOCK I/O\tPIDS") + case "raw": + return errors.New("unsupported format: \"raw\"") + default: + tmpl, err = templates.Parse(format) + if err != nil { + break + } + } + + for _, c := range ccstats { + rc := statsutil.RenderEntry(&c, noTrunc) + if tmpl != nil { + var b bytes.Buffer + if err := tmpl.Execute(&b, rc); err != nil { + break + } + if _, err = fmt.Fprintf(cmd.OutOrStdout(), b.String()+"\n"); err != nil { + break + } + } else { + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + rc.ID, + rc.Name, + rc.CPUPerc, + rc.MemUsage, + rc.MemPerc, + rc.NetIO, + rc.BlockIO, + rc.PIDs, + ); err != nil { + break + } + } + } + if f, ok := w.(Flusher); ok { + f.Flush() + } + + if len(cStats.cs) == 0 && !showAll { + break + } + if noStream { + break + } + select { + case err, ok := <-closeChan: + if ok { + if err != nil { + return err + } + } + default: + // just skip + } + } + + return err +} + +func collect(cmd *cobra.Command, s *statsutil.Stats, waitFirst *sync.WaitGroup, id string, noStream bool) { + + logrus.Debugf("collecting stats for %s", s.Container) + var ( + getFirst bool + u = make(chan error, 1) + ) + + defer func() { + // if error happens and we get nothing of stats, release wait group whatever + if !getFirst { + getFirst = true + waitFirst.Done() + } + }() + + client, ctx, cancel, err := newClient(cmd) + if err != nil { + s.SetError(err) + return + } + defer cancel() + container, err := client.LoadContainer(ctx, id) + if err != nil { + s.SetError(err) + return + } + + task, err := container.Task(ctx, nil) + if err != nil { + s.SetError(err) + return + } + + clabels, err := container.Labels(ctx) + if err != nil { + s.SetError(err) + return + } + + go func() { + + var ( + previousCgroupCPU uint64 + previousCgroupSystem uint64 + previousCgroup2CPU uint64 + previousCgroup2System uint64 + previousWindowsCPU uint64 + previousWindowsSystem uint64 + statsEntry statsutil.StatsEntry + ) + + for { + //sleep to create distant CPU readings + time.Sleep(500 * time.Millisecond) + + metric, err := task.Metrics(ctx) + if err != nil { + u <- err + continue + } + anydata, err := typeurl.UnmarshalAny(metric.Data) + if err != nil { + u <- err + continue + } + var ( + data *v1.Metrics + data2 *v2.Metrics + windowsStats *wstats.WindowsContainerStatistics + ) + switch v := anydata.(type) { + case *v1.Metrics: + data = v + case *v2.Metrics: + data2 = v + case *wstats.WindowsContainerStatistics: + windowsStats = v + default: + u <- errors.New("cannot convert metric data to cgroups.Metrics or windows.Statistics") + } + + if data != nil { + statsEntry, err = statsutil.SetCgroupStatsFields(&previousCgroupCPU, &previousCgroupSystem, data) + previousCgroupCPU = data.CPU.Usage.Total + previousCgroupSystem = data.CPU.Usage.Kernel + if err != nil { + u <- err + continue + } + } else if data2 != nil { + statsEntry, err = statsutil.SetCgroup2StatsFields(&previousCgroup2CPU, &previousCgroup2System, data2) + previousCgroup2CPU = data2.CPU.UsageUsec * 1000 + previousCgroup2System = data2.CPU.SystemUsec * 1000 + if err != nil { + u <- err + continue + } + } else { + statsEntry, err = statsutil.SetWindowsStatsFields(&previousWindowsCPU, &previousWindowsSystem, windowsStats) + previousWindowsCPU = windowsStats.Processor.TotalRuntimeNS + previousWindowsSystem = windowsStats.Processor.RuntimeKernelNS + if err != nil { + u <- err + continue + } + } + + statsEntry.Name = clabels[labels.Name] + statsEntry.ID = container.ID() + + s.SetStatistics(statsEntry) + u <- nil + } + }() + for { + select { + case <-time.After(6 * time.Second): + // zero out the values if we have not received an update within + // the specified duration. + s.SetErrorAndReset(errors.New("timeout waiting for stats")) + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + case err := <-u: + if err != nil { + s.SetError(err) + continue + } + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + } + } +} + +func statsShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // show running container names + statusFilterFn := func(st containerd.ProcessStatus) bool { + return st == containerd.Running + } + return shellCompleteContainerNames(cmd, statusFilterFn) +} diff --git a/cmd/nerdctl/stats_test.go b/cmd/nerdctl/stats_test.go new file mode 100644 index 00000000000..5a60e812b0e --- /dev/null +++ b/cmd/nerdctl/stats_test.go @@ -0,0 +1,43 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/containerd/nerdctl/pkg/infoutil" + "github.com/containerd/nerdctl/pkg/rootlessutil" + "github.com/containerd/nerdctl/pkg/testutil" +) + +func TestStats(t *testing.T) { + // this comment is for `nerdctl ps` but it also valid for `nerdctl stats` : + // https://github.com/containerd/nerdctl/pull/223#issuecomment-851395178 + if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" { + t.Skip("test skipped for rootless containers on cgroup v1") + } + const ( + testContainerName = "nerdctl-test-stats" + ) + + base := testutil.NewBase(t) + defer base.Cmd("rm", "-f", testContainerName).Run() + + base.Cmd("run", "-d", "--name", testContainerName, testutil.AlpineImage, "sleep", "5").AssertOK() + base.Cmd("stats", "--no-stream", testContainerName).AssertOK() + +} diff --git a/go.mod b/go.mod index 19043b7b6fc..763e18703cd 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/Microsoft/go-winio v0.5.1 + github.com/Microsoft/hcsshim v0.8.21 github.com/compose-spec/compose-go v1.0.4 github.com/containerd/cgroups v1.0.2 github.com/containerd/console v1.0.3 @@ -23,6 +24,7 @@ require ( github.com/docker/docker v20.10.10+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/fatih/color v1.13.0 github.com/gogo/protobuf v1.3.2 github.com/mattn/go-isatty v0.0.14 @@ -35,8 +37,10 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 // replaced, see the bottom of this file github.com/spf13/pflag v1.0.5 // replaced, see the bottom of this file + github.com/stretchr/objx v0.2.0 // indirect github.com/tidwall/gjson v1.11.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 diff --git a/go.sum b/go.sum index ed5313505a6..7dbea2e7ebe 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,9 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -628,8 +629,9 @@ github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOH github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -822,8 +824,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/pkg/statsutil/stats.go b/pkg/statsutil/stats.go new file mode 100644 index 00000000000..33065351843 --- /dev/null +++ b/pkg/statsutil/stats.go @@ -0,0 +1,380 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package statsutil + +import ( + "fmt" + "runtime" + "sync" + "time" + + wstats "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" + v1 "github.com/containerd/cgroups/stats/v1" + v2 "github.com/containerd/cgroups/v2/stats" + units "github.com/docker/go-units" +) + +// StatsEntry represents the statistics data collected from a container +type StatsEntry struct { + Container string + Name string + ID string + CPUPercentage float64 + Memory float64 + MemoryLimit float64 + MemoryPercentage float64 + NetworkRx float64 + NetworkTx float64 + BlockRead float64 + BlockWrite float64 + PidsCurrent uint64 + IsInvalid bool +} + +// FormattedStatsEntry represents a formatted StatsEntry +type FormattedStatsEntry struct { + Name string + ID string + CPUPerc string + MemUsage string + MemPerc string + NetIO string + BlockIO string + PIDs string +} + +// Stats represents an entity to store containers statistics synchronously +type Stats struct { + mutex sync.Mutex + StatsEntry + err error +} + +//NewStats is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L113-L116 +func NewStats(container string) *Stats { + return &Stats{StatsEntry: StatsEntry{Container: container}} +} + +//SetStatistics is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L87-L93 +func (cs *Stats) SetStatistics(s StatsEntry) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + s.Container = cs.Container + cs.StatsEntry = s +} + +//GetStatistics is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L95-L100 +func (cs *Stats) GetStatistics() StatsEntry { + cs.mutex.Lock() + defer cs.mutex.Unlock() + return cs.StatsEntry +} + +//GetError is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L51-L57 +func (cs *Stats) GetError() error { + cs.mutex.Lock() + defer cs.mutex.Unlock() + return cs.err +} + +//SetErrorAndReset is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L59-L75 +func (cs *Stats) SetErrorAndReset(err error) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + cs.CPUPercentage = 0 + cs.Memory = 0 + cs.MemoryPercentage = 0 + cs.MemoryLimit = 0 + cs.NetworkRx = 0 + cs.NetworkTx = 0 + cs.BlockRead = 0 + cs.BlockWrite = 0 + cs.PidsCurrent = 0 + cs.err = err + cs.IsInvalid = true +} + +//SetError is from https://github.com/docker/cli/blob/3fb4fb83dfb5db0c0753a8316f21aea54dab32c5/cli/command/container/formatter_stats.go#L77-L85 +func (cs *Stats) SetError(err error) { + cs.mutex.Lock() + defer cs.mutex.Unlock() + cs.err = err + if err != nil { + cs.IsInvalid = true + } +} + +var ( + memPercent, cpuPercent float64 + blkRead, blkWrite uint64 // Only used on Linux + mem, memLimit float64 + netRx, netTx float64 + pidsStatsCurrent uint64 +) + +func SetCgroupStatsFields(previousCgroupCPU, previousCgroupSystem *uint64, data *v1.Metrics) (StatsEntry, error) { + + cpuPercent = calculateCgroupCPUPercent(previousCgroupCPU, previousCgroupSystem, data) + blkRead, blkWrite = calculateCgroupBlockIO(data) + mem = calculateCgroupMemUsage(data) + memLimit = float64(data.Memory.Usage.Limit) + memPercent = calculateMemPercent(memLimit, mem) + pidsStatsCurrent = data.Pids.Current + netRx, netTx = calculateNetwork(data) + + return StatsEntry{ + CPUPercentage: cpuPercent, + Memory: mem, + MemoryPercentage: memPercent, + MemoryLimit: memLimit, + NetworkRx: netRx, + NetworkTx: netTx, + BlockRead: float64(blkRead), + BlockWrite: float64(blkWrite), + PidsCurrent: pidsStatsCurrent, + }, nil + +} + +func SetCgroup2StatsFields(previousCgroup2CPU, previousCgroup2System *uint64, metrics *v2.Metrics) (StatsEntry, error) { + + cpuPercent = calculateCgroup2CPUPercent(previousCgroup2CPU, previousCgroup2System, metrics) + blkRead, blkWrite = calculateCgroup2IO(metrics) + mem = calculateCgroup2MemUsage(metrics) + memLimit = float64(metrics.Memory.UsageLimit) + memPercent = calculateMemPercent(memLimit, mem) + pidsStatsCurrent = metrics.Pids.Current + + return StatsEntry{ + CPUPercentage: cpuPercent, + Memory: mem, + MemoryPercentage: memPercent, + MemoryLimit: memLimit, + BlockRead: float64(blkRead), + BlockWrite: float64(blkWrite), + PidsCurrent: pidsStatsCurrent, + }, nil + +} + +func calculateCgroupCPUPercent(previousCPU, previousSystem *uint64, metrics *v1.Metrics) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(metrics.CPU.Usage.Total) - float64(*previousCPU) + // calculate the change for the entire system between readings + systemDelta = float64(metrics.CPU.Usage.Kernel) - float64(*previousSystem) + onlineCPUs = float64(len(metrics.CPU.Usage.PerCPU)) + ) + + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta / systemDelta) * onlineCPUs * 100.0 + } + return cpuPercent +} + +//PercpuUsage is not supported in CgroupV2 +func calculateCgroup2CPUPercent(previousCPU, previousSystem *uint64, metrics *v2.Metrics) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(metrics.CPU.UsageUsec*1000) - float64(*previousCPU) + // calculate the change for the entire system between readings + systemDelta = float64(metrics.CPU.SystemUsec*1000) - float64(*previousSystem) + ) + + u, _ := time.ParseDuration("2µs") + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta + systemDelta) / float64(u.Nanoseconds()) * 100.0 + } + return cpuPercent +} + +func calculateCgroupMemUsage(metrics *v1.Metrics) float64 { + if v := metrics.Memory.TotalInactiveFile; v < metrics.Memory.Usage.Usage { + return float64(metrics.Memory.Usage.Usage - v) + } + return float64(metrics.Memory.Usage.Usage) +} + +func calculateCgroup2MemUsage(metrics *v2.Metrics) float64 { + if v := metrics.Memory.InactiveFile; v < metrics.Memory.Usage { + return float64(metrics.Memory.Usage - v) + } + return float64(metrics.Memory.Usage) +} + +func calculateMemPercent(limit float64, usedNo float64) float64 { + // Limit will never be 0 unless the container is not running and we haven't + // got any data from cgroup + if limit != 0 { + return usedNo / limit * 100.0 + } + return 5 +} + +func calculateNetwork(metrics *v1.Metrics) (float64, float64) { + var rx, tx float64 + + for _, v := range metrics.Network { + rx += float64(v.RxBytes) + tx += float64(v.TxBytes) + } + return rx, tx +} + +func calculateCgroupBlockIO(metrics *v1.Metrics) (uint64, uint64) { + var blkRead, blkWrite uint64 + for _, bioEntry := range metrics.Blkio.IoServiceBytesRecursive { + if len(bioEntry.Op) == 0 { + continue + } + switch bioEntry.Op[0] { + case 'r', 'R': + blkRead = blkRead + bioEntry.Value + case 'w', 'W': + blkWrite = blkWrite + bioEntry.Value + } + } + return blkRead, blkWrite +} + +func calculateCgroup2IO(metrics *v2.Metrics) (uint64, uint64) { + var ioRead, ioWrite uint64 + + for _, iOEntry := range metrics.Io.Usage { + if iOEntry.Rios == 0 && iOEntry.Wios == 0 { + continue + } + + if iOEntry.Rios != 0 { + ioRead = ioRead + iOEntry.Rbytes + } + + if iOEntry.Wios != 0 { + ioWrite = ioWrite + iOEntry.Wbytes + } + } + + return ioRead, ioWrite +} + +func SetWindowsStatsFields(previousWindowsCPU, previousWindowsSystem *uint64, stats *wstats.WindowsContainerStatistics) (StatsEntry, error) { + + cpuPercent = calculateWindowsProcessorPercent(previousWindowsCPU, previousWindowsSystem, stats) + return StatsEntry{ + CPUPercentage: cpuPercent, + }, nil + +} + +func calculateWindowsProcessorPercent(previousWindowsCPU, previousWindowsSystem *uint64, stats *wstats.WindowsContainerStatistics) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(stats.Processor.TotalRuntimeNS) - float64(*previousWindowsCPU) + // calculate the change for the entire system between readings + systemDelta = float64(stats.Processor.RuntimeKernelNS) - float64(*previousWindowsSystem) + ) + + u, _ := time.ParseDuration("2µs") + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta + systemDelta) / float64(u.Nanoseconds()) * 100.0 + } + return cpuPercent +} + +// Rendering a FormattedStatsEntry from StatsEntry +func RenderEntry(in *StatsEntry, noTrunc bool) FormattedStatsEntry { + return FormattedStatsEntry{ + Name: in.EntryName(), + ID: in.EntryID(noTrunc), + CPUPerc: in.CPUPerc(), + MemUsage: in.MemUsage(), + MemPerc: in.MemPerc(), + NetIO: in.NetIO(), + BlockIO: in.BlockIO(), + PIDs: in.PIDs(), + } +} + +/* +a set of functions to format container stats +*/ +func (s *StatsEntry) EntryName() string { + if len(s.Name) > 1 { + if len(s.Name) > 12 { + return s.Name[:12] + } + return s.Name + } + return "--" +} + +func (s *StatsEntry) EntryID(noTrunc bool) string { + if !noTrunc { + if len(s.ID) > 12 { + return s.ID[:12] + } + } + return s.ID +} + +func (s *StatsEntry) CPUPerc() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%.2f%%", s.CPUPercentage) +} + +func (s *StatsEntry) MemUsage() string { + if s.IsInvalid { + return fmt.Sprintf("-- / --") + } + if runtime.GOOS == "windows" { + return units.BytesSize(s.Memory) + } + return fmt.Sprintf("%s / %s", units.BytesSize(s.Memory), units.BytesSize(s.MemoryLimit)) +} + +func (s *StatsEntry) MemPerc() string { + if s.IsInvalid || runtime.GOOS == "windows" { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%.2f%%", s.MemoryPercentage) +} + +func (s *StatsEntry) NetIO() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%s / %s", units.HumanSizeWithPrecision(s.NetworkRx, 3), units.HumanSizeWithPrecision(s.NetworkTx, 3)) +} + +func (s *StatsEntry) BlockIO() string { + if s.IsInvalid { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%s / %s", units.HumanSizeWithPrecision(s.BlockRead, 3), units.HumanSizeWithPrecision(s.BlockWrite, 3)) +} + +func (s *StatsEntry) PIDs() string { + if s.IsInvalid || runtime.GOOS == "windows" { + return fmt.Sprintf("--") + } + return fmt.Sprintf("%d", s.PidsCurrent) +}