diff --git a/client/driver/docker.go b/client/driver/docker.go index 3cd6b9ee182..616e78b8e69 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -34,8 +34,8 @@ var ( client *docker.Client // The statistics the Docker driver exposes - DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage"} - DockerMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"} + DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"} + DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"} ) const ( @@ -487,7 +487,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " ")) config.Cmd = cmd } else if len(driverConfig.Args) != 0 { - d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified") + config.Cmd = parsedArgs } if len(driverConfig.Labels) > 0 { @@ -998,20 +998,22 @@ func (h *DockerHandle) collectStats() { } cs := &cstructs.CpuStats{ - SystemMode: float64(s.CPUStats.CPUUsage.UsageInKernelmode), - UserMode: float64(s.CPUStats.CPUUsage.UsageInKernelmode), ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, Measured: DockerMeasuredCpuStats, } // Calculate percentage - cs.Percent = 0.0 - cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage) - systemDelta := float64(s.CPUStats.SystemCPUUsage) - float64(s.PreCPUStats.SystemCPUUsage) - if cpuDelta > 0.0 && systemDelta > 0.0 { - cs.Percent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0 - } + cores := len(s.CPUStats.CPUUsage.PercpuUsage) + cs.Percent = calculatePercent( + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, + s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, cores) + cs.SystemMode = calculatePercent( + s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode, + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores) + cs.UserMode = calculatePercent( + s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode, + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores) h.resourceUsageLock.Lock() h.resourceUsage = &cstructs.TaskResourceUsage{ @@ -1028,3 +1030,13 @@ func (h *DockerHandle) collectStats() { } } } + +func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 { + numerator := newSample - oldSample + denom := newTotal - oldTotal + if numerator <= 0 || denom <= 0 { + return 0.0 + } + + return (float64(numerator) / float64(denom)) * float64(cores) * 100.0 +} diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index c89e00c9108..75fcfae5619 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -34,12 +34,15 @@ const ( // tree for finding out the pids that the executor and it's child processes // have forked pidScanInterval = 5 * time.Second + + // nanosecondsInSecond is the number of nanoseconds in a second. + nanosecondsInSecond float64 = 1000000000.0 ) var ( // The statistics the basic executor exposes ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"} - ExecutorBasicMeasuredCpuStats = []string{"SystemMode", "UserMode", "Percent"} + ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"} ) // Executor is the interface which allows a driver to launch and supervise @@ -188,18 +191,22 @@ type UniversalExecutor struct { cgPaths map[string]string cgLock sync.Mutex - consulService *consul.ConsulService - consulCtx *ConsulContext - cpuStats *stats.CpuStats - logger *log.Logger + consulService *consul.ConsulService + consulCtx *ConsulContext + totalCpuStats *stats.CpuStats + userCpuStats *stats.CpuStats + systemCpuStats *stats.CpuStats + logger *log.Logger } // NewExecutor returns an Executor func NewExecutor(logger *log.Logger) Executor { exec := &UniversalExecutor{ - logger: logger, - processExited: make(chan interface{}), - cpuStats: stats.NewCpuStats(), + logger: logger, + processExited: make(chan interface{}), + totalCpuStats: stats.NewCpuStats(), + userCpuStats: stats.NewCpuStats(), + systemCpuStats: stats.NewCpuStats(), } return exec @@ -515,12 +522,12 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro cs := &cstructs.CpuStats{} if cpuStats, err := p.Times(); err == nil { - cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System) - cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User) + cs.SystemMode = pid.cpuStatsSys.Percent(cpuStats.System * nanosecondsInSecond) + cs.UserMode = pid.cpuStatsUser.Percent(cpuStats.User * nanosecondsInSecond) cs.Measured = ExecutorBasicMeasuredCpuStats // calculate cpu usage percent - cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total()) + cs.Percent = pid.cpuStatsTotal.Percent(cpuStats.Total() * nanosecondsInSecond) } stats[strconv.Itoa(pid.pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs} } @@ -781,13 +788,13 @@ func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstruct SystemMode: systemModeCPU, UserMode: userModeCPU, Percent: percent, - Measured: ExecutorBasicMeasuredMemStats, + Measured: ExecutorBasicMeasuredCpuStats, } totalMemory := &cstructs.MemoryStats{ RSS: totalRSS, Swap: totalSwap, - Measured: ExecutorBasicMeasuredCpuStats, + Measured: ExecutorBasicMeasuredMemStats, } resourceUsage := cstructs.ResourceUsage{ diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index cdcea6a09e1..c95afb53767 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -40,12 +40,9 @@ var ( // clockTicks is the clocks per second of the machine clockTicks = uint64(system.GetClockTicks()) - // nanosecondsInSecond is the number of nanoseconds in a second. - nanosecondsInSecond = uint64(1000000000) - // The statistics the executor exposes when using cgroups - ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage", "KernelUsage", "KernelMaxUsage"} - ExecutorCgroupMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"} + ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"} + ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"} ) // configureIsolation configures chroot and creates cgroups @@ -165,23 +162,19 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { } // CPU Related Stats - totalProcessCPUUsage := stats.CpuStats.CpuUsage.TotalUsage - userModeTime := stats.CpuStats.CpuUsage.UsageInUsermode - kernelModeTime := stats.CpuStats.CpuUsage.UsageInKernelmode - - umTicks := (userModeTime * clockTicks) / nanosecondsInSecond - kmTicks := (kernelModeTime * clockTicks) / nanosecondsInSecond + totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage) + userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode) + kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode) cs := &cstructs.CpuStats{ - SystemMode: float64(kmTicks), - UserMode: float64(umTicks), + SystemMode: e.systemCpuStats.Percent(kernelModeTime), + UserMode: e.userCpuStats.Percent(userModeTime), + Percent: e.totalCpuStats.Percent(totalProcessCPUUsage), ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods, ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime, Measured: ExecutorCgroupMeasuredCpuStats, } - if e.cpuStats != nil { - cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage / nanosecondsInSecond)) - } + taskResUsage := cstructs.TaskResourceUsage{ ResourceUsage: &cstructs.ResourceUsage{ MemoryStats: ms, diff --git a/client/stats/cpu.go b/client/stats/cpu.go index ca92c0e7327..db3d7ad3096 100644 --- a/client/stats/cpu.go +++ b/client/stats/cpu.go @@ -7,8 +7,8 @@ import ( // CpuStats calculates cpu usage percentage type CpuStats struct { - prevProcessUsage float64 - prevTime time.Time + prevCpuTime float64 + prevTime time.Time totalCpus int } @@ -20,30 +20,32 @@ func NewCpuStats() *CpuStats { } // Percent calculates the cpu usage percentage based on the current cpu usage -// and the previous cpu usage -func (c *CpuStats) Percent(currentProcessUsage float64) float64 { +// and the previous cpu usage where usage is given as time in nanoseconds spend +// in the cpu +func (c *CpuStats) Percent(cpuTime float64) float64 { now := time.Now() - if c.prevProcessUsage == 0.0 { + if c.prevCpuTime == 0.0 { // invoked first time - c.prevProcessUsage = currentProcessUsage + c.prevCpuTime = cpuTime c.prevTime = now return 0.0 } - delta := (now.Sub(c.prevTime).Seconds()) * float64(c.totalCpus) - ret := c.calculatePercent(c.prevProcessUsage, currentProcessUsage, delta) - c.prevProcessUsage = currentProcessUsage + timeDelta := now.Sub(c.prevTime).Nanoseconds() + ret := c.calculatePercent(c.prevCpuTime, cpuTime, timeDelta) + c.prevCpuTime = cpuTime c.prevTime = now return ret } -func (c *CpuStats) calculatePercent(t1, t2 float64, delta float64) float64 { - if delta == 0 { - return 0 +func (c *CpuStats) calculatePercent(t1, t2 float64, timeDelta int64) float64 { + vDelta := t2 - t1 + if timeDelta <= 0 || vDelta <= 0.0 { + return 0.0 } - delta_proc := t2 - t1 - overall_percent := ((delta_proc / delta) * 100) * float64(c.totalCpus) + + overall_percent := (vDelta / float64(timeDelta)) * 100.0 return overall_percent } diff --git a/command/alloc_status.go b/command/alloc_status.go index fe5943eb2d9..aa04a5bf3a2 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -361,10 +361,13 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri cpuUsage := strconv.Itoa(resource.CPU) memUsage := strconv.Itoa(resource.MemoryMB) if ru, ok := stats[task]; ok && ru != nil && ru.ResourceUsage != nil { - cpuTicksConsumed := (ru.ResourceUsage.CpuStats.Percent / 100) * float64(resource.CPU) - memoryStats := ru.ResourceUsage.MemoryStats - cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU) - memUsage = fmt.Sprintf("%v/%v", memoryStats.RSS/(1024*1024), resource.MemoryMB) + if cs := ru.ResourceUsage.CpuStats; cs != nil { + cpuTicksConsumed := (cs.Percent / 100) * float64(resource.CPU) + cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cpuTicksConsumed), resource.CPU) + } + if ms := ru.ResourceUsage.MemoryStats; ms != nil { + memUsage = fmt.Sprintf("%v/%v", ms.RSS/(1024*1024), resource.MemoryMB) + } } resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v|%v MB|%v MB|%v|%v", cpuUsage, @@ -387,26 +390,65 @@ func (c *AllocStatusCommand) taskResources(alloc *api.Allocation, stats map[stri func (c *AllocStatusCommand) printTaskResourceUsage(task string, resourceUsage *api.ResourceUsage) { memoryStats := resourceUsage.MemoryStats cpuStats := resourceUsage.CpuStats - c.Ui.Output("Memory Stats") - out := make([]string, 2) - out[0] = "RSS|Cache|Swap|Max Usage|Kernel Usage|Kernel Max Usage" - out[1] = fmt.Sprintf("%v|%v|%v|%v|%v|%v", - humanize.Bytes(memoryStats.RSS), - humanize.Bytes(memoryStats.Cache), - humanize.Bytes(memoryStats.Swap), - humanize.Bytes(memoryStats.MaxUsage), - humanize.Bytes(memoryStats.KernelUsage), - humanize.Bytes(memoryStats.KernelMaxUsage), - ) - c.Ui.Output(formatList(out)) - - c.Ui.Output("") - - c.Ui.Output("CPU Stats") - out = make([]string, 2) - out[0] = "Percent|Throttled Periods|Throttled Time" - percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) - out[1] = fmt.Sprintf("%v %%|%v|%v", percent, - cpuStats.ThrottledPeriods, cpuStats.ThrottledTime) - c.Ui.Output(formatList(out)) + if memoryStats != nil && len(memoryStats.Measured) > 0 { + c.Ui.Output("Memory Stats") + + // Sort the measured stats + sort.Strings(memoryStats.Measured) + + var measuredStats []string + for _, measured := range memoryStats.Measured { + switch measured { + case "RSS": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.RSS)) + case "Cache": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Cache)) + case "Swap": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.Swap)) + case "Max Usage": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.MaxUsage)) + case "Kernel Usage": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelUsage)) + case "Kernel Max Usage": + measuredStats = append(measuredStats, humanize.Bytes(memoryStats.KernelMaxUsage)) + } + } + + out := make([]string, 2) + out[0] = strings.Join(memoryStats.Measured, "|") + out[1] = strings.Join(measuredStats, "|") + c.Ui.Output(formatList(out)) + c.Ui.Output("") + } + + if cpuStats != nil && len(cpuStats.Measured) > 0 { + c.Ui.Output("CPU Stats") + + // Sort the measured stats + sort.Strings(cpuStats.Measured) + + var measuredStats []string + for _, measured := range cpuStats.Measured { + switch measured { + case "Percent": + percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) + measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) + case "Throttled Periods": + measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) + case "Throttled Time": + measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) + case "User Mode": + percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) + measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) + case "System Mode": + percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) + measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) + } + } + + out := make([]string, 2) + out[0] = strings.Join(cpuStats.Measured, "|") + out[1] = strings.Join(measuredStats, "|") + c.Ui.Output(formatList(out)) + } }