Respect telemetry.collection_interval (#130)

hashicorp · Aug 11, 2021 · 057d0c4 · 057d0c4
1 parent 57a9efe
commit 057d0c4
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 16 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@ FEATURES:
 
 BUG FIXES:
 * log: Use error key context to log errors rather than Go err style. [[GH-126](https://github.com/hashicorp/nomad-driver-podman/pull/126)]
+* telemetry: respect telemetry.collection_interval to reduce cpu churn when running many containers [[GH-130](https://github.com/hashicorp/nomad-driver-podman/pull/130)]
 
 ## 0.3.0
 

diff --git a/driver.go b/driver.go
@@ -271,13 +271,14 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
 	}
 
 	h := &TaskHandle{
-		containerID: taskState.ContainerID,
-		driver:      d,
-		taskConfig:  taskState.TaskConfig,
-		procState:   drivers.TaskStateUnknown,
-		startedAt:   taskState.StartedAt,
-		exitResult:  &drivers.ExitResult{},
-		logger:      d.logger.Named("podmanHandle"),
+		containerID:        taskState.ContainerID,
+		driver:             d,
+		taskConfig:         taskState.TaskConfig,
+		procState:          drivers.TaskStateUnknown,
+		startedAt:          taskState.StartedAt,
+		exitResult:         &drivers.ExitResult{},
+		logger:             d.logger.Named("podmanHandle"),
+		collectionInterval: time.Second,
 
 		totalCPUStats:  stats.NewCpuStats(),
 		userCPUStats:   stats.NewCpuStats(),
@@ -555,13 +556,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	}
 
 	h := &TaskHandle{
-		containerID: containerID,
-		driver:      d,
-		taskConfig:  cfg,
-		procState:   drivers.TaskStateRunning,
-		exitResult:  &drivers.ExitResult{},
-		startedAt:   time.Now().Round(time.Millisecond),
-		logger:      d.logger.Named("podmanHandle"),
+		containerID:        containerID,
+		driver:             d,
+		taskConfig:         cfg,
+		procState:          drivers.TaskStateRunning,
+		exitResult:         &drivers.ExitResult{},
+		startedAt:          time.Now().Round(time.Millisecond),
+		logger:             d.logger.Named("podmanHandle"),
+		collectionInterval: time.Second,
 
 		totalCPUStats:  stats.NewCpuStats(),
 		userCPUStats:   stats.NewCpuStats(),

diff --git a/examples/nomad/client.hcl b/examples/nomad/client.hcl
@@ -29,6 +29,13 @@ plugin "raw_exec" {
   }
 }
 
+telemetry {
+  # you should align the collection_interval to your
+  # metrics system. A very short interval of 1-2 secs
+  # puts considerable strain on your system
+  collection_interval = "10s"
+}
+
 # different port than server
 ports {
   http = 7646

diff --git a/handle.go b/handle.go
@@ -28,6 +28,8 @@ type TaskHandle struct {
 	userCPUStats   *stats.CpuStats
 	systemCPUStats *stats.CpuStats
 
+	collectionInterval time.Duration
+
 	// stateLock syncs access to all fields below
 	stateLock sync.RWMutex
 
@@ -95,6 +97,7 @@ func (h *TaskHandle) runExitWatcher(ctx context.Context, exitChannel chan *drive
 func (h *TaskHandle) runStatsEmitter(ctx context.Context, statsChannel chan *drivers.TaskResourceUsage, interval time.Duration) {
 	timer := time.NewTimer(0)
 	h.logger.Debug("Starting statsEmitter", "container", h.containerID)
+	h.collectionInterval = interval
 	for {
 		select {
 		case <-ctx.Done():
@@ -141,7 +144,6 @@ func (h *TaskHandle) runStatsEmitter(ctx context.Context, statsChannel chan *dri
 func (h *TaskHandle) runContainerMonitor() {
 
 	timer := time.NewTimer(0)
-	interval := time.Second * 1
 	h.logger.Debug("Monitoring container", "container", h.containerID)
 
 	cleanup := func() {
@@ -155,7 +157,7 @@ func (h *TaskHandle) runContainerMonitor() {
 			return
 
 		case <-timer.C:
-			timer.Reset(interval)
+			timer.Reset(h.collectionInterval)
 		}
 
 		containerStats, statsErr := h.driver.podman.ContainerStats(h.driver.ctx, h.containerID)