Skip to content

Commit

Permalink
oomwatch: small tweaks
Browse files Browse the repository at this point in the history
- Change memory usage percent threshold to `uint8` to no longer allow
  fractions.
- Validate interval to prevent configurations `<50ms`.

Signed-off-by: Hidde Beydals <[email protected]>
  • Loading branch information
hiddeco committed Mar 6, 2023
1 parent 3dbb013 commit 718daea
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 10 deletions.
16 changes: 10 additions & 6 deletions internal/oomwatch/watch.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ type Watcher struct {
memoryCurrentPath string
// memoryUsagePercentThreshold is the threshold at which the system is
// considered to be near OOM.
memoryUsagePercentThreshold float64
memoryUsagePercentThreshold uint8
// interval is the interval at which to check for OOM.
interval time.Duration
// logger is the logger to use.
Expand All @@ -62,9 +62,13 @@ type Watcher struct {
}

// New returns a new Watcher.
func New(memoryMaxPath, memoryCurrentPath string, memoryUsagePercentThreshold float64, interval time.Duration, logger logr.Logger) (*Watcher, error) {
func New(memoryMaxPath, memoryCurrentPath string, memoryUsagePercentThreshold uint8, interval time.Duration, logger logr.Logger) (*Watcher, error) {
if memoryUsagePercentThreshold < 1 || memoryUsagePercentThreshold > 100 {
return nil, fmt.Errorf("memory usage percent threshold must be between 1 and 100, got %.2f", memoryUsagePercentThreshold)
return nil, fmt.Errorf("memory usage percent threshold must be between 1 and 100, got %d", memoryUsagePercentThreshold)
}

if minInterval := 50 * time.Millisecond; interval < minInterval {
return nil, fmt.Errorf("interval must be at least %s, got %s", minInterval, interval)
}

if _, err := os.Lstat(memoryCurrentPath); err != nil {
Expand All @@ -86,7 +90,7 @@ func New(memoryMaxPath, memoryCurrentPath string, memoryUsagePercentThreshold fl
}

// NewDefault returns a new Watcher with default path values.
func NewDefault(memoryUsagePercentThreshold float64, interval time.Duration, logger logr.Logger) (*Watcher, error) {
func NewDefault(memoryUsagePercentThreshold uint8, interval time.Duration, logger logr.Logger) (*Watcher, error) {
return New(
filepath.Join(DefaultCgroupPath, MemoryMaxFile),
filepath.Join(DefaultCgroupPath, MemoryCurrentFile),
Expand Down Expand Up @@ -128,13 +132,13 @@ func (w *Watcher) watchForNearOOM(ctx context.Context) {
}

currentPercentage := float64(current) / float64(w.memoryMax) * 100
if currentPercentage >= w.memoryUsagePercentThreshold {
if currentPercentage >= float64(w.memoryUsagePercentThreshold) {
w.logger.Info(fmt.Sprintf("Memory usage is near OOM (%s/%s), shutting down",
formatSize(current), formatSize(w.memoryMax)))
w.cancel()
return
}
w.logger.V(2).Info(fmt.Sprintf("Current memory usage %s/%s (%.2f%% out of %.2f%%)",
w.logger.V(2).Info(fmt.Sprintf("Current memory usage %s/%s (%.2f%% out of %d%%)",
formatSize(current), formatSize(w.memoryMax), currentPercentage, w.memoryUsagePercentThreshold))
}
}
Expand Down
4 changes: 2 additions & 2 deletions internal/oomwatch/watch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,14 @@ func TestNew(t *testing.T) {

_, err := New("", "", 0, 0, logr.Discard())
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(MatchError("memory usage percent threshold must be between 1 and 100, got 0.00"))
g.Expect(err).To(MatchError("memory usage percent threshold must be between 1 and 100, got 0"))
})
t.Run("greater than 100", func(t *testing.T) {
g := NewWithT(t)

_, err := New("", "", 101, 0, logr.Discard())
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(MatchError("memory usage percent threshold must be between 1 and 100, got 101.00"))
g.Expect(err).To(MatchError("memory usage percent threshold must be between 1 and 100, got 101"))
})
})

Expand Down
4 changes: 2 additions & 2 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func main() {
leaderElectionOptions leaderelection.Options
rateLimiterOptions helper.RateLimiterOptions
oomWatchInterval time.Duration
oomWatchMemoryThreshold float64
oomWatchMemoryThreshold uint8
)

flag.StringVar(&metricsAddr, "metrics-addr", ":8080",
Expand All @@ -107,7 +107,7 @@ func main() {
"The maximum number of retries when failing to fetch artifacts over HTTP.")
flag.StringVar(&intkube.DefaultServiceAccountName, "default-service-account", "",
"Default service account used for impersonation.")
flag.Float64Var(&oomWatchMemoryThreshold, "oom-watch-memory-threshold", 95,
flag.Uint8Var(&oomWatchMemoryThreshold, "oom-watch-memory-threshold", 95,
"The memory threshold in percentage at which the OOM watcher will trigger a graceful shutdown. Requires feature gate 'OOMWatch' to be enabled.")
flag.DurationVar(&oomWatchInterval, "oom-watch-interval", 500*time.Millisecond,
"The interval at which the OOM watcher will check for memory usage. Requires feature gate 'OOMWatch' to be enabled.")
Expand Down

0 comments on commit 718daea

Please sign in to comment.