Skip to content

Commit

Permalink
Add logging for unexpected pod eviction errors
Browse files Browse the repository at this point in the history
  • Loading branch information
dewjam committed Mar 2, 2022
1 parent 761554c commit ab0d4cf
Showing 1 changed file with 10 additions and 9 deletions.
19 changes: 10 additions & 9 deletions pkg/controllers/termination/eviction.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,10 @@ func (e *EvictionQueue) Start(ctx context.Context) {
break
}
nn := item.(types.NamespacedName)

// Evict pod
if e.evict(ctx, nn) {
logging.FromContext(ctx).Debugf("Evicted pod %s", nn.String())
logging.FromContext(ctx).Debug("Evicted pod")
e.RateLimitingInterface.Forget(nn)
e.Set.Remove(nn)
e.RateLimitingInterface.Done(nn)
Expand All @@ -83,26 +84,26 @@ func (e *EvictionQueue) Start(ctx context.Context) {
// Requeue pod if eviction failed
e.RateLimitingInterface.AddRateLimited(nn)
}
logging.FromContext(ctx).Errorf("EvictionQueue is broken and has shutdown.")
logging.FromContext(ctx).Errorf("EvictionQueue is broken and has shutdown")
}

// evict returns true if successful eviction call, error is returned if not eviction-related error
func (e *EvictionQueue) evict(ctx context.Context, nn types.NamespacedName) bool {
ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("pod", nn))

err := e.coreV1Client.Pods(nn.Namespace).Evict(ctx, &v1beta1.Eviction{
ObjectMeta: metav1.ObjectMeta{Name: nn.Name, Namespace: nn.Namespace},
})
if errors.IsInternalError(err) { // 500
logging.FromContext(ctx).Errorf("Could not evict pod %s due to PDB misconfiguration error.", nn.String())
return false

if errors.IsNotFound(err) { // 404
return true
}
if errors.IsTooManyRequests(err) { // 429
logging.FromContext(ctx).Debugf("Did not evict pod %s due to PDB violation.", nn.String())
logging.FromContext(ctx).Debug(err)
return false
}
if errors.IsNotFound(err) { // 404
return true
}
if err != nil {
logging.FromContext(ctx).Error(err)
return false
}
return true
Expand Down

0 comments on commit ab0d4cf

Please sign in to comment.