diff --git a/salt/_modules/metalk8s_drain.py b/salt/_modules/metalk8s_drain.py index 601d95e2e7..37fb391be8 100644 --- a/salt/_modules/metalk8s_drain.py +++ b/salt/_modules/metalk8s_drain.py @@ -8,6 +8,7 @@ module when called by salt by virtue of its `__virtualname__` attribute. ''' +import json import logging import operator import time @@ -345,6 +346,7 @@ def run_drain(self, dry_run=False): Returns: string message Raises: CommandExecutionError in case of timeout or eviction failure ''' + log.debug("Beginning drain of Node {}".format(self.node_name)) try: pods = self.get_pods_for_eviction() except DrainException as exc: @@ -358,15 +360,20 @@ def run_drain(self, dry_run=False): ) ) + if pods: + pods_to_evict = ", ".join([ + pod['metadata']['name'] for pod in pods + ]) + else + pods_to_evict = "no pods to evict." + if dry_run: # Would be nice to create the Eviction in dry-run mode and see if # we hit some 429 Too Many Requests (because a disruption budget # would prevent the eviction) - return "Prepared for eviction of pods: {0}".format( - ", ".join([pod['metadata']['name'] for pod in pods]) - if pods else "no pods to evict." - ) + return "Prepared for eviction of pods: {}".format(pods_to_evict) + log.debug("Starting eviction of pods: {}".format(pods_to_evict)) try: self.evict_pods(pods) except DrainTimeoutException as exc: @@ -434,6 +441,11 @@ def wait_for_eviction(self, pods): response['metadata']['uid'] != pod['metadata']['uid']: log.info("%s evicted", pod['metadata']['name']) else: + log.debug( + "Waiting for eviction of Pod %s (current status: %s)", + pod['metadata']['name'], + pod.get('status', {}).get('phase'), + ) pending.append(pod) if not pending: @@ -507,10 +519,22 @@ def evict_pod(name, namespace='default', grace_period=1, if isinstance(exc, ApiException): if exc.status == 404: # Seems to be ignored in kubectl, let's do the same + log.debug( + "Received '404 Not Found' when creating Eviction for %s, " + "ignoring", + name, + ) return True if exc.status == 429: # Too Many Requests: the eviction is rejected, but indicates # we should retry later (probably due to a disruption budget) + status = json.loads(exc.body, encoding="utf-8") + log.info( + "Cannot evict %s at the moment: %s", + name, + status['message'], + ) + # When implemented by Kubernetes, read the `Retry-After` advice return False raise CommandExecutionError(