[JENKINS-35246] Test that Jenkins nodes are deleted after agent disconnection

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java

@@ -1,14 +1,17 @@
+
 package org.csanchez.jenkins.plugins.kubernetes;
 
 import java.io.IOException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.UnrecoverableKeyException;
 import java.security.cert.CertificateEncodingException;
+import java.util.concurrent.ExecutionException;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -18,7 +21,6 @@
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.Validate;
 import org.jenkinsci.plugins.durabletask.executors.OnceRetentionStrategy;
-import org.jvnet.localizer.Localizable;
 import org.jvnet.localizer.ResourceBundleHolder;
 import org.kohsuke.stapler.DataBoundConstructor;
 
@@ -190,13 +192,14 @@ protected void _terminate(TaskListener listener) throws IOException, Interrupted
             return;
         }
 
-        OfflineCause offlineCause = OfflineCause.create(new Localizable(HOLDER, "offline"));
+        OfflineCause offlineCause = OfflineCause.create(hudson.model.Messages._Hudson_NodeBeingRemoved());
 
         Future<?> disconnected = computer.disconnect(offlineCause);
         // wait a bit for disconnection to avoid stack traces in logs
         try {
             disconnected.get(DISCONNECTION_TIMEOUT, TimeUnit.SECONDS);
-        } catch (Exception e) {
+            LOGGER.log(Level.INFO, "Disconnected computer: {0}", name);
+        } catch (ExecutionException | TimeoutException e) {
             String msg = String.format("Ignoring error waiting for agent disconnection %s: %s", name, e.getMessage());
             LOGGER.log(Level.INFO, msg, e);
         }
@@ -245,7 +248,6 @@ protected void _terminate(TaskListener listener) throws IOException, Interrupted
         String msg = String.format("Terminated Kubernetes instance for agent %s/%s", actualNamespace, name);
         LOGGER.log(Level.INFO, msg);
         listener.getLogger().println(msg);
-        LOGGER.log(Level.INFO, "Disconnected computer {0}", name);
     }
 
     @Override

src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java

@@ -43,6 +43,7 @@
 
 import org.apache.commons.lang.StringUtils;
 
+import hudson.model.Node;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 
@@ -57,6 +58,7 @@
 import io.fabric8.kubernetes.client.Watch;
 import io.fabric8.kubernetes.client.Watcher;
 import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable;
+import jenkins.model.Jenkins;
 
 public class KubernetesTestUtil {
 
@@ -123,9 +125,35 @@ public static Map<String, String> getLabels(Object o) {
         return l;
     }
 
+    /**
+     * Wait for Jenkins nodes (agents) to be deleted
+     */
+    public static void waitForNodeDeletion(Jenkins jenkins) throws Exception {
+        try {
+            new ForkJoinPool(1).submit(() -> IntStream.range(1, 1_000_000).anyMatch(i -> {
+                try {
+                    List<Node> nodes = jenkins.getNodes();
+                    LOGGER.log(INFO, "Still waiting for nodes to be deleted: {0}", nodes);
+                    if (nodes.isEmpty()) {
+                        LOGGER.log(INFO, "All nodes are deleted");
+                    } else {
+                        LOGGER.log(INFO, "Still waiting for nodes to be deleted: {0}", nodes);
+                        Thread.sleep(5000);
+                    }
+                    return nodes.isEmpty();
+                } catch (InterruptedException e) {
+                    LOGGER.log(INFO, "Waiting for nodes to be deleted - interrupted");
+                    return true;
+                }
+            })).get(30, TimeUnit.SECONDS);
+        } catch (TimeoutException e) {
+            LOGGER.log(INFO, "Waiting for nodes to be deleted - timed out");
+        }
+    }
+
     /**
      * Delete pods with matching labels
-     * 
+     *
      * @param client
      * @param labels
      * @param wait
@@ -185,4 +213,4 @@ private static List<String> print(FilterWatchListDeletable<Pod, PodList, Boolean
                 .map(pod -> String.format("%s (%s)", pod.getMetadata().getName(), pod.getStatus().getPhase()))
                 .collect(Collectors.toList());
     }
-}
+}

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java

@@ -38,6 +38,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.jvnet.hudson.test.Issue;
 import org.jvnet.hudson.test.JenkinsRuleNonLocalhost;
 
 import hudson.model.Result;
@@ -102,7 +103,12 @@ public void runInPodFromYaml() throws Exception {
         PodTemplate template = templates.get(0);
         assertEquals(Integer.MAX_VALUE, template.getInstanceCap());
         r.assertBuildStatusSuccess(r.waitForCompletion(b));
-        r.assertLogContains("script file contents: ", b);
+        r.assertLogContains("PID file contents: ", b);
+
+        // check that nodes and pods are deleted
+        waitForNodeDeletion(r.getInstance());
+        assertEquals("There are agents left in Jenkins after test execution", Collections.emptyList(),
+                r.getInstance().getNodes());
         assertFalse("There are pods leftover after test execution, see previous logs",
                 deletePods(cloud.connect(), getLabels(this), true));
     }
@@ -343,4 +349,15 @@ public void runWithActiveDeadlineSeconds() throws Exception {
         r.assertLogNotContains("Hello from container!", b);
     }
 
+    @Test
+    @Issue("JENKINS-35246")
+    public void failing() throws Exception {
+        WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
+        p.setDefinition(new CpsFlowDefinition(loadPipelineScript("failing.groovy"), true));
+        WorkflowRun b = p.scheduleBuild2(0).waitForStart();
+        assertNotNull(b);
+        r.assertBuildStatus(Result.FAILURE, r.waitForCompletion(b));
+        r.assertLogContains("will fail", b);
+    }
+
 }

src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/failing.groovy

@@ -0,0 +1,15 @@
+podTemplate(label: 'failing', containers: [
+        containerTemplate(name: 'busybox', image: 'busybox', ttyEnabled: true, command: '/bin/cat'),
+    ]) {
+
+    node ('failing') {
+      stage('Run') {
+        container('busybox') {
+          sh """
+            echo will fail
+            false
+          """
+        }
+      }
+    }
+}