re-enable the PDB eviction test

- the pod must be in a non-pending state for eviction
  to consider the PDB
- the Status.ObservedGeneration must be >= the pdb.Generation
  or the eviction results in a 429 retry with a 10 second delay

Fixes aws#638

pkg/controllers/termination/suite_test.go

@@ -204,9 +204,17 @@ var _ = Describe("Termination", func() {
 			podNoEvict := test.Pod(test.PodOptions{
 				NodeName:   node.Name,
 				ObjectMeta: metav1.ObjectMeta{Labels: labelSelector},
+				Phase:      v1.PodRunning,
 			})
 
-			ExpectCreated(ctx, env.Client, node, podNoEvict, pdb)
+			ExpectCreated(ctx, env.Client, node, pdb)
+			ExpectCreatedWithStatus(ctx, env.Client, podNoEvict)
+
+			// kube-controller-manager normally does this, but we don't have one. If this isn't modified
+			// the eviction controller assumes that the PDB hasn't been processed by the disruption controller
+			// yet and adds a 10 second retry to our evict() call
+			pdb.Status.ObservedGeneration = 1
+			ExpectStatusUpdated(ctx, env.Client, pdb)
 
 			// Trigger Termination Controller
 			Expect(env.Client.Delete(ctx, node)).To(Succeed())
@@ -219,15 +227,18 @@ var _ = Describe("Termination", func() {
 			// Expect node to exist and be draining
 			ExpectNodeDraining(env.Client, node.Name)
 
+			// Expect podNoEvict to fail eviction due to PDB, and be retried
+			ExpectEvictionWasRetried(podNoEvict)
+
 			// Delete pod to simulate successful eviction
 			ExpectDeleted(ctx, env.Client, podNoEvict)
+			ExpectNotFound(ctx, env.Client, podNoEvict)
 
 			// Reconcile to delete node
 			node = ExpectNodeExists(ctx, env.Client, node.Name)
 			ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(node))
 			ExpectNotFound(ctx, env.Client, node)
 		})
-
 		It("should evict non-critical pods first", func() {
 			podEvict := test.Pod(test.PodOptions{NodeName: node.Name})
 			podNodeCritical := test.Pod(test.PodOptions{NodeName: node.Name, PriorityClassName: "system-node-critical"})
@@ -357,6 +368,13 @@ var _ = Describe("Termination", func() {
 	})
 })
 
+func ExpectEvictionWasRetried(pod *v1.Pod) {
+	Eventually(func(g Gomega) bool {
+		return g.Expect(evictionQueue.NumRequeues(client.ObjectKeyFromObject(pod))).
+			Should(BeNumerically(">", 1))
+	}).Should(BeTrue())
+}
+
 func ExpectEnqueuedForEviction(e *termination.EvictionQueue, pods ...*v1.Pod) {
 	for _, pod := range pods {
 		Expect(e.Contains(client.ObjectKeyFromObject(pod))).To(BeTrue())