feature(): use custom scheduller in k8s (#1080)

* feature(): use custom scheduller * tests(): add tests to increase coverage * docs(charts): added k8s schedulerName tips * feature(): use custom scheduller * tests(): add tests to increase coverage * docs(charts): added k8s schedulerName tips
zalando · Dec 10, 2019 · 714f87c · 714f87c
1 parent 50bfb41
commit 714f87c
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 4 deletions.
diff --git a/charts/zalenium/templates/_pod-template.yaml b/charts/zalenium/templates/_pod-template.yaml
@@ -5,6 +5,10 @@ metadata:
   annotations:
     {{- toYaml .Values.hub.podAnnotations | nindent 4 }}
 spec:
+  {{- if (.Values.hub.schedulerName) }}
+  schedulerName: {{ .Values.hub.schedulerName }}
+  {{- end}}
+
   securityContext:
     {{- toYaml .Values.hub.podSecurityContext | nindent 4 }}
   {{- if .Values.hub.pullSecrets }}

diff --git a/charts/zalenium/values.yaml b/charts/zalenium/values.yaml
@@ -73,6 +73,16 @@ hub:
   # Annotations to be added to the service.
   ##
   serviceAnnotations: {}
+
+  # This enables using a custom scheduler for both hub and browser pods
+  # This could be useful to have a cost effective pod distribution with
+  # warmmed up browsers in k8s nodes by using a custom scheduler
+  # (i.e. by deploying a new one in your cluster) configured with
+  # MostRequestedPriority (https://kubernetes.io/docs/concepts/scheduling/)
+  # so that nodes tend to take all resources in nodes instead of flooding
+  # the nodes, easing downscaling.
+  #schedulerName: null
+
   # Below we expose metrics to Prometheus.
   #   prometheus.io/path: /metrics
   #   prometheus.io/scrape: "true"

diff --git a/src/main/java/de/zalando/ep/zalenium/container/kubernetes/KubernetesContainerClient.java b/src/main/java/de/zalando/ep/zalenium/container/kubernetes/KubernetesContainerClient.java
@@ -12,6 +12,7 @@
 import io.fabric8.kubernetes.api.model.HostAlias;
 import io.fabric8.kubernetes.api.model.LocalObjectReference;
 import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.PodFluent;
 import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.api.model.PodSecurityContext;
 import io.fabric8.kubernetes.api.model.Quantity;
@@ -73,6 +74,7 @@ public class KubernetesContainerClient implements ContainerClient {
     private Map<String, String> nodeSelector = new HashMap<>();
     private List<Toleration> tolerations = new ArrayList<>();
     private String imagePullPolicy;
+    private String schedulerName;
     private List<LocalObjectReference> imagePullSecrets;
     private PodSecurityContext configuredPodSecurityContext;
     private SecurityContext configuredContainerSecurityContext;
@@ -114,6 +116,7 @@ public KubernetesContainerClient(Environment environment,
             discoverTolerations();
             discoverImagePullSecrets();
             discoverPodSecurityContext();
+            discoverSchedulerName();
             discoverContainerSecurityContext();
             buildResourceMaps();
 
@@ -187,6 +190,10 @@ private void discoverTolerations() {
         }
     }
 
+    private void discoverSchedulerName() {
+        schedulerName = zaleniumPod.getSpec().getSchedulerName();
+    }
+
     private void discoverImagePullSecrets() {
         List<LocalObjectReference> configuredPullSecrets = zaleniumPod.getSpec().getImagePullSecrets();
         if (!configuredPullSecrets.isEmpty()) {
@@ -225,7 +232,7 @@ private String findHostname() {
 
         return hostname;
     }
-    
+
     private void discoverPodSecurityContext() {
     	configuredPodSecurityContext = zaleniumPod.getSpec().getSecurityContext();
     }
@@ -367,6 +374,7 @@ public ContainerCreationStatus createContainer(String zaleniumContainerName, Str
         config.setPodLimits(seleniumPodLimits);
         config.setPodRequests(seleniumPodRequests);
         config.setOwner(zaleniumPod);
+        config.setSchedulerName(schedulerName);
         config.setPodSecurityContext(configuredPodSecurityContext);
         config.setContainerSecurityContext(configuredContainerSecurityContext);
 
@@ -576,7 +584,8 @@ private enum ImagePullPolicyType {
     }
 
     public static DoneablePod createDoneablePodDefaultImpl(PodConfiguration config) {
-        DoneablePod doneablePod = config.getClient().pods()
+
+        PodFluent.SpecNested<DoneablePod> doneablePodSpecNested = config.getClient().pods()
                 .createNew()
                 .withNewMetadata()
                     .withGenerateName(config.getContainerIdPrefix())
@@ -623,8 +632,13 @@ public static DoneablePod createDoneablePodDefaultImpl(PodConfiguration config)
                         .endReadinessProbe()
                     .endContainer()
                     .withRestartPolicy("Never")
-                    .withImagePullSecrets(config.getImagePullSecrets())
-                .endSpec();
+                    .withImagePullSecrets(config.getImagePullSecrets());
+
+        if(config.getSchedulerName() != null) {
+            doneablePodSpecNested = doneablePodSpecNested.withSchedulerName(config.getSchedulerName());
+        }
+
+        DoneablePod doneablePod = doneablePodSpecNested.endSpec();
 
         // Add the shared folders if available
         for (Map.Entry<VolumeMount, Volume> entry : config.getMountedSharedFoldersMap().entrySet()) {

diff --git a/src/main/java/de/zalando/ep/zalenium/container/kubernetes/PodConfiguration.java b/src/main/java/de/zalando/ep/zalenium/container/kubernetes/PodConfiguration.java
@@ -36,6 +36,7 @@ public class PodConfiguration {
     private OwnerReference ownerReference;
     private PodSecurityContext podSecurityContext;
     private SecurityContext containerSecurityContext;
+    private String schedulerName;
 
     public String getNodePort() {
         return nodePort;
@@ -146,4 +147,12 @@ public SecurityContext getContainerSecurityContext() {
     public void setContainerSecurityContext(SecurityContext containerSecurityContext) {
         this.containerSecurityContext = containerSecurityContext;
     }
+
+    public String getSchedulerName() {
+        return schedulerName;
+    }
+
+    public void setSchedulerName(String schedulerName) {
+        this.schedulerName = schedulerName;
+    }
 }
diff --git a/src/test/java/de/zalando/ep/zalenium/container/kubernetes/PodConfigurationTest.java b/src/test/java/de/zalando/ep/zalenium/container/kubernetes/PodConfigurationTest.java
@@ -163,4 +163,12 @@ public void testSetContainerSecurityContext() {
         podConfiguration.setContainerSecurityContext(securityContext);
         assertThat(podConfiguration.getContainerSecurityContext(), is(securityContext));
     }
+
+    @Test
+    public void testSetSchedulerName() {
+        final String schedulerName = "custom-scheduler";
+        podConfiguration.setSchedulerName(schedulerName);
+        assertThat(podConfiguration.getSchedulerName(), is(schedulerName));
+    }
+
 }
diff --git a/src/test/java/de/zalando/ep/zalenium/util/KubernetesContainerMock.java b/src/test/java/de/zalando/ep/zalenium/util/KubernetesContainerMock.java
@@ -103,6 +103,7 @@ public static KubernetesContainerClient getMockedKubernetesContainerClient() {
                 .withVolumes(videosVolume, generalVolume)
                 .withHostAliases(hostAlias)
                 .withNodeSelector(nodeSelector)
+                .withSchedulerName("custom-scheduler")
                 .build();
         zaleniumPod.setSpec(zaleniumPodSpec);