Merge pull request #343 from opsani/fred/eng-439-k8s-connector-suppor…

…t-for-configuration

k8s support for static env var config

servo/connectors/kubernetes.py

@@ -2997,6 +2997,9 @@ async def create(
                     f'no container named "{container_config.name}" exists in the Pod (found {names})'
                 )
 
+            if container_config.static_environment_variables:
+                raise NotImplementedError("Configurable environment variables are not currently supported under Deployment optimization (saturation mode)")
+
             name = container_config.alias or (
                 f"{deployment.name}/{container.name}" if container else deployment.name
             )
@@ -3420,6 +3423,22 @@ async def _configure_tuning_pod_template_spec(self) -> None:
         container = Container(container_obj, None)
         servo.logger.debug(f"Initialized new tuning container from Pod spec template: {container.name}")
 
+        if self.container_config.static_environment_variables:
+            if container.obj.env is None:
+                container.obj.env = []
+
+            # Filter out vars with the same name as the ones we are setting
+            container.obj.env = list(filter(
+                lambda e: e.name not in self.container_config.static_environment_variables,
+                container.obj.env
+            ))
+
+            env_list = [
+                kubernetes_asyncio.client.V1EnvVar(name=k, value=v)
+                for k, v in self.container_config.static_environment_variables.items()
+            ]
+            container.obj.env.extend(env_list)
+
         if self.tuning_container:
             servo.logger.debug(f"Copying resource requirements from existing tuning pod container '{self.tuning_pod.name}/{self.tuning_container.name}'")
             resource_requirements = self.tuning_container.resources
@@ -4067,7 +4086,8 @@ class ContainerConfiguration(servo.BaseConfiguration):
     command: Optional[str]  # TODO: create model...
     cpu: CPU
     memory: Memory
-    env: Optional[List[str]]  # TODO: create model...
+    env: Optional[List[str]]  # (adjustable environment variables) TODO: create model...
+    static_environment_variables: Optional[Dict[str, str]]
 
 
 

servo/connectors/opsani_dev.py

@@ -57,6 +57,7 @@ class OpsaniDevConfiguration(servo.BaseConfiguration):
     port: Optional[Union[pydantic.StrictInt, str]] = None
     cpu: CPU
     memory: Memory
+    static_environment_variables: Optional[Dict[str, str]]
     prometheus_base_url: str = PROMETHEUS_SIDECAR_BASE_URL
     envoy_sidecar_image: str = ENVOY_SIDECAR_IMAGE_TAG
     timeout: servo.Duration = "5m"
@@ -109,6 +110,7 @@ def generate_kubernetes_config(
                     alias="main",
                     cpu=self.cpu,
                     memory=self.memory,
+                    static_environment_variables=self.static_environment_variables,
                 )
             ],
         )

tests/connectors/kubernetes_test.py

@@ -45,7 +45,7 @@
 )
 from servo.errors import AdjustmentFailedError, AdjustmentRejectedError
 import servo.runner
-from servo.types import Adjustment
+from servo.types import Adjustment, Component, Description, Replicas
 from tests.helpers import *
 
 
@@ -1164,12 +1164,46 @@ async def test_read_pod(self, config, kube) -> None:
 
     ##
     # Canary Tests
-    # async def test_create_canary(self, tuning_config, namespace: str) -> None:
-   #      connector = KubernetesConnector(config=tuning_config)
-   #      dep = await Deployment.read("fiber-http", namespace)
-   #      debug(dep)
-        # description = await connector.startup()
-        # debug(description)
+    async def test_create_tuning(
+        self,
+        tuning_config: KubernetesConfiguration,
+        kube: kubetest.client.TestClient
+    ) -> None:
+        # verify existing env vars are overriden by config var with same name
+        main_dep = kube.get_deployments()["fiber-http"]
+        main_dep.obj.spec.template.spec.containers[0].env = [kubernetes.client.models.V1EnvVar(name="FOO", value="BAZ")]
+        main_dep.api_client.patch_namespaced_deployment(main_dep.name, main_dep.namespace, main_dep.obj)
+        tuning_config.deployments[0].containers[0].static_environment_variables = { "FOO": "BAR" }
+
+        connector = KubernetesConnector(config=tuning_config)
+        description = await connector.describe()
+
+        assert description == Description(components=[
+            Component(
+                name='fiber-http/fiber-http',
+                settings=[
+                    CPU(name='cpu', type='range', pinned=True, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']),
+                    Memory(name='mem', type='range', pinned=True, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']),
+                    Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=99999, step=1)
+                ]
+            ),
+            Component(
+                name='fiber-http/fiber-http-tuning',
+                settings=[
+                    CPU(name='cpu', type='range', pinned=False, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']),
+                    Memory(name='mem', type='range', pinned=False, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']),
+                    Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=1, step=1)
+                ]
+            )
+        ])
+
+        tuning_pod = kube.get_pods()["fiber-http-tuning"]
+        assert tuning_pod.obj.metadata.annotations["opsani.com/opsani_tuning_for"] == "fiber-http/fiber-http-tuning"
+        assert tuning_pod.obj.metadata.labels["opsani_role"] == "tuning"
+        target_container = next(filter(lambda c: c.name == "fiber-http" , tuning_pod.obj.spec.containers))
+        assert target_container.resources.requests == {'cpu': '125m', 'memory': '128Mi'}
+        assert target_container.resources.limits == {'cpu': '125m', 'memory': '128Mi'}
+        assert target_container.env == [kubernetes.client.models.V1EnvVar(name="FOO", value="BAR")]
 
     async def test_adjust_tuning_insufficient_mem(
         self,
@@ -2384,10 +2418,40 @@ def _rollout_tuning_config(self, tuning_config: KubernetesConfiguration) -> Kube
 
     ##
     # Canary Tests
-    async def test_create_rollout_tuning(self, _rollout_tuning_config: KubernetesConfiguration, namespace: str) -> None:
+    async def test_create_rollout_tuning(
+        self, _rollout_tuning_config: KubernetesConfiguration, kube: kubetest.client.TestClient, namespace: str
+    ) -> None:
+        _rollout_tuning_config.rollouts[0].containers[0].static_environment_variables = { "FOO": "BAR" }
         connector = KubernetesConnector(config=_rollout_tuning_config)
         rol = await Rollout.read("fiber-http", namespace)
-        await connector.describe()
+        description = await connector.describe()
+
+        assert description == Description(components=[
+            Component(
+                name='fiber-http/fiber-http',
+                settings=[
+                    CPU(name='cpu', type='range', pinned=True, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']),
+                    Memory(name='mem', type='range', pinned=True, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']),
+                    Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=99999, step=1)
+                ]
+            ),
+            Component(
+                name='fiber-http/fiber-http-tuning',
+                settings=[
+                    CPU(name='cpu', type='range', pinned=False, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']),
+                    Memory(name='mem', type='range', pinned=False, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']),
+                    Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=1, step=1)
+                ]
+            )
+        ])
+
+        tuning_pod = kube.get_pods()["fiber-http-tuning"]
+        assert tuning_pod.obj.metadata.annotations["opsani.com/opsani_tuning_for"] == "fiber-http/fiber-http-tuning"
+        assert tuning_pod.obj.metadata.labels["opsani_role"] == "tuning"
+        target_container = next(filter(lambda c: c.name == "fiber-http" , tuning_pod.obj.spec.containers))
+        assert target_container.resources.requests == {'cpu': '125m', 'memory': '128Mi'}
+        assert target_container.resources.limits == {'cpu': '125m', 'memory': '128Mi'}
+        assert target_container.env == [kubernetes.client.models.V1EnvVar(name="FOO", value="BAR")]
 
         # verify tuning pod is registered as service endpoint
         service = await servo.connectors.kubernetes.Service.read("fiber-http", namespace)

tests/connectors/opsani_dev_test.py

@@ -71,7 +71,10 @@ def rollout_checks(rollout_config: servo.connectors.opsani_dev.OpsaniDevConfigur
 class TestConfig:
     def test_generate(self) -> None:
         config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate()
-        assert list(config.dict().keys()) == ['description', 'namespace', 'deployment', 'rollout', 'container', 'service', 'port', 'cpu', 'memory', 'prometheus_base_url', 'envoy_sidecar_image', 'timeout', 'settlement']
+        assert list(config.dict().keys()) == [
+            'description', 'namespace', 'deployment', 'rollout', 'container', 'service','port', 'cpu', 'memory',
+            'static_environment_variables', 'prometheus_base_url', 'envoy_sidecar_image', 'timeout', 'settlement'
+        ]
 
     def test_generate_yaml(self) -> None:
         config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate()
@@ -92,6 +95,26 @@ def test_assign_optimizer(self) -> None:
         config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate()
         config.__optimizer__ = None
 
+    def test_generate_kubernetes_config(self) -> None:
+        opsani_dev_config = servo.connectors.opsani_dev.OpsaniDevConfiguration(
+            namespace="test",
+            deployment="fiber-http",
+            container="fiber-http",
+            service="fiber-http",
+            cpu=servo.connectors.kubernetes.CPU(min="125m", max="4000m", step="125m"),
+            memory=servo.connectors.kubernetes.Memory(min="128 MiB", max="4.0 GiB", step="128 MiB"),
+            static_environment_variables={"FOO": "BAR", "BAZ": 1},
+            __optimizer__=servo.configuration.Optimizer(id="test.com/foo", token="12345")
+        )
+        kubernetes_config = opsani_dev_config.generate_kubernetes_config()
+        assert kubernetes_config.namespace == "test"
+        assert kubernetes_config.deployments[0].namespace == "test"
+        assert kubernetes_config.deployments[0].name == "fiber-http"
+        assert kubernetes_config.deployments[0].containers[0].name == "fiber-http"
+        assert kubernetes_config.deployments[0].containers[0].cpu == servo.connectors.kubernetes.CPU(min="125m", max="4000m", step="125m")
+        assert kubernetes_config.deployments[0].containers[0].memory == servo.connectors.kubernetes.Memory(min="128 MiB", max="4.0 GiB", step="128 MiB")
+        assert kubernetes_config.deployments[0].containers[0].static_environment_variables == {"FOO": "BAR", "BAZ": "1"}
+
     def test_generate_rollout_config(self) -> None:
         rollout_config = servo.connectors.opsani_dev.OpsaniDevConfiguration(
             namespace="test",