From b9f16dc68100e41f19a6a78f92c5e27dc5fab177 Mon Sep 17 00:00:00 2001 From: dan mcweeney Date: Fri, 2 Feb 2024 17:37:38 -0500 Subject: [PATCH] Add ability to scale Ephemeral storage along with memory, similar to CPU (#5008) * Add ability to scale Ephemeral storage along with memory, similar to CPU. * Enforce the limit even when scaling ephemeral storage * Code formatter --- .../src/main/resources/application.conf | 5 ++- .../kubernetes/KubernetesClient.scala | 2 +- .../kubernetes/WhiskPodBuilder.scala | 9 +++- .../test/WhiskPodBuilderTests.scala | 42 ++++++++++++++++++- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/core/invoker/src/main/resources/application.conf b/core/invoker/src/main/resources/application.conf index f04dfdbac52..61c33e063d8 100644 --- a/core/invoker/src/main/resources/application.conf +++ b/core/invoker/src/main/resources/application.conf @@ -120,10 +120,13 @@ whisk { #} #if missing, the pod will be created without ephermal disk request/limit - #if specified, the pod will be created with ephemeral-storage request+limit set + #if specified, the pod will be created with ephemeral-storage request+limit set or using the scale factor + #as a multiple of the request memory. If the scaled value exceeds the limit, the limit will be used so, it's good + #practice to set the limit if you plan on using the scale-factor. #See: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#local-ephemeral-storage #ephemeral-storage { # limit = 2 g + # scale-factor = 2.0 #} #enable PodDisruptionBudget creation for pods? (will include same labels as pods, and specify minAvailable=1 to prevent termination of action pods during maintenance) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala index ccfcf5c4afc..1e878c19f41 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/KubernetesClient.scala @@ -74,7 +74,7 @@ case class KubernetesCpuScalingConfig(millicpus: Int, memory: ByteSize, maxMilli /** * Configuration for kubernetes ephemeral storage limit for the action container */ -case class KubernetesEphemeralStorageConfig(limit: ByteSize) +case class KubernetesEphemeralStorageConfig(limit: ByteSize, scaleFactor: Double) /** * Exception to indicate a pod took too long to become ready. diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala index 17b81366d0e..af3dedac7ef 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/kubernetes/WhiskPodBuilder.scala @@ -108,7 +108,14 @@ class WhiskPodBuilder(client: NamespacedKubernetesClient, config: KubernetesClie .getOrElse(Map.empty) val diskLimit = config.ephemeralStorage - .map(diskConfig => Map("ephemeral-storage" -> new Quantity(diskConfig.limit.toMB + "Mi"))) + .map( + diskConfig => + // Scale the ephemeral storage unless it exceeds the limit, if it exceeds the limit use the limit. + if ((diskConfig.scaleFactor > 0) && (diskConfig.scaleFactor * memory.toMB < diskConfig.limit.toMB)) { + Map("ephemeral-storage" -> new Quantity(diskConfig.scaleFactor * memory.toMB + "Mi")) + } else { + Map("ephemeral-storage" -> new Quantity(diskConfig.limit.toMB + "Mi")) + }) .getOrElse(Map.empty) //In container its assumed that env, port, resource limits are set explicitly diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala index 07bb3ad7d63..d15c5803a89 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/kubernetes/test/WhiskPodBuilderTests.scala @@ -133,7 +133,47 @@ class WhiskPodBuilderTests extends FlatSpec with Matchers with KubeClientSupport Some(KubernetesCpuScalingConfig(300, 3.MB, 1000)), false, None, - Some(KubernetesEphemeralStorageConfig(1.GB))) + Some(KubernetesEphemeralStorageConfig(1.GB, 0.0))) + val builder = new WhiskPodBuilder(kubeClient, config) + + val (pod, _) = builder.buildPodSpec(name, testImage, 2.MB, Map("foo" -> "bar"), Map("fooL" -> "barV"), config) + withClue(Serialization.asYaml(pod)) { + val c = getActionContainer(pod) + c.getResources.getLimits.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("1024Mi") + c.getResources.getRequests.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("1024Mi") + } + } + it should "scale ephemeral storage when scale factor is given" in { + val config = KubernetesClientConfig( + KubernetesClientTimeoutConfig(1.second, 1.second), + KubernetesInvokerNodeAffinity(false, "k", "v"), + true, + None, + None, + Some(KubernetesCpuScalingConfig(300, 3.MB, 1000)), + false, + None, + Some(KubernetesEphemeralStorageConfig(1.GB, 1.25))) + val builder = new WhiskPodBuilder(kubeClient, config) + + val (pod, _) = builder.buildPodSpec(name, testImage, 2.MB, Map("foo" -> "bar"), Map("fooL" -> "barV"), config) + withClue(Serialization.asYaml(pod)) { + val c = getActionContainer(pod) + c.getResources.getLimits.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("2.5Mi") + c.getResources.getRequests.asScala.get("ephemeral-storage").map(_.getAmount) shouldBe Some("2.5Mi") + } + } + it should "use ephemeral storage limit when scale factor suggests larger size" in { + val config = KubernetesClientConfig( + KubernetesClientTimeoutConfig(1.second, 1.second), + KubernetesInvokerNodeAffinity(false, "k", "v"), + true, + None, + None, + Some(KubernetesCpuScalingConfig(300, 3.MB, 1000)), + false, + None, + Some(KubernetesEphemeralStorageConfig(1.GB, 1000))) val builder = new WhiskPodBuilder(kubeClient, config) val (pod, _) = builder.buildPodSpec(name, testImage, 2.MB, Map("foo" -> "bar"), Map("fooL" -> "barV"), config)