diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b81aaf8ae..e8770573af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,10 @@ - Add liveness probe to `keepalived` pod (PR[#4118](https://github.com/scality/metalk8s/pull/4118)) +- Add `kubeReserved` and `systemReserved` resources allocation to `KubeletConfiguration` + following [Google's recommandations](https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#memory_and_cpu_reservations) + (PR[#4134](https://github.com/scality/metalk8s/pull/4134)) + ## Release 125.0.6 (In development) ### Enhancements diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 93640042c7..f32338d313 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -667,6 +667,7 @@ def task(self) -> types.TaskDict: Path("salt/_modules/metalk8s_kubernetes_utils.py"), Path("salt/_modules/metalk8s_monitoring.py"), Path("salt/_modules/metalk8s_network.py"), + Path("salt/_modules/metalk8s_os.py"), Path("salt/_modules/metalk8s_package_manager_yum.py"), Path("salt/_modules/metalk8s_service_configuration.py"), Path("salt/_modules/metalk8s_solutions.py"), diff --git a/salt/_modules/metalk8s_os.py b/salt/_modules/metalk8s_os.py new file mode 100644 index 0000000000..470e5d469a --- /dev/null +++ b/salt/_modules/metalk8s_os.py @@ -0,0 +1,59 @@ +""" +MetalK8s OS module +""" + +__virtualname__ = "metalk8s_os" + + +def __virtual__(): + return __virtualname__ + + +def get_kubereserved(): + """Get kubeReserved memory and cpu allocations following Google's GKE recommandations. + + For CPU resources, GKE reserves the following: + - 6% of the first core + - 1% of the next core (up to 2 cores) + - 0.5% of the next 2 cores (up to 4 cores) + - 0.25% of any cores above 4 cores + + For memory resources, GKE reserves the following: + - 255 MiB of memory for machines with less than 1 GB of memory + - 25% of the first 4GB of memory + - 20% of the next 4GB of memory (up to 8GB) + - 10% of the next 8GB of memory (up to 16GB) + - 6% of the next 112GB of memory (up to 128GB) + - 2% of any memory above 128GB + + + https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes#memory_and_cpu_reservations + https://learnk8s.io/allocatable-resources + """ + + os_cpu = __grains__["num_cpus"] + + core_1 = 0.06 + cores_2 = 0.01 * max([(min([os_cpu, 2]) - 1), 0]) + cores_4 = 0.005 * max([(min([os_cpu, 4]) - 2), 0]) + cores_above_4 = 0.0025 * max([(os_cpu - 4), 0]) + + kube_cpu = round((core_1 + cores_2 + cores_4 + cores_above_4) * 1000) + + os_memory = __grains__["mem_total"] + + gb_4 = 4 * 1024 + gb_8 = 8 * 1024 + gb_16 = 16 * 1024 + gb_128 = 128 * 1024 + + memory_4 = 0.25 * min([os_memory, gb_4]) + memory_8 = 0.2 * max([(min([os_memory, gb_8]) - gb_4), 0]) + memory_16 = 0.1 * max([(min([os_memory, gb_16]) - gb_8), 0]) + memory_128 = 0.06 * max([(min([os_memory, gb_128]) - gb_16), 0]) + memory_above_128 = 0.02 * max([(os_memory - gb_128), 0]) + + kube_memory = round(memory_4 + memory_8 + memory_16 + memory_128 + memory_above_128) + + # CPU is in millicores, memory is in MiB + return {"cpu": f"{kube_cpu}m", "memory": f"{kube_memory}Mi"} diff --git a/salt/metalk8s/kubernetes/kubelet/standalone.sls b/salt/metalk8s/kubernetes/kubelet/standalone.sls index 78ea7469c7..ec9ed92fe6 100644 --- a/salt/metalk8s/kubernetes/kubelet/standalone.sls +++ b/salt/metalk8s/kubernetes/kubelet/standalone.sls @@ -103,6 +103,10 @@ Create kubelet config file: {%- if pillar.get("kubernetes:kubelet:config:maxPods") %} maxPods: {{ pillar.kubernetes.kubelet.config.maxPods }} {%- endif %} + systemReserved: + cpu: 200m + memory: 200Mi + kubeReserved: {{ salt.metalk8s_os.get_kubereserved() | tojson }} {%- for key, value in kubelet.config.items() %} {{ key }}: {{ value }} {%- endfor %} diff --git a/salt/tests/unit/formulas/fixtures/salt.py b/salt/tests/unit/formulas/fixtures/salt.py index 705f2ab6c2..d95c004281 100644 --- a/salt/tests/unit/formulas/fixtures/salt.py +++ b/salt/tests/unit/formulas/fixtures/salt.py @@ -26,6 +26,7 @@ "file_client": "remote", } + # The "public methods" are dynamically added by the `register` decorator # pylint: disable=too-few-public-methods class SaltMock: @@ -187,6 +188,7 @@ def register_basic(func_name: str) -> Callable[[MockFunc], MockFunc]: # }}} # Mock definitions {{{ + # Data-driven mocks {{{ @register("config.get") def config_get(salt_mock: SaltMock, *args: Any, **kwargs: Any) -> Any: @@ -548,5 +550,9 @@ def random_get_str(length: int = 20) -> str: return "".join(random.choices(allowed_chars, k=length)) +register_basic("metalk8s_os.get_kubereserved")( + MagicMock(return_value={"cpu": "100m", "memory": "1000Mi"}) +) + # }}} # }}} diff --git a/salt/tests/unit/modules/files/test_metalk8s_os.yaml b/salt/tests/unit/modules/files/test_metalk8s_os.yaml new file mode 100644 index 0000000000..1f2b4a057b --- /dev/null +++ b/salt/tests/unit/modules/files/test_metalk8s_os.yaml @@ -0,0 +1,67 @@ +get_kubereserved: + # 00 - 1 core and 2 GB of RAM + - num_cpus: 1 + mem_total: 2048 + result: + cpu: 60m + memory: 512Mi + # 01 - 2 cores and 4 GB of RAM + - num_cpus: 2 + mem_total: 4096 + result: + cpu: 70m + memory: 1024Mi + # 02 - 4 cores and 8 GB of RAM + - num_cpus: 4 + mem_total: 8192 + result: + cpu: 80m + memory: 1843Mi + # 03 - 8 cores and 16 GB of RAM + - num_cpus: 8 + mem_total: 16384 + result: + cpu: 90m + memory: 2662Mi + # 04 - 16 cores and 32 GB of RAM + - num_cpus: 16 + mem_total: 32768 + result: + cpu: 110m + memory: 3645Mi + # 05 - 32 cores and 64 GB of RAM + - num_cpus: 32 + mem_total: 65536 + result: + cpu: 150m + memory: 5612Mi + # 06 - 64 cores and 128 GB of RAM + - num_cpus: 64 + mem_total: 131072 + result: + cpu: 230m + memory: 9544Mi + # 07 - 128 cores and 256 GB of RAM + - num_cpus: 128 + mem_total: 262144 + result: + cpu: 390m + memory: 12165Mi + # 08 - 256 cores and 512 GB of RAM + - num_cpus: 256 + mem_total: 524288 + result: + cpu: 710m + memory: 17408Mi + # 09 - 512 cores and 1024 GB of RAM + - num_cpus: 512 + mem_total: 1048576 + result: + cpu: 1350m + memory: 27894Mi + # 10 - 3 cores and 14.2 GB of RAM + - num_cpus: 3 + mem_total: 14540 + result: + cpu: 75m + memory: 2478Mi \ No newline at end of file diff --git a/salt/tests/unit/modules/test_metalk8s_os.py b/salt/tests/unit/modules/test_metalk8s_os.py new file mode 100644 index 0000000000..ba2b6890c9 --- /dev/null +++ b/salt/tests/unit/modules/test_metalk8s_os.py @@ -0,0 +1,45 @@ +import os.path +import yaml + +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from _modules import metalk8s_os + +from tests.unit import mixins +from tests.unit import utils + + +YAML_TESTS_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "files", "test_metalk8s_os.yaml" +) +with open(YAML_TESTS_FILE) as fd: + YAML_TESTS_CASES = yaml.safe_load(fd) + + +class Metalk8sOsTestCase(TestCase, mixins.LoaderModuleMockMixin): + """ + TestCase for `metalk8s_os` module + """ + + loader_module = metalk8s_os + + def test_virtual(self): + """ + Tests the return of `__virtual__` function + """ + self.assertEqual(metalk8s_os.__virtual__(), "metalk8s_os") + + @utils.parameterized_from_cases(YAML_TESTS_CASES["get_kubereserved"]) + def test_get_kubereserved(self, num_cpus, mem_total, result): + """ + Tests the return of `get_kubereserved` function + """ + + grains_dict = {"num_cpus": num_cpus, "mem_total": mem_total} + + with patch.dict(metalk8s_os.__grains__, grains_dict): + self.assertEqual( + metalk8s_os.get_kubereserved(), + result, + )