diff --git a/pkg/util/runtime/cpu_linux.go b/pkg/util/runtime/cpu_linux.go index cfc49d9241..7db6090539 100644 --- a/pkg/util/runtime/cpu_linux.go +++ b/pkg/util/runtime/cpu_linux.go @@ -36,15 +36,42 @@ import ( // // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt func NumCPU() int { + return NumCPUWithCustomPath("") +} + +func NumCPUWithCustomPath(path string) int { cpus := runtime.NumCPU() - cgroupPath, err := libcontainercgroups.FindCgroupMountpoint("", "cpu") - if err != nil { - return cpus + cgroupVersionCheckPath := path + + if cgroupVersionCheckPath == "" { + cgroupVersionCheckPath = "/sys/fs/cgroup/" } - cpuQuota := readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us") - cpuPeriod := readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us") + cgroupVersion := GetCgroupVersion(cgroupVersionCheckPath) + cpuQuota := int64(-1) + cpuPeriod := int64(-1) + + if cgroupVersion == 1 { + cgroupPath := "" + if path == "" { + cgroupPathRd, err := libcontainercgroups.FindCgroupMountpoint("", "cpu") + if err != nil { + return cpus + } + cgroupPath = cgroupPathRd + } else { + cgroupPath = path + } + cpuQuota = readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us") + cpuPeriod = readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us") + } else if cgroupVersion == 2 { + cgroupPath := "/sys/fs/cgroup/" + if path != "" { + cgroupPath = path + } + cpuQuota, cpuPeriod = readCgroup2FileToInt64Tuple(cgroupPath, "cpu.max") + } if cpuQuota == -1 || cpuPeriod == -1 { return cpus @@ -53,16 +80,66 @@ func NumCPU() int { return int(math.Ceil(float64(cpuQuota) / float64(cpuPeriod))) } -func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 { +func GetCgroupVersion(cgroupPath string) int64 { + // /sys/fs/cgroup/cgroup.controllers will not exist with cgroupsv1 + if _, err := os.Stat(filepath.Join(cgroupPath, "cgroup.controllers")); err == nil { + return 2 + } + + return 1 +} + +func readCgroup2StringToInt64Tuple(cgroupString string) (quota, period int64) { + // file contents looks like: $MAX $PERIOD + // $MAX can have value "max" indicating no limit + // it is possible for $PERIOD to be unset + + values := strings.Fields(cgroupString) + + if values[0] == "max" { + return -1, -1 + } + + cpuQuota, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return -1, -1 + } + + if len(values) == 1 { + return cpuQuota, 100000 + } + + cpuPeriod, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return -1, -1 + } + + return cpuQuota, cpuPeriod +} + +func readCgroup2FileToInt64Tuple(cgroupPath, cgroupFile string) (quota, period int64) { contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile)) if err != nil { - return -1 + return -1, -1 } - strValue := strings.TrimSpace(string(contents)) + return readCgroup2StringToInt64Tuple(string(contents)) +} + +func readCgroupStringToInt64(contents string) int64 { + strValue := strings.TrimSpace(contents) if value, err := strconv.ParseInt(strValue, 10, 64); err == nil { return value } return -1 } + +func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 { + contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile)) + if err != nil { + return -1 + } + + return readCgroupStringToInt64(string(contents)) +} diff --git a/test/e2e/cgroups/cgroups.go b/test/e2e/cgroups/cgroups.go new file mode 100644 index 0000000000..eab1943240 --- /dev/null +++ b/test/e2e/cgroups/cgroups.go @@ -0,0 +1,115 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cgroups + +import ( + "log" + "os" + "path/filepath" + + "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + + "k8s.io/ingress-nginx/test/e2e/framework" + + "k8s.io/ingress-nginx/pkg/util/runtime" +) + +var _ = framework.IngressNginxDescribeSerial("[CGroups] cgroups", func() { + f := framework.NewDefaultFramework("cgroups") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + f.NewSlowEchoDeployment() + }) + + ginkgo.It("detects cgroups version v1", func() { + cgroupPath := "/testing/sys/fs/cgroup/" + if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil { + log.Fatal(err) + } + + quotaFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_quota_us")) + if err != nil { + log.Fatal(err) + } + + periodFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_period_us")) + if err != nil { + log.Fatal(err) + } + + _, err = quotaFile.WriteString("4") + if err != nil { + log.Fatal(err) + } + + err = quotaFile.Sync() + if err != nil { + log.Fatal(err) + } + + _, err = periodFile.WriteString("2") + if err != nil { + log.Fatal(err) + } + + err = periodFile.Sync() + if err != nil { + log.Fatal(err) + } + + assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(1)) + assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2) + + os.Remove(filepath.Join(cgroupPath, "cpu.cfs_quota_us")) + os.Remove(filepath.Join(cgroupPath, "cpu.cfs_period_us")) + }) + + ginkgo.It("detect cgroups version v2", func() { + cgroupPath := "/testing/sys/fs/cgroup/" + if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil { + log.Fatal(err) + } + + _, err := os.Create(filepath.Join(cgroupPath, "cgroup.controllers")) + if err != nil { + log.Fatal(err) + } + + file, err := os.Create(filepath.Join(cgroupPath, "cpu.max")) + if err != nil { + log.Fatal(err) + } + + _, err = file.WriteString("4 2") + if err != nil { + log.Fatal(err) + } + + err = file.Sync() + if err != nil { + log.Fatal(err) + } + + assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(2)) + assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2) + + os.Remove(filepath.Join(cgroupPath, "cpu.max")) + os.Remove(filepath.Join(cgroupPath, "cgroup.controllers")) + }) +}) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 25d714f884..631e70ae46 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -32,6 +32,7 @@ import ( _ "k8s.io/ingress-nginx/test/e2e/admission" _ "k8s.io/ingress-nginx/test/e2e/annotations" _ "k8s.io/ingress-nginx/test/e2e/annotations/modsecurity" + _ "k8s.io/ingress-nginx/test/e2e/cgroups" _ "k8s.io/ingress-nginx/test/e2e/dbg" _ "k8s.io/ingress-nginx/test/e2e/defaultbackend" _ "k8s.io/ingress-nginx/test/e2e/endpointslices"