Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for resource limits to play kube #7853

Merged
merged 1 commit into from
Oct 12, 2020
Merged

Add support for resource limits to play kube #7853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions pkg/domain/infra/abi/play.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ import (
"github.com/sirupsen/logrus"
v1apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)

const (
// https://kubernetes.io/docs/concepts/storage/volumes/#hostpath
kubeDirectoryPermission = 0755
// https://kubernetes.io/docs/concepts/storage/volumes/#hostpath
kubeFilePermission = 0644
// Kubernetes sets CPUPeriod to 100000us (100ms): https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/
defaultCPUPeriod = 100000
)

func (ic *ContainerEngine) PlayKube(ctx context.Context, path string, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) {
Expand Down Expand Up @@ -506,6 +509,27 @@ func kubeContainerToCreateConfig(ctx context.Context, containerYAML v1.Container
// but apply to the containers with the prefixed name
securityConfig.SeccompProfilePath = seccompPaths.findForContainer(containerYAML.Name)

var err error
milliCPU, err := quantityToInt64(containerYAML.Resources.Limits.Cpu())
if err != nil {
return nil, errors.Wrap(err, "Failed to set CPU quota")
}
if milliCPU > 0 {
containerConfig.Resources.CPUPeriod = defaultCPUPeriod
// CPU quota is a fraction of the period: milliCPU / 1000.0 * period
// Or, without floating point math:
containerConfig.Resources.CPUQuota = milliCPU * defaultCPUPeriod / 1000
}

containerConfig.Resources.Memory, err = quantityToInt64(containerYAML.Resources.Limits.Memory())
if err != nil {
return nil, errors.Wrap(err, "Failed to set memory limit")
}
containerConfig.Resources.MemoryReservation, err = quantityToInt64(containerYAML.Resources.Requests.Memory())
if err != nil {
return nil, errors.Wrap(err, "Failed to set memory reservation")
}

containerConfig.Command = []string{}
if imageData != nil && imageData.Config != nil {
containerConfig.Command = imageData.Config.Entrypoint
Expand Down Expand Up @@ -748,3 +772,15 @@ func verifySeccompPath(path string, profileRoot string) (string, error) {
return "", errors.Errorf("invalid seccomp path: %s", path)
}
}

func quantityToInt64(quantity *resource.Quantity) (int64, error) {
if i, ok := quantity.AsInt64(); ok {
return i, nil
}

if i, ok := quantity.AsDec().Unscaled(); ok {
return i, nil
}

return 0, errors.Errorf("Quantity cannot be represented as int64: %v", quantity)
}
9 changes: 9 additions & 0 deletions test/e2e/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,15 @@ func SkipIfRootlessCgroupsV1(reason string) {
}
}

func SkipIfUnprevilegedCPULimits() {
info := GetHostDistributionInfo()
if isRootless() &&
info.Distribution == "fedora" &&
(info.Version == "31" || info.Version == "32") {
ginkgo.Skip("Rootless Fedora doesn't have permission to set CPU limits before version 33")
}
}

func SkipIfRootless(reason string) {
checkReason(reason)
if os.Geteuid() != 0 {
Expand Down
131 changes: 127 additions & 4 deletions test/e2e/play_kube_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"text/template"

Expand Down Expand Up @@ -111,7 +112,19 @@ spec:
image: {{ .Image }}
name: {{ .Name }}
imagePullPolicy: {{ .PullPolicy }}
resources: {}
{{- if or .CpuRequest .CpuLimit .MemoryRequest .MemoryLimit }}
resources:
{{- if or .CpuRequest .MemoryRequest }}
requests:
{{if .CpuRequest }}cpu: {{ .CpuRequest }}{{ end }}
{{if .MemoryRequest }}memory: {{ .MemoryRequest }}{{ end }}
{{- end }}
{{- if or .CpuLimit .MemoryLimit }}
limits:
{{if .CpuLimit }}cpu: {{ .CpuLimit }}{{ end }}
{{if .MemoryLimit }}memory: {{ .MemoryLimit }}{{ end }}
{{- end }}
{{- end }}
xordspar0 marked this conversation as resolved.
Show resolved Hide resolved
{{ if .SecurityContext }}
securityContext:
allowPrivilegeEscalation: true
Expand Down Expand Up @@ -223,7 +236,19 @@ spec:
image: {{ .Image }}
name: {{ .Name }}
imagePullPolicy: {{ .PullPolicy }}
resources: {}
{{- if or .CpuRequest .CpuLimit .MemoryRequest .MemoryLimit }}
resources:
{{- if or .CpuRequest .MemoryRequest }}
requests:
{{if .CpuRequest }}cpu: {{ .CpuRequest }}{{ end }}
{{if .MemoryRequest }}memory: {{ .MemoryRequest }}{{ end }}
{{- end }}
{{- if or .CpuLimit .MemoryLimit }}
limits:
{{if .CpuLimit }}cpu: {{ .CpuLimit }}{{ end }}
{{if .MemoryLimit }}memory: {{ .MemoryLimit }}{{ end }}
{{- end }}
{{- end }}
{{ if .SecurityContext }}
securityContext:
allowPrivilegeEscalation: true
Expand Down Expand Up @@ -261,6 +286,8 @@ var (
defaultDeploymentName = "testDeployment"
defaultConfigMapName = "testConfigMap"
seccompPwdEPERM = []byte(`{"defaultAction":"SCMP_ACT_ALLOW","syscalls":[{"name":"getcwd","action":"SCMP_ACT_ERRNO"}]}`)
// CPU Period in ms
defaultCPUPeriod = 100
)

func writeYaml(content string, fileName string) error {
Expand Down Expand Up @@ -503,6 +530,10 @@ type Ctr struct {
Image string
Cmd []string
Arg []string
CpuRequest string
CpuLimit string
MemoryRequest string
MemoryLimit string
SecurityContext bool
Caps bool
CapAdd []string
Expand All @@ -521,7 +552,25 @@ type Ctr struct {
// getCtr takes a list of ctrOptions and returns a Ctr with sane defaults
// and the configured options
func getCtr(options ...ctrOption) *Ctr {
c := Ctr{defaultCtrName, defaultCtrImage, defaultCtrCmd, defaultCtrArg, true, false, nil, nil, "", "", "", false, "", "", false, []Env{}, []EnvFrom{}}
c := Ctr{
Name: defaultCtrName,
Image: defaultCtrImage,
Cmd: defaultCtrCmd,
Arg: defaultCtrArg,
SecurityContext: true,
Caps: false,
CapAdd: nil,
CapDrop: nil,
PullPolicy: "",
HostIP: "",
Port: "",
VolumeMount: false,
VolumeMountPath: "",
VolumeName: "",
VolumeReadOnly: false,
Env: []Env{},
EnvFrom: []EnvFrom{},
}
for _, option := range options {
option(&c)
}
Expand All @@ -548,6 +597,30 @@ func withImage(img string) ctrOption {
}
}

func withCpuRequest(request string) ctrOption {
return func(c *Ctr) {
c.CpuRequest = request
}
}

func withCpuLimit(limit string) ctrOption {
return func(c *Ctr) {
c.CpuLimit = limit
}
}

func withMemoryRequest(request string) ctrOption {
return func(c *Ctr) {
c.MemoryRequest = request
}
}

func withMemoryLimit(limit string) ctrOption {
return func(c *Ctr) {
c.MemoryLimit = limit
}
}

func withSecurityContext(sc bool) ctrOption {
return func(c *Ctr) {
c.SecurityContext = sc
Expand Down Expand Up @@ -648,7 +721,12 @@ type EnvFrom struct {
From string
}

var _ = Describe("Podman generate kube", func() {
func milliCPUToQuota(milliCPU string) int {
milli, _ := strconv.Atoi(strings.Trim(milliCPU, "m"))
return milli * defaultCPUPeriod
}

var _ = Describe("Podman play kube", func() {
var (
tempdir string
err error
Expand Down Expand Up @@ -1324,4 +1402,49 @@ spec:
Expect(inspect.OutputToString()).To(ContainSubstring(correctLabels))
}
})

It("podman play kube allows setting resource limits", func() {
SkipIfContainerized("Resource limits require a running systemd")
SkipIfRootlessCgroupsV1("Limits require root or cgroups v2")
SkipIfUnprevilegedCPULimits()
podmanTest.CgroupManager = "systemd"

var (
numReplicas int32 = 3
expectedCpuRequest string = "100m"
expectedCpuLimit string = "200m"
expectedMemoryRequest string = "10000000"
expectedMemoryLimit string = "20000000"
)

expectedCpuQuota := milliCPUToQuota(expectedCpuLimit)

deployment := getDeployment(
withReplicas(numReplicas),
withPod(getPod(withCtr(getCtr(
withCpuRequest(expectedCpuRequest),
withCpuLimit(expectedCpuLimit),
withMemoryRequest(expectedMemoryRequest),
withMemoryLimit(expectedMemoryLimit),
)))))
err := generateKubeYaml("deployment", deployment, kubeYaml)
Expect(err).To(BeNil())

kube := podmanTest.Podman([]string{"play", "kube", kubeYaml})
kube.WaitWithDefaultTimeout()
Expect(kube.ExitCode()).To(Equal(0))

for _, pod := range getPodNamesInDeployment(deployment) {
inspect := podmanTest.Podman([]string{"inspect", getCtrNameInPod(&pod), "--format", `
CpuPeriod: {{ .HostConfig.CpuPeriod }}
CpuQuota: {{ .HostConfig.CpuQuota }}
Memory: {{ .HostConfig.Memory }}
MemoryReservation: {{ .HostConfig.MemoryReservation }}`})
inspect.WaitWithDefaultTimeout()
Expect(inspect.ExitCode()).To(Equal(0))
Expect(inspect.OutputToString()).To(ContainSubstring(fmt.Sprintf("%s: %d", "CpuQuota", expectedCpuQuota)))
Expect(inspect.OutputToString()).To(ContainSubstring("MemoryReservation: " + expectedMemoryRequest))
Expect(inspect.OutputToString()).To(ContainSubstring("Memory: " + expectedMemoryLimit))
}
})
})