Skip to content

Commit

Permalink
Setup new node image with swap enabled
Browse files Browse the repository at this point in the history
* Support Ubuntu and Fedora
* Tested with docker command on node

Co-Authored-By: Elana Hashman <[email protected]>
  • Loading branch information
ike-ma and ehashman committed Jun 17, 2021
1 parent 68fa3e6 commit 4fad20b
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 0 deletions.
76 changes: 76 additions & 0 deletions config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -565,3 +565,79 @@ presubmits:
env:
- name: GOPATH
value: /go

- name: pull-kubernetes-node-kubelet-swap-ubuntu
always_run: false
optional: true
skip_report: false
skip_branches:
- release-\d+\.\d+ # per-release image
annotations:
testgrid-dashboards: sig-node-kubelet
testgrid-tab-name: pr-kubelet-gce-e2e-swap-ubuntu
labels:
preset-service-account: "true"
preset-k8s-ssh: "true"
spec:
containers:
- image: gcr.io/k8s-testimages/kubekins-e2e:v20210601-ea6aa4e-master
args:
- --repo=k8s.io/kubernetes=$(PULL_REFS)
- --repo=k8s.io/release
- --timeout=240
- --root=/go/src
- --scenario=kubernetes_e2e
- --
- --deployment=node
- --gcp-project=k8s-jkns-pr-node-e2e
- --gcp-zone=us-west1-b
- --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/swap/image-config-swap.yaml
- --node-test-args=--feature-gates=NodeSwapEnabled=true --kubelet-flags="--cgroups-per-qos=true --cgroup-root=/ --fail-swap-on=false"
- --node-tests=true
- --provider=gce
- --test_args=--nodes=8 --focus="\[NodeConformance\]|\[NodeFeature:.+\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]"
- --timeout=180m
env:
- name: GOPATH
value: /go

- name: pull-kubernetes-node-swap-fedora
skip_branches:
- release-\d+\.\d+ # per-release image
annotations:
testgrid-dashboards: sig-node-kubelet
testgrid-tab-name: pr-kubelet-gce-e2e-swap-fedora
always_run: true
skip_report: true
optional: true
max_concurrency: 12
labels:
preset-service-account: "true"
preset-k8s-ssh: "true"
preset-pull-kubernetes-e2e: "true"
preset-pull-kubernetes-e2e-gce: "true"
spec:
containers:
- image: gcr.io/k8s-testimages/kubekins-e2e:v20210601-ea6aa4e-master
args:
- --root=/go/src
- "--job=$(JOB_NAME)"
- "--repo=k8s.io/$(REPO_NAME)=$(PULL_REFS)"
- "--service-account=/etc/service-account/service-account.json"
- "--upload=gs://kubernetes-jenkins/pr-logs"
- "--timeout=240"
- --scenario=kubernetes_e2e
- -- # end bootstrap args, scenario args below
- --deployment=node
- --env=KUBE_SSH_USER=core
- --gcp-project=k8s-jkns-pr-node-e2e
- --gcp-zone=us-west1-b
- '--node-test-args=--feature-gates=NodeSwapEnabled=true --container-runtime=remote --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--fail-swap-on=false --cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service --non-masquerade-cidr=0.0.0.0/0" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"'
- --node-tests=true
- --provider=gce
- --test_args=--nodes=8 --focus="\[NodeConformance\]|\[NodeFeature:.+\]" --skip="\[Flaky\]|\[Slow\]|\[Serial\]"
- --timeout=180m
- --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/swap/image-config-swap-fedora.yaml
resources:
requests:
memory: "6Gi"
1 change: 1 addition & 0 deletions gubernator/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ jobs:
- pull-kubernetes-node-crio-e2e
- pull-kubernetes-node-e2e
- pull-kubernetes-node-e2e-containerd
- pull-kubernetes-node-swap-fedora
- pull-kubernetes-typecheck
- pull-kubernetes-unit
- pull-kubernetes-verify
Expand Down
24 changes: 24 additions & 0 deletions jobs/e2e_node/swap/crio_swap1g.ign
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"ignition": {
"version": "3.1.0"
},
"systemd": {
"units": [
{
"contents": "[Unit]\nDescription=Enable swap on CoreOS\nBefore=swap-enable.service\nConditionFirstBoot=yes\n\n[Service]\nType=oneshot\nExecStart=/bin/sh -c \"sudo rpm-ostree install dbus-tools && sudo systemctl reboot\"\n[Install]\n\nWantedBy=multi-user.target\n",
"enabled": true,
"name": "dbus-send-install.service"
},
{
"contents": "[Unit]\nDescription=Enable swap on CoreOS\nBefore=crio-install.service\nConditionFirstBoot=no\n\n[Service]\nType=oneshot\nExecStart=/bin/sh -c \"sudo dd if=/dev/zero of=/var/swapfile count=1024 bs=1MiB && sudo chmod 600 /var/swapfile && sudo mkswap /var/swapfile && sudo swapon /var/swapfile && free -h\"\n[Install]\n\nWantedBy=multi-user.target\n",
"enabled": true,
"name": "swap-enable.service"
},
{
"contents": "[Unit]\nDescription=Download and install crio binaries and configurations.\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStartPre=/usr/sbin/setenforce 1\nExecStartPre=/usr/bin/bash -c '/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -o /usr/local/crio-install.sh https://raw.githubusercontent.com/cri-o/cri-o/master/scripts/get'\nExecStartPre=/usr/bin/bash /usr/local/crio-install.sh\nExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t var_lib_t /var/lib/kubelet\nExecStartPre=/usr/bin/mount /tmp /tmp -o remount,exec,suid\nExecStartPre=/usr/bin/chcon -u system_u -r object_r -t container_runtime_exec_t /usr/local/bin/crio /usr/local/bin/crio-status /usr/local/bin/runc /usr/local/bin/crun\nExecStartPre=/usr/bin/chcon -u system_u -r object_r -t bin_t /usr/local/bin/conmon /usr/local/bin/crictl /usr/local/bin/pinns\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t bin_t /opt/cni/bin/\nExecStartPre=/usr/bin/rm -f /etc/cni/net.d/87-podman-bridge.conflist\nExecStartPre=/usr/bin/bash -c 'echo -e \"[crio.runtime]\\n default_runtime = \\\\\\\"runc\\\\\\\"\\n[crio.runtime.runtimes]\\n [crio.runtime.runtimes.runc]\\n runtime_path=\\\\\\\"/usr/local/bin/runc\\\\\\\"\" \u003e /etc/crio/crio.conf.d/20-runc.conf'\nExecStartPre=/usr/bin/bash -c 'echo -e \"[crio.runtime]\\n [crio.runtime.runtimes]\\n [crio.runtime.runtimes.test-handler]\\n runtime_path=\\\\\\\"/usr/local/bin/crun\\\\\\\"\" \u003e /etc/crio/crio.conf.d/10-crun.conf'\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t container_config_t /etc/crio /etc/crio/crio.conf /usr/local/share/oci-umount/oci-umount.d/crio-umount.conf\nExecStartPre=/usr/bin/systemctl enable crio.service\nExecStartPre=/usr/bin/chcon -R -u system_u -r object_r -t systemd_unit_file_t /usr/local/lib/systemd/system/crio.service\nExecStart=/usr/bin/systemctl start crio.service\n\n[Install]\nWantedBy=multi-user.target\n",
"enabled": true,
"name": "crio-install.service"
}
]
}
}
9 changes: 9 additions & 0 deletions jobs/e2e_node/swap/image-config-swap-fedora.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# To copy an image between projects:
# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
images:
fedora:
image_family: fedora-coreos-stable
project: fedora-coreos-cloud
machine: n1-standard-2
metadata: "user-data</workspace/test-infra/jobs/e2e_node/swap/crio_swap1g.ign"
9 changes: 9 additions & 0 deletions jobs/e2e_node/swap/image-config-swap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# To copy an image between projects:
# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
images:
ubuntu:
image: ubuntu-gke-2004-1-20-v20210401 # docker 19.03.8 / containerd 1.4.3
project: ubuntu-os-gke-cloud
metadata: "user-data</workspace/test-infra/jobs/e2e_node/swap/ubuntu-swap-1G-allocation.yaml"
machine: n1-standard-2
16 changes: 16 additions & 0 deletions jobs/e2e_node/swap/ubuntu-swap-1G-allocation.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#cloud-config

runcmd:
- 'echo current swappiness: $(sysctl vm.swappiness)'
- 'echo create swap file'
- 'fallocate -l 1G /swapfile || true'
- 'chmod 600 /swapfile'
- 'mkswap /swapfile'
- 'swapon /swapfile'
- 'echo "/swapfile swap swap defaults 0 0" | tee -a /etc/fstab'
- 'echo swap files'
- 'swapon --show'
- 'free -h'
- 'sudo sed -i "s:console=ttyS0:console=ttyS0 cgroup_enable=memory swapaccount=1:g" /etc/default/grub.d/50-cloudimg-settings.cfg'
- 'sudo update-grub'
- 'sudo reboot'

0 comments on commit 4fad20b

Please sign in to comment.