Skip to content

Commit

Permalink
Refactor node command execution functions
Browse files Browse the repository at this point in the history 
Updated these functions to use the node inspector rather than the MCD.
Furthermore, I have renamed these functions to better reflect their purposes and usage.

Signed-off-by: Ronny Baturov <[email protected]>
  • Loading branch information
@rbaturov
rbaturov committed Jun 3, 2024
1 parent 22d060d commit a7b20dc
Show file tree
Hide file tree
Showing 19 changed files with 178 additions and 158 deletions.
8 changes: 4 additions & 4 deletions test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {
// test arbitrary one should be good enough
worker := &workers[0]
cmd := isFileExistCmd(kubeletMixedCPUsConfigFile)
found, err := nodes.ExecCommandOnNode(ctx, cmd, worker)
found, err := nodes.ExecCommandToString(ctx, cmd, worker)
Expect(err).ToNot(HaveOccurred(), "failed to execute command on node; cmd=%q node=%q", cmd, worker)
Expect(found).To(Equal("true"), "file not found; file=%q", kubeletMixedCPUsConfigFile)
})
Expand Down Expand Up @@ -112,7 +112,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {
"-c",
fmt.Sprintf("/bin/awk -F '\"' '/shared_cpuset.*/ { print $2 }' %s", runtime.CRIORuntimeConfigFile),
}
cpus, err := nodes.ExecCommandOnNode(ctx, cmd, worker)
cpus, err := nodes.ExecCommandToString(ctx, cmd, worker)
Expect(err).ToNot(HaveOccurred(), "failed to execute command on node; cmd=%q node=%q", cmd, worker)
cpus = strings.Trim(cpus, "\n")
crioShared := mustParse(cpus)
Expand Down Expand Up @@ -268,7 +268,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {

cmd := kubeletRestartCmd()
// The command would fail since it aborts all the pods during restart
_, _ = nodes.ExecCommandOnNode(ctx, cmd, node)
_, _ = nodes.ExecCommandToString(ctx, cmd, node)
// check that the node is ready after we restart Kubelet
nodes.WaitForReadyOrFail("post restart", node.Name, 20*time.Minute, 3*time.Second)

Expand Down Expand Up @@ -616,7 +616,7 @@ func fetchSharedCPUsFromEnv(c *kubernetes.Clientset, p *corev1.Pod, containerNam
// getCPUswithLoadBalanceDisabled Return cpus which are not in any scheduling domain
func getCPUswithLoadBalanceDisabled(ctx context.Context, targetNode *corev1.Node) ([]string, error) {
cmd := []string{"/bin/bash", "-c", "cat /proc/schedstat"}
schedstatData, err := nodes.ExecCommandOnNode(ctx, cmd, targetNode)
schedstatData, err := nodes.ExecCommandToString(ctx, cmd, targetNode)
if err != nil {
return nil, err
}
Expand Down
34 changes: 17 additions & 17 deletions test/e2e/performanceprofile/functests/1_performance/cpu_management.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
It("[test_id:37862][crit:high][vendor:[email protected]][level:acceptance] Verify CPU affinity mask, CPU reservation and CPU isolation on worker node", func() {
By("checking isolated CPU")
cmd := []string{"cat", "/sys/devices/system/cpu/isolated"}
sysIsolatedCpus, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
sysIsolatedCpus, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
if balanceIsolated {
Expect(sysIsolatedCpus).To(BeEmpty())
Expand All @@ -130,14 +130,14 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("checking reserved CPU in kubelet config file")
cmd = []string{"cat", "/rootfs/etc/kubernetes/kubelet.conf"}
conf, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
conf, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "failed to cat kubelet.conf")
// kubelet.conf changed formatting, there is a space after colons atm. Let's deal with both cases with a regex
Expect(conf).To(MatchRegexp(fmt.Sprintf(`"reservedSystemCPUs": ?"%s"`, reservedCPU)))

By("checking CPU affinity mask for kernel scheduler")
cmd = []string{"/bin/bash", "-c", "taskset -pc 1"}
sched, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
sched, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "failed to execute taskset")
mask := strings.SplitAfter(sched, " ")
maskSet, err := cpuset.Parse(mask[len(mask)-1])
Expand All @@ -149,7 +149,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
It("[test_id:34358] Verify rcu_nocbs kernel argument on the node", func() {
By("checking that cmdline contains rcu_nocbs with right value")
cmd := []string{"cat", "/proc/cmdline"}
cmdline, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
cmdline, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
re := regexp.MustCompile(`rcu_nocbs=\S+`)
rcuNocbsArgument := re.FindString(cmdline)
Expand All @@ -159,12 +159,12 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("checking that new rcuo processes are running on non_isolated cpu")
cmd = []string{"pgrep", "rcuo"}
rcuoList, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
rcuoList, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
for _, rcuo := range strings.Split(rcuoList, "\n") {
// check cpu affinity mask
cmd = []string{"/bin/bash", "-c", fmt.Sprintf("taskset -pc %s", rcuo)}
taskset, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
taskset, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
mask := strings.SplitAfter(taskset, " ")
maskSet, err := cpuset.Parse(mask[len(mask)-1])
Expand Down Expand Up @@ -246,7 +246,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
"unexpected QoS Class for %s/%s: %s (looking for %s)",
updatedPod.Namespace, updatedPod.Name, updatedPod.Status.QOSClass, expectedQos)

output, err := nodes.ExecCommandOnNode(ctx,
output, err := nodes.ExecCommandToString(ctx,
[]string{"/bin/bash", "-c", "ps -o psr $(pgrep -n stress) | tail -1"},
workerRTNode,
)
Expand Down Expand Up @@ -295,7 +295,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
"systemctl restart kubelet",
}

_, _ = nodes.ExecCommandOnNode(ctx, kubeletRestartCmd, workerRTNode)
_, _ = nodes.ExecCommandToString(ctx, kubeletRestartCmd, workerRTNode)
nodes.WaitForReadyOrFail("post kubele restart", workerRTNode.Name, 20*time.Minute, 3*time.Second)
// giving kubelet more time to stabilize and initialize itself before
testlog.Infof("post restart: entering cooldown time: %v", restartCooldownTime)
Expand Down Expand Up @@ -370,7 +370,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
// It may takes some time for the system to reschedule active IRQs
Eventually(func() bool {
getActiveIrq := []string{"/bin/bash", "-c", "for n in $(find /proc/irq/ -name smp_affinity_list); do echo $(cat $n); done"}
activeIrq, err := nodes.ExecCommandOnNode(context.TODO(), getActiveIrq, workerRTNode)
activeIrq, err := nodes.ExecCommandToString(context.TODO(), getActiveIrq, workerRTNode)
Expect(err).ToNot(HaveOccurred())
Expect(activeIrq).ToNot(BeEmpty())
for _, irq := range strings.Split(activeIrq, "\n") {
Expand Down Expand Up @@ -424,7 +424,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

Eventually(func() string {
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("find %s -name *%s*", cpusetPath, podUID)}
podCgroup, err = nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
podCgroup, err = nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
return podCgroup
}, cluster.ComputeTestTimeout(30*time.Second, RunningOnSingleNode), 5*time.Second).ShouldNot(BeEmpty(),
Expand All @@ -433,7 +433,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
containersCgroups := ""
Eventually(func() string {
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("find %s -name crio-*", podCgroup)}
containersCgroups, err = nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
containersCgroups, err = nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
return containersCgroups
}, cluster.ComputeTestTimeout(30*time.Second, RunningOnSingleNode), 5*time.Second).ShouldNot(BeEmpty(),
Expand All @@ -454,7 +454,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("Checking what CPU the infra container is using")
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("cat %s/cpuset.cpus", dir)}
output, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
output, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())

cpus, err := cpuset.Parse(output)
Expand Down Expand Up @@ -710,7 +710,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
Expect(err).ToNot(HaveOccurred(), "Unable to parse pod cpus")
kubepodsExclusiveCpus := fmt.Sprintf("%s/kubepods.slice/cpuset.cpus.exclusive", cgroupRoot)
cmd := []string{"cat", kubepodsExclusiveCpus}
exclusiveCpus, err := nodes.ExecCommandOnNode(ctx, cmd, targetNode)
exclusiveCpus, err := nodes.ExecCommandToString(ctx, cmd, targetNode)
Expect(err).ToNot(HaveOccurred())
exclusiveCpuset, err := cpuset.Parse(exclusiveCpus)
Expect(err).ToNot(HaveOccurred(), "unable to parse cpuset.cpus.exclusive")
Expand Down Expand Up @@ -752,7 +752,7 @@ func checkForWorkloadPartitioning(ctx context.Context) bool {
"-c",
"echo CHECK ; /bin/grep -rEo 'activation_annotation.*target\\.workload\\.openshift\\.io/management.*' /etc/crio/crio.conf.d/ || true",
}
output, err := nodes.ExecCommandOnNode(ctx, cmd, workerRTNode)
output, err := nodes.ExecCommandToString(ctx, cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "Unable to check cluster for Workload Partitioning enabled")
re := regexp.MustCompile(`activation_annotation.*target\.workload\.openshift\.io/management.*`)
return re.MatchString(fmt.Sprint(output))
Expand All @@ -773,7 +773,7 @@ func checkPodHTSiblings(ctx context.Context, testpod *corev1.Pod) bool {
node, err := nodes.GetByName(testpod.Spec.NodeName)
Expect(err).ToNot(HaveOccurred(), "failed to get node %q", testpod.Spec.NodeName)
Expect(testpod.Spec.NodeName).ToNot(BeEmpty(), "testpod %s/%s still pending - no nodeName set", testpod.Namespace, testpod.Name)
output, err := nodes.ExecCommandOnNode(ctx, cmd, node)
output, err := nodes.ExecCommandToString(ctx, cmd, node)
Expect(err).ToNot(HaveOccurred(), "Unable to crictl inspect containerID %q", containerID)

podcpus, err := cpuset.Parse(strings.Trim(output, "\n"))
Expand All @@ -797,7 +797,7 @@ func checkPodHTSiblings(ctx context.Context, testpod *corev1.Pod) bool {
"-c",
fmt.Sprintf("/bin/cat %s | /bin/sort -u", hostHTSiblingPaths.String()),
}
output, err = nodes.ExecCommandOnNode(ctx, cmd, workerRTNode)
output, err = nodes.ExecCommandToString(ctx, cmd, workerRTNode)
Expect(err).ToNot(
HaveOccurred(),
"Unable to read host thread_siblings_list files",
Expand Down Expand Up @@ -978,7 +978,7 @@ func logEventsForPod(testPod *corev1.Pod) {
// getCPUswithLoadBalanceDisabled Return cpus which are not in any scheduling domain
func getCPUswithLoadBalanceDisabled(ctx context.Context, targetNode *corev1.Node) ([]string, error) {
cmd := []string{"/bin/bash", "-c", "cat /proc/schedstat"}
schedstatData, err := nodes.ExecCommandOnNode(ctx, cmd, targetNode)
schedstatData, err := nodes.ExecCommandToString(ctx, cmd, targetNode)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/performanceprofile/functests/1_performance/hugepages.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ var _ = Describe("[performance]Hugepages", Ordered, func() {

func checkHugepagesStatus(ctx context.Context, path string, workerRTNode *corev1.Node) int {
command := []string{"cat", path}
out, err := nodes.ExecCommandOnMachineConfigDaemon(ctx, workerRTNode, command)
out, err := nodes.ExecCommand(ctx, workerRTNode, command)
Expect(err).ToNot(HaveOccurred())
n, err := strconv.Atoi(strings.Trim(string(out), "\n\r"))
Expect(err).ToNot(HaveOccurred())
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/performanceprofile/functests/1_performance/irqbalance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ var _ = Describe("[performance] Checking IRQBalance settings", Ordered, func() {
origBannedCPUsFile := "/etc/sysconfig/orig_irq_banned_cpus"
By(fmt.Sprintf("Checking content of %q on node %q", origBannedCPUsFile, node.Name))
fullPath := filepath.Join("/", "rootfs", origBannedCPUsFile)
out, err := nodes.ExecCommandOnNode(context.TODO(), []string{"/usr/bin/cat", fullPath}, node)
out, err := nodes.ExecCommandToString(context.TODO(), []string{"/usr/bin/cat", fullPath}, node)
Expect(err).ToNot(HaveOccurred())
out = strings.TrimSuffix(out, "\r\n")
Expect(out).To(Equal("0"), "file %s does not contain the expect output; expected=0 actual=%s", fullPath, out)
Expand All @@ -327,7 +327,7 @@ var _ = Describe("[performance] Checking IRQBalance settings", Ordered, func() {
// require close attention. For the time being we reimplement a form of nodes.BannedCPUs which can handle empty ban list.
func getIrqBalanceBannedCPUs(ctx context.Context, node *corev1.Node) (cpuset.CPUSet, error) {
cmd := []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}
conf, err := nodes.ExecCommandOnNode(ctx, cmd, node)
conf, err := nodes.ExecCommandToString(ctx, cmd, node)
if err != nil {
return cpuset.New(), err
}
Expand Down Expand Up @@ -361,7 +361,7 @@ func getIrqBalanceBannedCPUs(ctx context.Context, node *corev1.Node) (cpuset.CPU

func getIrqDefaultSMPAffinity(ctx context.Context, node *corev1.Node) (string, error) {
cmd := []string{"cat", "/rootfs/proc/irq/default_smp_affinity"}
return nodes.ExecCommandOnNode(ctx, cmd, node)
return nodes.ExecCommandToString(ctx, cmd, node)
}

func findIrqBalanceBannedCPUsVarFromConf(conf string) string {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/performanceprofile/functests/1_performance/netqueues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,15 +401,15 @@ func getReservedCPUSize(CPU *performancev2.CPU) int {
func getVendorID(ctx context.Context, node corev1.Node, device string) string {
cmd := []string{"bash", "-c",
fmt.Sprintf("cat /sys/class/net/%s/device/vendor", device)}
stdout, err := nodes.ExecCommandOnNode(ctx, cmd, &node)
stdout, err := nodes.ExecCommandToString(ctx, cmd, &node)
Expect(err).ToNot(HaveOccurred())
return stdout
}

func getDeviceID(ctx context.Context, node corev1.Node, device string) string {
cmd := []string{"bash", "-c",
fmt.Sprintf("cat /sys/class/net/%s/device/device", device)}
stdout, err := nodes.ExecCommandOnNode(ctx, cmd, &node)
stdout, err := nodes.ExecCommandToString(ctx, cmd, &node)
Expect(err).ToNot(HaveOccurred())
return stdout
}
Expand Down
Loading

0 comments on commit a7b20dc

Please sign in to comment.