Skip to content

Commit

Permalink
Refactor node command execution functions
Browse files Browse the repository at this point in the history 
Updated these functions to use the node inspector rather than the MCD.
Furthermore, I have renamed these functions to better reflect their purposes and usage.

Signed-off-by: Ronny Baturov <[email protected]>
  • Loading branch information
@rbaturov
rbaturov committed May 23, 2024
1 parent 67fc3e9 commit 90cba92
Show file tree
Hide file tree
Showing 19 changed files with 166 additions and 150 deletions.
6 changes: 3 additions & 3 deletions test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {
// test arbitrary one should be good enough
worker := &workers[0]
cmd := isFileExistCmd(kubeletMixedCPUsConfigFile)
found, err := nodes.ExecCommandOnNode(ctx, cmd, worker)
found, err := nodes.ExecCommandToString(ctx, cmd, worker)
Expect(err).ToNot(HaveOccurred(), "failed to execute command on node; cmd=%q node=%q", cmd, worker)
Expect(found).To(Equal("true"), "file not found; file=%q", kubeletMixedCPUsConfigFile)
})
Expand Down Expand Up @@ -112,7 +112,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {
"-c",
fmt.Sprintf("/bin/awk -F '\"' '/shared_cpuset.*/ { print $2 }' %s", runtime.CRIORuntimeConfigFile),
}
cpus, err := nodes.ExecCommandOnNode(ctx, cmd, worker)
cpus, err := nodes.ExecCommandToString(ctx, cmd, worker)
Expect(err).ToNot(HaveOccurred(), "failed to execute command on node; cmd=%q node=%q", cmd, worker)
cpus = strings.Trim(cpus, "\n")
crioShared := mustParse(cpus)
Expand Down Expand Up @@ -268,7 +268,7 @@ var _ = Describe("Mixedcpus", Ordered, func() {

cmd := kubeletRestartCmd()
// The command would fail since it aborts all the pods during restart
_, _ = nodes.ExecCommandOnNode(ctx, cmd, node)
_, _ = nodes.ExecCommandToString(ctx, cmd, node)
// check that the node is ready after we restart Kubelet
nodes.WaitForReadyOrFail("post restart", node.Name, 20*time.Minute, 3*time.Second)

Expand Down
30 changes: 15 additions & 15 deletions test/e2e/performanceprofile/functests/1_performance/cpu_management.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
It("[test_id:37862][crit:high][vendor:[email protected]][level:acceptance] Verify CPU affinity mask, CPU reservation and CPU isolation on worker node", func() {
By("checking isolated CPU")
cmd := []string{"cat", "/sys/devices/system/cpu/isolated"}
sysIsolatedCpus, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
sysIsolatedCpus, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
if balanceIsolated {
Expect(sysIsolatedCpus).To(BeEmpty())
Expand All @@ -130,14 +130,14 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("checking reserved CPU in kubelet config file")
cmd = []string{"cat", "/rootfs/etc/kubernetes/kubelet.conf"}
conf, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
conf, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "failed to cat kubelet.conf")
// kubelet.conf changed formatting, there is a space after colons atm. Let's deal with both cases with a regex
Expect(conf).To(MatchRegexp(fmt.Sprintf(`"reservedSystemCPUs": ?"%s"`, reservedCPU)))

By("checking CPU affinity mask for kernel scheduler")
cmd = []string{"/bin/bash", "-c", "taskset -pc 1"}
sched, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
sched, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "failed to execute taskset")
mask := strings.SplitAfter(sched, " ")
maskSet, err := cpuset.Parse(mask[len(mask)-1])
Expand All @@ -149,7 +149,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
It("[test_id:34358] Verify rcu_nocbs kernel argument on the node", func() {
By("checking that cmdline contains rcu_nocbs with right value")
cmd := []string{"cat", "/proc/cmdline"}
cmdline, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
cmdline, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
re := regexp.MustCompile(`rcu_nocbs=\S+`)
rcuNocbsArgument := re.FindString(cmdline)
Expand All @@ -159,12 +159,12 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("checking that new rcuo processes are running on non_isolated cpu")
cmd = []string{"pgrep", "rcuo"}
rcuoList, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
rcuoList, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
for _, rcuo := range strings.Split(rcuoList, "\n") {
// check cpu affinity mask
cmd = []string{"/bin/bash", "-c", fmt.Sprintf("taskset -pc %s", rcuo)}
taskset, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
taskset, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
mask := strings.SplitAfter(taskset, " ")
maskSet, err := cpuset.Parse(mask[len(mask)-1])
Expand Down Expand Up @@ -246,7 +246,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
"unexpected QoS Class for %s/%s: %s (looking for %s)",
updatedPod.Namespace, updatedPod.Name, updatedPod.Status.QOSClass, expectedQos)

output, err := nodes.ExecCommandOnNode(ctx,
output, err := nodes.ExecCommandToString(ctx,
[]string{"/bin/bash", "-c", "ps -o psr $(pgrep -n stress) | tail -1"},
workerRTNode,
)
Expand Down Expand Up @@ -370,7 +370,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
// It may takes some time for the system to reschedule active IRQs
Eventually(func() bool {
getActiveIrq := []string{"/bin/bash", "-c", "for n in $(find /proc/irq/ -name smp_affinity_list); do echo $(cat $n); done"}
activeIrq, err := nodes.ExecCommandOnNode(context.TODO(), getActiveIrq, workerRTNode)
activeIrq, err := nodes.ExecCommandToString(context.TODO(), getActiveIrq, workerRTNode)
Expect(err).ToNot(HaveOccurred())
Expect(activeIrq).ToNot(BeEmpty())
for _, irq := range strings.Split(activeIrq, "\n") {
Expand Down Expand Up @@ -424,7 +424,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

Eventually(func() string {
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("find %s -name *%s*", cpusetPath, podUID)}
podCgroup, err = nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
podCgroup, err = nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
return podCgroup
}, cluster.ComputeTestTimeout(30*time.Second, RunningOnSingleNode), 5*time.Second).ShouldNot(BeEmpty(),
Expand All @@ -433,7 +433,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {
containersCgroups := ""
Eventually(func() string {
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("find %s -name crio-*", podCgroup)}
containersCgroups, err = nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
containersCgroups, err = nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())
return containersCgroups
}, cluster.ComputeTestTimeout(30*time.Second, RunningOnSingleNode), 5*time.Second).ShouldNot(BeEmpty(),
Expand All @@ -454,7 +454,7 @@ var _ = Describe("[rfe_id:27363][performance] CPU Management", Ordered, func() {

By("Checking what CPU the infra container is using")
cmd := []string{"/bin/bash", "-c", fmt.Sprintf("cat %s/cpuset.cpus", dir)}
output, err := nodes.ExecCommandOnNode(context.TODO(), cmd, workerRTNode)
output, err := nodes.ExecCommandToString(context.TODO(), cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred())

cpus, err := cpuset.Parse(output)
Expand Down Expand Up @@ -749,7 +749,7 @@ func checkForWorkloadPartitioning(ctx context.Context) bool {
"-c",
"echo CHECK ; /bin/grep -rEo 'activation_annotation.*target\\.workload\\.openshift\\.io/management.*' /etc/crio/crio.conf.d/ || true",
}
output, err := nodes.ExecCommandOnNode(ctx, cmd, workerRTNode)
output, err := nodes.ExecCommandToString(ctx, cmd, workerRTNode)
Expect(err).ToNot(HaveOccurred(), "Unable to check cluster for Workload Partitioning enabled")
re := regexp.MustCompile(`activation_annotation.*target\.workload\.openshift\.io/management.*`)
return re.MatchString(fmt.Sprint(output))
Expand All @@ -770,7 +770,7 @@ func checkPodHTSiblings(ctx context.Context, testpod *corev1.Pod) bool {
node, err := nodes.GetByName(testpod.Spec.NodeName)
Expect(err).ToNot(HaveOccurred(), "failed to get node %q", testpod.Spec.NodeName)
Expect(testpod.Spec.NodeName).ToNot(BeEmpty(), "testpod %s/%s still pending - no nodeName set", testpod.Namespace, testpod.Name)
output, err := nodes.ExecCommandOnNode(ctx, cmd, node)
output, err := nodes.ExecCommandToString(ctx, cmd, node)
Expect(err).ToNot(HaveOccurred(), "Unable to crictl inspect containerID %q", containerID)

podcpus, err := cpuset.Parse(strings.Trim(output, "\n"))
Expand All @@ -794,7 +794,7 @@ func checkPodHTSiblings(ctx context.Context, testpod *corev1.Pod) bool {
"-c",
fmt.Sprintf("/bin/cat %s | /bin/sort -u", hostHTSiblingPaths.String()),
}
output, err = nodes.ExecCommandOnNode(ctx, cmd, workerRTNode)
output, err = nodes.ExecCommandToString(ctx, cmd, workerRTNode)
Expect(err).ToNot(
HaveOccurred(),
"Unable to read host thread_siblings_list files",
Expand Down Expand Up @@ -975,7 +975,7 @@ func logEventsForPod(testPod *corev1.Pod) {
// getCPUswithLoadBalanceDisabled Return cpus which are not in any scheduling domain
func getCPUswithLoadBalanceDisabled(ctx context.Context, targetNode *corev1.Node) ([]string, error) {
cmd := []string{"/bin/bash", "-c", "cat /proc/schedstat"}
schedstatData, err := nodes.ExecCommandOnNode(ctx, cmd, targetNode)
schedstatData, err := nodes.ExecCommandToString(ctx, cmd, targetNode)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/performanceprofile/functests/1_performance/hugepages.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ var _ = Describe("[performance]Hugepages", Ordered, func() {

func checkHugepagesStatus(ctx context.Context, path string, workerRTNode *corev1.Node) int {
command := []string{"cat", path}
out, err := nodes.ExecCommandOnMachineConfigDaemon(ctx, workerRTNode, command)
out, err := nodes.ExecCommand(ctx, workerRTNode, command)
Expect(err).ToNot(HaveOccurred())
n, err := strconv.Atoi(strings.Trim(string(out), "\n\r"))
Expect(err).ToNot(HaveOccurred())
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/performanceprofile/functests/1_performance/irqbalance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ var _ = Describe("[performance] Checking IRQBalance settings", Ordered, func() {
origBannedCPUsFile := "/etc/sysconfig/orig_irq_banned_cpus"
By(fmt.Sprintf("Checking content of %q on node %q", origBannedCPUsFile, node.Name))
fullPath := filepath.Join("/", "rootfs", origBannedCPUsFile)
out, err := nodes.ExecCommandOnNode(context.TODO(), []string{"/usr/bin/cat", fullPath}, node)
out, err := nodes.ExecCommandToString(context.TODO(), []string{"/usr/bin/cat", fullPath}, node)
Expect(err).ToNot(HaveOccurred())
out = strings.TrimSuffix(out, "\r\n")
Expect(out).To(Equal("0"), "file %s does not contain the expect output; expected=0 actual=%s", fullPath, out)
Expand All @@ -327,7 +327,7 @@ var _ = Describe("[performance] Checking IRQBalance settings", Ordered, func() {
// require close attention. For the time being we reimplement a form of nodes.BannedCPUs which can handle empty ban list.
func getIrqBalanceBannedCPUs(ctx context.Context, node *corev1.Node) (cpuset.CPUSet, error) {
cmd := []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}
conf, err := nodes.ExecCommandOnNode(ctx, cmd, node)
conf, err := nodes.ExecCommandToString(ctx, cmd, node)
if err != nil {
return cpuset.New(), err
}
Expand Down Expand Up @@ -361,7 +361,7 @@ func getIrqBalanceBannedCPUs(ctx context.Context, node *corev1.Node) (cpuset.CPU

func getIrqDefaultSMPAffinity(ctx context.Context, node *corev1.Node) (string, error) {
cmd := []string{"cat", "/rootfs/proc/irq/default_smp_affinity"}
return nodes.ExecCommandOnNode(ctx, cmd, node)
return nodes.ExecCommandToString(ctx, cmd, node)
}

func findIrqBalanceBannedCPUsVarFromConf(conf string) string {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/performanceprofile/functests/1_performance/netqueues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,15 +401,15 @@ func getReservedCPUSize(CPU *performancev2.CPU) int {
func getVendorID(ctx context.Context, node corev1.Node, device string) string {
cmd := []string{"bash", "-c",
fmt.Sprintf("cat /sys/class/net/%s/device/vendor", device)}
stdout, err := nodes.ExecCommandOnNode(ctx, cmd, &node)
stdout, err := nodes.ExecCommandToString(ctx, cmd, &node)
Expect(err).ToNot(HaveOccurred())
return stdout
}

func getDeviceID(ctx context.Context, node corev1.Node, device string) string {
cmd := []string{"bash", "-c",
fmt.Sprintf("cat /sys/class/net/%s/device/device", device)}
stdout, err := nodes.ExecCommandOnNode(ctx, cmd, &node)
stdout, err := nodes.ExecCommandToString(ctx, cmd, &node)
Expect(err).ToNot(HaveOccurred())
return stdout
}
Expand Down
Loading

0 comments on commit 90cba92

Please sign in to comment.