Skip to content

Commit

Permalink
E2E Dualstack CNI Support (#2730)
Browse files Browse the repository at this point in the history 
* Added switch case for CNIs
* Curl  fixes, apply manifests correctly

Signed-off-by: Derek Nola <[email protected]>
  • Loading branch information
@dereknola
dereknola authored Apr 6, 2022
1 parent 53f219e commit abd8fd3
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 40 deletions.
22 changes: 9 additions & 13 deletions tests/e2e/dualstack/Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
CNI = (ENV['E2E_CNI'] || "canal") # canal, cilium and calico supported
NETWORK4_PREFIX = "10.10.10"
NETWORK6_PREFIX = "a11:decf:c0ff:ee"
install_type = ""
Expand All @@ -25,23 +26,22 @@ def provision(vm, roles, role_num, node_num)
:libvirt__guest_ipv6 => "yes",
:libvirt__ipv6_address => "#{NETWORK6_PREFIX}::1",
:libvirt__ipv6_prefix => "64"

vm.provision "shell", path: "../scripts/ipv6.sh", args: [node_ip6]



vagrant_defaults = '../vagrantdefaults.rb'
load vagrant_defaults if File.exists?(vagrant_defaults)

defaultOSConfigure(vm)

vm.provision "IPv6 Setup", type: "shell", path: "../scripts/ipv6.sh", args: [node_ip4, node_ip6, CNI, vm.box]

if !RELEASE_VERSION.empty?
install_type = "INSTALL_RKE2_VERSION=#{RELEASE_VERSION}"
else
# Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/rke2_commits"]
vm.provision "Find Latest Commit", type: "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/rke2_commits"]
install_type = "INSTALL_RKE2_COMMIT=$(head\ -n\ 1\ /tmp/rke2_commits)"
end
vm.provision "shell", inline: "ping -c 2 rke2.io"
vm.provision "Ping Check", type: "shell", inline: "ping -c 2 rke2.io"

if roles.include?("server") && role_num == 0
vm.provision :rke2, run: 'once' do |rke2|
Expand All @@ -55,7 +55,7 @@ def provision(vm, roles, role_num, node_num)
cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112
bind-address: #{NETWORK4_PREFIX}.100
cni: calico
cni: #{CNI}
YAML
end
elsif roles.include?("server") && role_num != 0
Expand All @@ -70,7 +70,7 @@ def provision(vm, roles, role_num, node_num)
token: vagrant-rke2
cluster-cidr: 10.42.0.0/16,2001:cafe:42:0::/56
service-cidr: 10.43.0.0/16,2001:cafe:42:1::/112
cni: calico
cni: #{CNI}
YAML
end
end
Expand All @@ -82,6 +82,7 @@ def provision(vm, roles, role_num, node_num)
rke2.config = <<~YAML
write-kubeconfig-mode: '0644'
node-external-ip: #{node_ip4},#{node_ip6}
node-ip: #{node_ip4},#{node_ip6}
server: https://#{NETWORK4_PREFIX}.100:9345
token: vagrant-rke2
YAML
Expand All @@ -91,15 +92,10 @@ end

Vagrant.configure("2") do |config|
config.vagrant.plugins = ["vagrant-rke2", "vagrant-reload", "vagrant-libvirt"]
# Default provider is libvirt, virtualbox is only provided as a backup
config.vm.provider "libvirt" do |v|
v.cpus = NODE_CPUS
v.memory = NODE_MEMORY
end
config.vm.provider "virtualbox" do |v|
v.cpus = NODE_CPUS
v.memory = NODE_MEMORY
end

if NODE_ROLES.kind_of?(String)
NODE_ROLES = NODE_ROLES.split(" ", -1)
Expand Down
17 changes: 10 additions & 7 deletions tests/e2e/dualstack/dualstack_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@ func getObjIPs(cmd string) ([]objIP, error) {
fields := strings.Fields(obj)
if len(fields) > 2 {
objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1], ipv6: fields[2]})
} else if len(fields) > 1 {
objIPs = append(objIPs, objIP{name: fields[0], ipv4: fields[1]})
} else {
return nil, fmt.Errorf("%s does not have IPv4 and Ipv6 assigned", obj)
objIPs = append(objIPs, objIP{name: fields[0]})
}
}
return objIPs, nil
Expand Down Expand Up @@ -134,7 +136,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("ds-clusterip-pod"))
}, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod"))

// Checks both IPv4 and IPv6
clusterips, err := e2e.FetchClusterIP(kubeConfigFile, "ds-clusterip-svc", true)
Expand All @@ -149,9 +151,10 @@ var _ = Describe("Verify DualStack Configuration", func() {
if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") {
continue
}
cmd := fmt.Sprintf("kubectl exec %s --kubeconfig=%s -- /bin/bash -c ' curl -L --insecure http://%s'",
pod.Name, kubeConfigFile, ip)
Expect(e2e.RunCommand(cmd)).Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
cmd := fmt.Sprintf("curl -L --insecure http://%s", ip)
Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
}, "60s", "5s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
}
}
})
Expand Down Expand Up @@ -187,11 +190,11 @@ var _ = Describe("Verify DualStack Configuration", func() {
cmd = "curl -L --insecure http://" + node.ipv4 + ":" + nodeport + "/name.html"
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "5s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
}, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
cmd = "curl -L --insecure http://[" + node.ipv6 + "]:" + nodeport + "/name.html"
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "5s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
}, "10s", "1s").Should(ContainSubstring("ds-nodeport-pod"), "failed cmd: "+cmd)
}
})

Expand Down
67 changes: 50 additions & 17 deletions tests/e2e/scripts/ipv6.sh
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,57 @@
#!/bin/bash
ip6_addr=$1

netplan set ethernets.eth1.accept-ra=false
netplan apply
ip4_addr=$1
ip6_addr=$2
cni=$3
os=$4

sysctl -w net.ipv6.conf.all.disable_ipv6=0
sysctl -w net.ipv6.conf.eth1.accept_dad=0
ip -6 addr add "$ip6_addr"/64 dev eth1

if [ -z "${os##*ubuntu*}" ]; then
netplan set ethernets.eth1.accept-ra=false
netplan set ethernets.eth1.addresses=["$ip4_addr"/24,"$ip6_addr"/64]
netplan apply
else
ip -6 addr add "$ip6_addr"/64 dev eth1
fi
ip addr show dev eth1
# Override default canal and specify the interface since we don't have a default IPv6 route
# Override default CNI and specify the interface since we don't have a default IPv6 route
mkdir -p /var/lib/rancher/rke2/server/manifests
# echo "apiVersion: helm.cattle.io/v1
# kind: HelmChartConfig
# metadata:
# name: rke2-canal
# namespace: kube-system
# spec:
# valuesContent: |-
# flannel:
# iface: \"eth1\"" >> /var/lib/rancher/rke2/server/manifests/e2e-canal.yaml

echo "apiVersion: helm.cattle.io/v1
case "$cni" in
"canal")
echo "Creating canal chart"
echo "apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: rke2-canal
namespace: kube-system
spec:
valuesContent: |-
flannel:
iface: \"eth1\"
calico:
ipAutoDetectionMethod: \"interface=eth1.*\"
ip6AutoDetectionMethod: \"interface=eth1.*\"" >> /var/lib/rancher/rke2/server/manifests/e2e-canal.yaml
;;

"cilium")
echo "Creating cilium chart"
echo "apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: rke2-cilium
namespace: kube-system
spec:
valuesContent: |-
devices: eth1
ipv6:
enabled: true">> /var/lib/rancher/rke2/server/manifests/e2e-cilium.yaml
;;

"calico")
echo "Creating calico chart"
echo "apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: rke2-calico
Expand All @@ -32,4 +63,6 @@ spec:
nodeAddressAutodetectionV4:
interface: eth1.*
nodeAddressAutodetectionV6:
interface: eth1.* " >> /var/lib/rancher/rke2/server/manifests/e2e-calico.yaml
interface: eth1.* " >> /var/lib/rancher/rke2/server/manifests/e2e-calico.yaml
;;
esac
6 changes: 3 additions & 3 deletions tests/e2e/vagrantdefaults.rb
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
def defaultOSConfigure(vm)

if vm.box.include?("ubuntu2004")
vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0", run: 'once'
vm.provision "shell", inline: "apt install -y jq", run: 'once'
vm.provision "netplan dns", type: "shell", inline: "netplan set ethernets.eth0.nameservers.addresses=[8.8.8.8,1.1.1.1]; netplan apply", run: 'once'
vm.provision "Install jq", type: "shell", inline: "apt install -y jq", run: 'once'
end
if vm.box.include?("Leap")
vm.provision "shell", inline: "zypper install -y jq", run: 'once'
vm.provision "Install jq", type: "shell", inline: "zypper install -y jq", run: 'once'
end

end

0 comments on commit abd8fd3

Please sign in to comment.