Merge pull request #802 from nik-netlox/main

gh-87 : sctpmh CICD test suites updated

.github/workflows/sctpmh-seagull-sanity.yml

@@ -0,0 +1,36 @@
+name: SCTP-MH-LB-Seagull-Sanity-CI
+on:
+ # schedule:
+ # Runs "At 11:00 UTC every day-of-week"
+ #- cron: '0 11 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'sctpmh-seagull-sanity'
+jobs:
+  test-runner:
+    name: k8s-calico-incluster-sanity
+    runs-on: [self-hosted, large]
+    if: github.repository == 'loxilb-io/loxilb' 
+          &&  github.event.inputs.tagName == ''
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+          submodules: recursive
+
+    - name: Run the test
+      run: |
+         cd cicd/sctpmh-seagull
+         ./config.sh
+         ./validation.sh
+         cd -
+         
+    - name: Clean test-bed
+      if: success() || failure()
+      run: |
+        cd cicd/sctpmh-seagull || true
+        ./rmconfig.sh
+        cd -

cicd/common.sh

@@ -561,7 +561,7 @@ function create_lb_rule() {
     return
   fi
 
-  hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func)
+  hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func)
   if [[ $hook != *"tc_packet_func"* ]]; then
     echo "ERROR : No hook point found";
     exit 1

cicd/docker-k3s-calico/common.sh

@@ -536,7 +536,7 @@ function create_lb_rule() {
   echo "$1: loxicmd create lb ${args[*]}"
   $dexec $1 loxicmd create lb ${args[*]}
 
-  hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func)
+  hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func)
   if [[ $hook != *"tc_packet_func"* ]]; then
     echo "ERROR : No hook point found";
     exit 1

cicd/docker-k3s-cilium/common.sh

@@ -536,7 +536,7 @@ function create_lb_rule() {
   echo "$1: loxicmd create lb ${args[*]}"
   $dexec $1 loxicmd create lb ${args[*]}
 
-  hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func)
+  hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func)
   if [[ $hook != *"tc_packet_func"* ]]; then
     echo "ERROR : No hook point found";
     exit 1

cicd/k0s-incluster/common.sh

@@ -536,7 +536,7 @@ function create_lb_rule() {
   echo "$1: loxicmd create lb ${args[*]}"
   $dexec $1 loxicmd create lb ${args[*]}
 
-  hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func)
+  hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func)
   if [[ $hook != *"tc_packet_func"* ]]; then
     echo "ERROR : No hook point found";
     exit 1

cicd/microk8s-incluster/common.sh

@@ -536,8 +536,8 @@ function create_lb_rule() {
   echo "$1: loxicmd create lb ${args[*]}"
   $dexec $1 loxicmd create lb ${args[*]}
 
-  hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_hook)
-  if [[ $hook != *"tc_packet_hook"* ]]; then
+  hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func)
+  if [[ $hook != *"tc_packet_func"* ]]; then
     echo "ERROR : No hook point found";
     exit 1
   fi

cicd/sctpmh-seagull/Vagrantfile

@@ -0,0 +1,27 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64")
+box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
+box_version = "0.7.1"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "bastion" do |bastion|
+    bastion.vm.hostname = 'bastion'
+    #bastion.vm.network :private_network, ip: "4.0.5.3", :netmask => "255.255.255.0"
+    bastion.vm.network :private_network, ip: "4.0.4.3", :netmask => "255.255.255.0"
+    bastion.vm.provision :shell, :path => "bastion.sh"
+    bastion.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 8196]
+        vbox.customize ["modifyvm", :id, "--cpus", 12]
+        #vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        #vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+end

cicd/sctpmh-seagull/bastion.sh

@@ -0,0 +1,12 @@
+apt-get update
+apt-get install -y software-properties-common curl wget lksctp-tools jq
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+echo "blacklist sctp" >>  /etc/modprobe.d/blacklist.conf
+echo "install sctp /bin/false" >>  /etc/modprobe.d/blacklist.conf
+
+echo "Rebooting Now!"
+reboot
+"

cicd/sctpmh-seagull/check_ha.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+master="llb1"
+backup="llb2"
+
+function check_ha() {
+    while : ; do
+        status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
+        status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
+        count=0
+        if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]];
+        then
+            master="llb1"
+            backup="llb2"
+            break
+        elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; 
+        then
+            master="llb2"
+            backup="llb1"
+            break
+        else
+            count=$(( $count + 1 ))
+            if [[ $count -ge 20 ]]; then
+                echo "KA llb1-$status1, llb2-$status2 [NOK] - Exiting" >&2
+                exit 1;
+            fi
+            echo "KA llb1-$status1, llb2-$status2 [NOK]" >&2
+            sleep 5
+        fi
+    done
+}
+
+function checkSync() {
+    count=1
+    sync=0
+    while [[ $count -le 5 ]] ; do
+        echo -e "\nStatus at MASTER:$master\n" >&2
+        ct=`$dexec $master loxicmd get ct | grep est`
+        echo "${ct//'\n'/$'\n'}" >&2
+
+        echo -e "\nStatus at BACKUP:$backup\n" >&2
+        ct=`$dexec $backup loxicmd get ct | grep est`
+        echo "${ct//'\n'/$'\n'}" >&2
+
+        nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
+        nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
+
+        if [[ $nres1 == 0 ]]; then
+            echo -e "No active connections in Master:$master. Exiting!" >&2
+            return 2
+        fi
+
+        if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then
+            echo -e "\nConnections sync successful!!!\n" >&2
+            sync=1
+            break;
+        fi
+        echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2
+        count=$(( $count + 1 ))
+        sleep 2
+    done
+
+    if [[ $sync == 0 ]]; then
+        echo -e "\nConnection Sync failed\n" >&2
+        return 0
+    fi
+    echo "$sync"
+}
+
+function restart_mloxilb() {
+    if [[ $master == "llb1" ]]; then
+        pat="cluster=172.17.0.3"
+        copts=" --cluster=172.17.0.3"
+        self=" --self=0"
+        ka=" --ka=172.17.0.3:172.17.0.2"
+    else
+        pat="cluster=172.17.0.2"
+        copts=" --cluster=172.17.0.2"
+        self=" --self=1"
+        ka=" --ka=172.17.0.2:172.17.0.3"
+    fi
+    echo "Restarting MASTER: $master"
+    pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2)
+    echo "Killing $pid" >&2
+    docker exec -dt $master kill -9 $pid
+    docker exec -dt $master ip link del llb0
+    docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null &
+    pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2)
+    echo "New loxilb pid: $pid" >&2
+}
+
+function restart_loxilbs() {
+    if [[ $master == "llb1" ]]; then
+        mpat="cluster=172.17.0.3"
+        mcopts=" --cluster=172.17.0.3"
+        mself=" --self=0"
+        mka=" --ka=172.17.0.3:172.17.0.2"
+
+        bpat="cluster=172.17.0.2"
+        bcopts=" --cluster=172.17.0.2"
+        bself=" --self=1"
+        bka=" --ka=172.17.0.2:172.17.0.3"
+    else
+        mpat="cluster=172.17.0.2"
+        mcopts=" --cluster=172.17.0.2"
+        mself=" --self=1"
+        mka=" --ka=172.17.0.2:172.17.0.3"
+
+        bpat="cluster=172.17.0.3"
+        bcopts=" --cluster=172.17.0.3"
+        bself=" --self=0"
+        bka=" --ka=172.17.0.3:172.17.0.2"
+    fi
+    echo "Restarting $master"
+    pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2)
+    echo "Killing $mpid" >&2
+    docker exec -dt $master kill -9 $pid
+    docker exec -dt $master ip link del llb0
+    docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $mcopts $mself $mka > /dev/null &
+    pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2)
+    echo "New loxilb pid: $pid" >&2
+
+    echo "Restarting $backup"
+    pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2)
+    echo "Killing $pid" >&2
+    docker exec -dt $backup kill -9 $pid
+    docker exec -dt $backup ip link del llb0
+    docker exec -dt $backup nohup /root/loxilb-io/loxilb/loxilb $bcopts $bself $bka > /dev/null &
+    pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2)
+    echo "New loxilb pid: $pid" >&2
+
+}
+
+