update scripts

Signed-off-by: Jaehyun Nam <[email protected]>
daemon1024 · Jul 1, 2022 · 0752687 · 0752687
1 parent c57fb93
commit 0752687
Show file tree

Hide file tree

Showing 13 changed files with 103 additions and 85 deletions.
diff --git a/contribution/k3s/install_k3s.sh b/contribution/k3s/install_k3s.sh
@@ -2,6 +2,16 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2021 Authors of KubeArmor
 
+if [ "$RUNTIME" == "" ]; then
+    if [ -f /var/run/docker.sock ]; then
+        RUNTIME="docker"
+    elif [ -f /var/run/crio/crio.sock ]; then
+        RUNTIME="crio"
+    else # default
+        RUNTIME="containerd"
+    fi
+fi
+
 # create a single-node K3s cluster
 if [ "$RUNTIME" == "docker" ]; then # docker
     CGROUP_SYSTEMD=$(docker info 2> /dev/null | grep -i cgroup | grep systemd | wc -l)

diff --git a/contribution/local-registry/docker-registry.sh b/contribution/local-registry/docker-registry.sh
@@ -8,13 +8,13 @@ docker run -d -p 0.0.0.0:5000:5000 --restart=always --name registry registry:2
 REGIP=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')
 sudo cat <<EOF > daemon.json
 {
-"insecure-registries" : ["$REGIP:5000"]
+    "insecure-registries" : ["$REGIP:5000"]
 }
 EOF
 
 # replace daemon.json
 if [[ -f /etc/docker/daemon.json ]] && [[ ! -f /etc/docker/daemon.json.bak ]]; then
-	sudo mv /etc/docker/daemon.json /etc/docker/daemon.json.bak
+    sudo mv /etc/docker/daemon.json /etc/docker/daemon.json.bak
 fi
 sudo mv daemon.json /etc/docker/daemon.json
 sudo cat /etc/docker/daemon.json

diff --git a/contribution/self-managed-k8s-selinux/crio/install_crio.sh b/contribution/self-managed-k8s-selinux/crio/install_crio.sh
@@ -4,16 +4,16 @@
 
 . /etc/os-release
 
-if [ "$ID" != "centos" ]; then
-    echo "Supports CentOS"
+if [[ "$NAME" != "CentOS Linux" ] || [ "$VERSION" != "8" ]]; then
+    echo "Support CentOS 8"
     exit
 fi
 
 OS="CentOS_${VERSION_ID}"
 VERSION=1.19
 
 if [ "$NAME" == "CentOS Stream" ]; then
-	OS="${OS}_Stream"
+    OS="${OS}_Stream"
 fi
 
 # remove podman

diff --git a/contribution/self-managed-k8s-selinux/docker/install_docker.sh b/contribution/self-managed-k8s-selinux/docker/install_docker.sh
@@ -25,13 +25,13 @@ sudo dnf -y install docker-ce docker-ce-cli containerd.io
 sudo mkdir -p /etc/docker
 cat <<EOF | sudo tee /etc/docker/daemon.json
 {
-  "exec-opts": ["native.cgroupdriver=systemd"],
-  "log-driver": "json-file",
-  "log-opts": {
-    "max-size": "100m"
-  },
-  "storage-driver": "overlay2",
-  "selinux-enabled": true
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "100m"
+    },
+    "storage-driver": "overlay2",
+    "selinux-enabled": true
 }
 EOF
 

diff --git a/contribution/self-managed-k8s-selinux/enable_selinux.sh b/contribution/self-managed-k8s-selinux/enable_selinux.sh
@@ -5,8 +5,10 @@
 # before enabling selinux in k8s, you should install docker, k8s first
 sudo sed -i 's/SELINUX=permissive/SELINUX=enforcing/g' /etc/selinux/config
 sudo setenforce 1
-sudo setsebool container_manage_cgroup 1
 
-# change contexts
-sudo chcon -R -t svirt_sandbox_file_t /var/lib/etcd
-sudo chcon -R -t svirt_sandbox_file_t /etc/kubernetes/
+if [ -f /var/run/docker.sock ]; then
+    # change contexts
+    sudo setsebool container_manage_cgroup 1
+    sudo chcon -R -t svirt_sandbox_file_t /var/lib/etcd
+    sudo chcon -R -t svirt_sandbox_file_t /etc/kubernetes/
+fi
diff --git a/contribution/self-managed-k8s-selinux/k8s/initialize_kubernetes.sh b/contribution/self-managed-k8s-selinux/k8s/initialize_kubernetes.sh
@@ -7,9 +7,20 @@ if [ "$CNI" == "" ]; then
     CNI=cilium
 fi
 
+# use docker as default CRI
+if [ "$CRI_SOCKET" == "" ]; then
+    if [ -f /var/run/docker.sock ]; then
+        CRI_SOCKET=unix:///var/run/docker.sock
+    elif [ -f /var/run/containerd/containerd.sock ]; then
+        CRI_SOCKET=unix:///var/run/containerd/containerd.sock
+    elif [ -f /var/run/crio/crio.sock ]; then
+        CRI_SOCKET=unix:///var/run/crio/crio.sock
+    fi
+fi
+
 # check supported CNI
 if [ "$CNI" != "flannel" ] && [ "$CNI" != "weave" ] && [ "$CNI" != "calico" ] && [ "$CNI" != "cilium" ]; then
-    echo "Usage: CNI={flannel|weave|calico|cilium} MASTER={true|false} $0"
+    echo "Usage: CNI={flannel|weave|calico|cilium} CRI_SOCKET=unix:///path/to/socket_file MASTER={true|false} $0"
     exit
 fi
 
@@ -27,9 +38,9 @@ sudo chcon -R -t svirt_sandbox_file_t /etc/kubernetes/
 
 # initialize the master node
 if [ "$CNI" == "calico" ]; then
-    sudo kubeadm init --pod-network-cidr=192.168.0.0/16 | tee -a ~/k8s_init.log
+    sudo kubeadm init --cri-socket=$CRI_SOCKET --pod-network-cidr=192.168.0.0/16 | tee -a ~/k8s_init.log
 else # weave, flannel, cilium
-    sudo kubeadm init --pod-network-cidr=10.244.0.0/16 | tee -a ~/k8s_init.log
+    sudo kubeadm init --cri-socket=$CRI_SOCKET --pod-network-cidr=10.244.0.0/16 | tee -a ~/k8s_init.log
 fi
 
 # make kubectl work for non-root user

diff --git a/...aged-k8s/containerd/install-containerd.sh → ...aged-k8s/containerd/install_containerd.sh b/...aged-k8s/containerd/install-containerd.sh → ...aged-k8s/containerd/install_containerd.sh
diff --git a/...ion/self-managed-k8s/crio/install-crio.sh → ...ion/self-managed-k8s/crio/install_crio.sh b/...ion/self-managed-k8s/crio/install-crio.sh → ...ion/self-managed-k8s/crio/install_crio.sh
@@ -27,7 +27,7 @@ sudo apt-get install cri-o cri-o-runc
 
 # this option is not supported in ubuntu 18.04
 if [ "$VERSION_ID" == "18.04" ]; then
-	sudo sed -i 's/,metacopy=on//g' /etc/containers/storage.conf
+    sudo sed -i 's/,metacopy=on//g' /etc/containers/storage.conf
 fi
 
 sudo systemctl daemon-reload

diff --git a/...n/self-managed-k8s/crio/uninstall-crio.sh → ...n/self-managed-k8s/crio/uninstall_crio.sh b/...n/self-managed-k8s/crio/uninstall-crio.sh → ...n/self-managed-k8s/crio/uninstall_crio.sh
diff --git a/contribution/self-managed-k8s/docker/install_docker.sh b/contribution/self-managed-k8s/docker/install_docker.sh
@@ -46,12 +46,12 @@ esac
 sudo mkdir -p /etc/docker
 cat <<EOF | sudo tee /etc/docker/daemon.json
 {
-  "exec-opts": ["native.cgroupdriver=systemd"],
-  "log-driver": "json-file",
-  "log-opts": {
-    "max-size": "100m"
-  },
-  "storage-driver": "overlay2"
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "100m"
+    },
+    "storage-driver": "overlay2"
 }
 EOF
 

diff --git a/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh b/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh
@@ -9,18 +9,18 @@ fi
 
 # use docker as default CRI
 if [ "$CRI_SOCKET" == "" ]; then
-	if [ -f /var/run/docker.sock ]; then
-		CRI_SOCKET=unix:///var/run/docker.sock
-	elif [ -f /var/run/containerd/containerd.sock ]; then
-		CRI_SOCKET=unix:///var/run/containerd/containerd.sock
-	elif [ -f /var/run/crio/crio.sock ]; then
-		CRI_SOCKET=unix:///var/run/crio/crio.sock
-	fi
+    if [ -f /var/run/docker.sock ]; then
+        CRI_SOCKET=unix:///var/run/docker.sock
+    elif [ -f /var/run/containerd/containerd.sock ]; then
+        CRI_SOCKET=unix:///var/run/containerd/containerd.sock
+    elif [ -f /var/run/crio/crio.sock ]; then
+        CRI_SOCKET=unix:///var/run/crio/crio.sock
+    fi
 fi
 
 # check supported CNI
 if [ "$CNI" != "flannel" ] && [ "$CNI" != "weave" ] && [ "$CNI" != "calico" ] && [ "$CNI" != "cilium" ]; then
-    echo "Usage: CNI={flannel|weave|calico|cilium} CRI_SOCKET="unix:///path/to/socket_file" MASTER={true|false} $0"
+    echo "Usage: CNI={flannel|weave|calico|cilium} CRI_SOCKET=unix:///path/to/socket_file MASTER={true|false} $0"
     exit
 fi
 

diff --git a/contribution/vagrant/Vagrantfile b/contribution/vagrant/Vagrantfile
@@ -69,75 +69,52 @@ Vagrant.configure("2") do |config|
     # install base dependencies
     config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/setup.sh"
 
-    if ENV['RUNTIME'] == "containerd" then # TODO: still use docker / need to support containerd
-      # install Docker
-      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/docker/install_docker.sh"
-
-      # install Kubernetes
-      config.vm.provision :shell, :inline => "RUNTIME=docker /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh"
-
-    elsif ENV['RUNTIME'] == "crio" then
+    if ENV['RUNTIME'] == "crio" then
       # install CRI-O
       config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/crio/install_crio.sh"
-
-      # install Kubernetes
-      config.vm.provision :shell, :inline => "RUNTIME=crio /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh"
-
     else # default == 'docker'
       # install Docker
       config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/docker/install_docker.sh"
-
-      if ENV['RUNTIME'] == "k3s" then
-        # install k3s
-        config.vm.provision :shell, path: kubearmor_home + "/contribution/k3s/install_k3s.sh"
-      else
-        # install Kubernetes
-        config.vm.provision :shell, :inline => "RUNTIME=docker /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh"
-      end
     end
 
-    if ENV['RUNTIME'] != "k3s" then
+    if ENV['K8S'] == "kubeadm" then
+      # install Kubernetes
+      config.vm.provision :shell, :inline => "RUNTIME=crio /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh"
+
       # initialize Kubernetes
       config.vm.provision :shell, :inline => "CNI=cilium MASTER=true /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/initialize_kubernetes.sh"
-
-      # enable SELinux
-      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/enable_selinux.sh"
+    else # k3s by default
+      # install k3s
+      config.vm.provision :shell, path: kubearmor_home + "/contribution/k3s/install_k3s.sh"
     end
 
+    # enable SELinux
+    config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/enable_selinux.sh"
+
   else # ubuntu
     # install base dependencies
     config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/setup.sh"
 
     if ENV['RUNTIME'] == "containerd" then
       # install Containerd
-      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/containerd/install-containerd.sh"
-
-      # install Kubernetes
-      config.vm.provision :shell, :inline => "RUNTIME=containerd /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh"
-
+      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/containerd/install_containerd.sh"
     elsif ENV['RUNTIME'] == "crio" then
       # install CRI-O
-      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/crio/install-crio.sh"
-
-      # install Kubernetes
-      config.vm.provision :shell, :inline => "CRI_SOCKET=unix:///var/run/crio/crio.sock /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh"
-
+      config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/crio/install_crio.sh"
     else # default == 'docker'
       # install Docker
       config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/docker/install_docker.sh"
-
-      if ENV['RUNTIME'] == "k3s" then
-        # install k3s
-        config.vm.provision :shell, path: kubearmor_home + "/contribution/k3s/install_k3s.sh"
-      else
-        # install Kubernetes
-        config.vm.provision :shell, :inline => "RUNTIME=docker /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh"
-      end
     end
 
-    if ENV['RUNTIME'] != "k3s" then
+    if ENV['K8S'] == "kubeadm" then
+      # install Kubernetes
+      config.vm.provision :shell, :inline => "RUNTIME=containerd /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh"
+
       # initialize Kubernetes
       config.vm.provision :shell, :inline => "CNI=cilium MASTER=true /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh"
+    else # k3s by default
+      # install k3s
+      config.vm.provision :shell, path: kubearmor_home + "/contribution/k3s/install_k3s.sh"
     end
   end
 

diff --git a/tests/test-scenarios-github.sh b/tests/test-scenarios-github.sh
@@ -22,14 +22,6 @@ realpath() {
 TEST_HOME=`dirname $(realpath "$0")`
 CRD_HOME=`dirname $(realpath "$0")`/../deployments/CRD
 ARMOR_HOME=`dirname $(realpath "$0")`/../KubeArmor
-IGN_FILE=$TEST_HOME/tests.ignore
-
-# skip tests that don't work with some runtimes
-if [ "$RUNTIME" == "crio" ]; then
-	# see #697
-	echo "github_test_13" | tee -a $IGN_FILE
-	echo "github_test_09" | tee -a $IGN_FILE
-fi
 
 LSM="none"
 
@@ -630,6 +622,15 @@ res_microservice=0
 
 is_test_ignored()
 {
+    IGN_FILE=$TEST_HOME/tests.ignore
+
+    # skip tests that don't work with some runtimes
+    if [ "$RUNTIME" == "crio" ]; then
+        # skip tests for net_raw capability (see #697)
+        echo "github_test_09" | tee -a $IGN_FILE
+        echo "github_test_13" | tee -a $IGN_FILE
+    fi
+
     [[ ! -f $IGN_FILE ]] && return 0
     for line in `grep "^[a-zA-Z].*" $IGN_FILE`; do
         echo $testcase | grep $line >/dev/null
@@ -638,6 +639,20 @@ is_test_ignored()
     return 0
 }
 
+is_test_allowed()
+{
+    cnt=0
+    ALLOW_FILE=$TEST_HOME/tests.allow
+    [[ ! -f $ALLOW_FILE ]] && return 1
+    for line in `grep "^[a-zA-Z].*" $ALLOW_FILE`; do
+        echo $testcase | grep $line >/dev/null
+        [[ $? -eq 0 ]] && echo "does not match ignore pattern [$line]" && return 1
+        ((cnt++))
+    done
+    [[ $cnt -gt 0 ]] && echo "Testcase does not match any allowed pattern in [$ALLOW_FILE]" && return 0
+    return 1
+}
+
 if [[ $SKIP_CONTAINER_POLICY -eq 0 || $SKIP_NATIVE_POLICY -eq 0 ]]; then
     INFO "Running Container Scenarios"
 
@@ -663,6 +678,9 @@ if [[ $SKIP_CONTAINER_POLICY -eq 0 || $SKIP_NATIVE_POLICY -eq 0 ]]; then
         is_test_ignored
         [[ $? -eq 1 ]] && WARN "Testcase $testcase ignored" && continue
 
+        is_test_allowed
+        [[ $? -eq 0 ]] && WARN "Testcase $testcase disallowed" && continue
+
         res_case=0
 
         INFO "Testing $testcase"