Skip to content
This repository has been archived by the owner on Oct 22, 2024. It is now read-only.

Commit

Permalink
tests: run device manager unit tests inside virtual machine
Browse files Browse the repository at this point in the history 
Device manager tests need root/sudo user permission, and loop control device on
host where they run. So to avoid this we run these tests under a new virtual
machine.

To make use of existing test cluster scripts, made needed changes to them to
support creating a single pmem node.
  • Loading branch information
@avalluri
avalluri committed Nov 7, 2019
1 parent b877718 commit 45f21a6
Show file tree
Hide file tree
Showing 6 changed files with 223 additions and 162 deletions.
157 changes: 87 additions & 70 deletions test/setup-clear-govm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,22 @@ set -x
set -o errexit # TODO: replace with explicit error checking and error messages.
set -o pipefail

: ${TEST_INIT_REGION:=true}
: ${INIT_KUBERNETES:=true}
HOSTNAME=${HOSTNAME:-$1}
IPADDR=${IPADDR:-127.0.0.1}
BUNDLES="cloud-native-basic containers-basic ${TEST_CLEAR_LINUX_BUNDLES}"
BUNDLES=" ${TEST_CLEAR_LINUX_BUNDLES}"
if ${INIT_KUBERNETES}; then
BUNDLES="${BUNDLES} cloud-native-basic containers-basic"
fi

function error_handler(){
local line="${1}"
echo >&2 "ERROR: the command '${BASH_COMMAND}' at $0:${line} failed"
}
trap 'error_handler ${LINENO}' ERR

function install_kubernetes(){
trap 'error_handler ${LINENO}' ERR
function install_bundles(){
# Setup clearlinux environment
# Disable swupd autoupdate service
swupd autoupdate --disable
Expand Down Expand Up @@ -54,100 +59,112 @@ EOF
fi
swapoff -a

# We put config changes in place for both runtimes, even though only one of them will
# be used by Kubernetes, just in case that someone wants to use them manually.
if ${INIT_KUBERNETES}; then
# We put config changes in place for both runtimes, even though only one of them will
# be used by Kubernetes, just in case that someone wants to use them manually.

# Proxy settings for CRI-O.
mkdir /etc/systemd/system/crio.service.d
cat >/etc/systemd/system/crio.service.d/proxy.conf <<EOF
# Proxy settings for CRI-O.
mkdir /etc/systemd/system/crio.service.d
cat >/etc/systemd/system/crio.service.d/proxy.conf <<EOF
[Service]
Environment="HTTP_PROXY=${HTTP_PROXY}" "HTTPS_PROXY=${HTTPS_PROXY}" "NO_PROXY=${NO_PROXY}"
EOF

# Testing may involve a Docker registry running on the build host (see
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
# registry, otherwise CRI-O will fail to pull images from it.
# Testing may involve a Docker registry running on the build host (see
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
# registry, otherwise CRI-O will fail to pull images from it.

mkdir -p /etc/containers
cat >/etc/containers/registries.conf <<EOF
mkdir -p /etc/containers
cat >/etc/containers/registries.conf <<EOF
[registries.insecure]
registries = [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ]
EOF

# The same for Docker.
mkdir -p /etc/docker
cat >/etc/docker/daemon.json <<EOF
# The same for Docker.
mkdir -p /etc/docker
cat >/etc/docker/daemon.json <<EOF
{ "insecure-registries": [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ] }
EOF

# Proxy settings for Docker.
mkdir -p /etc/systemd/system/docker.service.d/
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
# Proxy settings for Docker.
mkdir -p /etc/systemd/system/docker.service.d/
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
[Service]
Environment="HTTP_PROXY=$HTTP_PROXY" "HTTPS_PROXY=$HTTPS_PROXY" "NO_PROXY=$NO_PROXY"
EOF

# Disable the use of Kata containers as default runtime in Docker.
# The Kubernetes control plan (apiserver, etc.) fails to run otherwise
# ("Host networking requested, not supported by runtime").
# Disable the use of Kata containers as default runtime in Docker.
# The Kubernetes control plan (apiserver, etc.) fails to run otherwise
# ("Host networking requested, not supported by runtime").

cat >/etc/systemd/system/docker.service.d/51-runtime.conf <<EOF
cat >/etc/systemd/system/docker.service.d/51-runtime.conf <<EOF
[Service]
Environment="DOCKER_DEFAULT_RUNTIME=--default-runtime runc"
EOF
mkdir -p /etc/systemd/system/kubelet.service.d/
case $TEST_CRI in
docker)
cri_daemon=docker
# Choose Docker by disabling the use of CRI-O in KUBELET_EXTRA_ARGS.
cat >/etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF

mkdir -p /etc/systemd/system/kubelet.service.d/
case $TEST_CRI in
docker)
cri_daemon=docker
# Choose Docker by disabling the use of CRI-O in KUBELET_EXTRA_ARGS.
cat >/etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS="
EOF
;;
crio)
cri_daemon=cri-o
;;
*)
echo "ERROR: unsupported TEST_CRI=$TEST_CRI"
exit 1
;;
esac

# kubelet must start after the container runtime that it depends on.
# This is not currently configured in Clear Linux (https://github.com/clearlinux/distribution/issues/1004).
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
;;
crio)
cri_daemon=cri-o
;;
*)
echo "ERROR: unsupported TEST_CRI=$TEST_CRI"
exit 1
;;
esac

# kubelet must start after the container runtime that it depends on.
# This is not currently configured in Clear Linux (https://github.com/clearlinux/distribution/issues/1004).
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
[Unit]
After=$cri_daemon.service
EOF
# flannel + CRI-O + Kata Containers needs a crio.conf change (https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes):
# If you are using CRI-O and flannel and you want to use Kata Containers, edit the /etc/crio/crio.conf file to add:
# [crio.runtime]
# manage_network_ns_lifecycle = true
#
# We don't use Kata Containers, so that particular change is not made to /etc/crio/crio.conf
# at this time.

# /opt/cni/bin is where runtimes like CRI-O expect CNI plugins. But cloud-native-basic installs into
# /usr/libexec/cni. Instructions at https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes#id2
# are inconsistent at this time (https://github.com/clearlinux/clear-linux-documentation/issues/388).
#
# We solve this by creating the directory and symlinking all existing CNI plugins into it.
mkdir -p /opt/cni/bin
for i in /usr/libexec/cni/*;do
ln -s $i /opt/cni/bin/
done

# Reconfiguration done, start daemons. Starting kubelet must wait until kubeadm has created
# the necessary config files.
systemctl daemon-reload
systemctl restart $cri_daemon || (
systemctl status $cri_daemon || true
journalctl -xe || true
false
)
systemctl enable $cri_daemon kubelet
fi
}

install_bundles

# flannel + CRI-O + Kata Containers needs a crio.conf change (https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes):
# If you are using CRI-O and flannel and you want to use Kata Containers, edit the /etc/crio/crio.conf file to add:
# [crio.runtime]
# manage_network_ns_lifecycle = true
#
# We don't use Kata Containers, so that particular change is not made to /etc/crio/crio.conf
# at this time.

# /opt/cni/bin is where runtimes like CRI-O expect CNI plugins. But cloud-native-basic installs into
# /usr/libexec/cni. Instructions at https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes#id2
# are inconsistent at this time (https://github.com/clearlinux/clear-linux-documentation/issues/388).
#
# We solve this by creating the directory and symlinking all existing CNI plugins into it.
mkdir -p /opt/cni/bin
for i in /usr/libexec/cni/*;do
ln -s $i /opt/cni/bin/
done

# Reconfiguration done, start daemons. Starting kubelet must wait until kubeadm has created
# the necessary config files.
systemctl daemon-reload
systemctl restart $cri_daemon || (
systemctl status $cri_daemon || true
journalctl -xe || true
false
)
systemctl enable $cri_daemon kubelet
function init_region(){
sudo ndctl disable-region region0
sudo ndctl init-labels nmem0
sudo ndctl enable-region region0
}

install_kubernetes
if $TEST_INIT_REGION; then
init_region
fi
102 changes: 60 additions & 42 deletions test/setup-fedora-govm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ set -x
set -o errexit # TODO: replace with explicit error checking and error messages.
set -o pipefail

: ${TEST_INIT_REGION:=true}
: ${INIT_KUBERNETES:=true}
HOSTNAME=${HOSTNAME:-$1}
IPADDR=${IPADDR:-127.0.0.1}

Expand All @@ -19,29 +21,32 @@ function error_handler(){
}
trap 'error_handler ${LINENO}' ERR

# Always use Docker, and always use the same version for reproducibility.
cat <<'EOF' > /etc/yum.repos.d/docker-ce.repo

# For PMEM.
packages+=" ndctl"

# Some additional utilities.
packages+=" device-mapper-persistent-data lvm2"

if ${INIT_KUBERNETES}; then
# Always use Docker, and always use the same version for reproducibility.
cat <<'EOF' > /etc/yum.repos.d/docker-ce.repo
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://download.docker.com/linux/centos/7/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg
EOF
packages+=" docker-ce-3:19.03.2-3.el7"

# For PMEM.
packages+=" ndctl"
packages+=" docker-ce-3:19.03.2-3.el7"

# Some additional utilities.
packages+=" device-mapper-persistent-data lvm2"
# Install according to https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
modprobe br_netfilter
echo 1 >/proc/sys/net/bridge/bridge-nf-call-iptables
setenforce 0
sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux

# Install according to https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
modprobe br_netfilter
echo 1 >/proc/sys/net/bridge/bridge-nf-call-iptables
setenforce 0
sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
Expand All @@ -54,14 +59,15 @@ EOF
# For the sake of reproducibility, use fixed versions.
# List generated with:
# for v in 1.13 1.14 1.15 1.16; do for i in kubelet kubeadm kubectl; do echo "$i-$(sudo yum --showduplicates list kubelet | grep " $v" | sed -e 's/.* \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/' | sort -u | tail -n 1)"; done; done
case ${TEST_KUBERNETES_VERSION} in
1.13) packages+=" kubelet-1.13.9-0 kubeadm-1.13.9-0 kubectl-1.13.9-0";;
1.14) packages+=" kubelet-1.14.7-0 kubeadm-1.14.7-0 kubectl-1.14.7-0";;
1.15) packages+=" kubelet-1.15.4-0 kubeadm-1.15.4-0 kubectl-1.15.4-0";;
1.16) packages+=" kubelet-1.16.0-0 kubeadm-1.16.0-0 kubectl-1.16.0-0";;
*) echo >&2 "Kubernetes version ${TEST_KUBERNETES_VERSION} not supported, package list in $0 must be updated."; exit 1;;
esac
packages+=" --disableexcludes=kubernetes"
case ${TEST_KUBERNETES_VERSION} in
1.13) packages+=" kubelet-1.13.9-0 kubeadm-1.13.9-0 kubectl-1.13.9-0";;
1.14) packages+=" kubelet-1.14.7-0 kubeadm-1.14.7-0 kubectl-1.14.7-0";;
1.15) packages+=" kubelet-1.15.4-0 kubeadm-1.15.4-0 kubectl-1.15.4-0";;
1.16) packages+=" kubelet-1.16.0-0 kubeadm-1.16.0-0 kubectl-1.16.0-0";;
*) echo >&2 "Kubernetes version ${TEST_KUBERNETES_VERSION} not supported, package list in $0 must be updated."; exit 1;;
esac
packages+=" --disableexcludes=kubernetes"
fi

# Sometimes we hit a bad mirror and get "Failed to synchronize cache for repo ...".
# https://unix.stackexchange.com/questions/487635/fedora-29-failed-to-synchronize-cache-for-repo-fedora-modular
Expand All @@ -81,34 +87,46 @@ while ! yum install -y $packages; do
fi
done

# Upstream kubelet looks in /opt/cni/bin, actual files are in
# /usr/libexec/cni from
# containernetworking-plugins-0.8.1-1.fc30.x86_64.
mkdir -p /opt/cni
ln -s /usr/libexec/cni /opt/cni/bin

# Testing may involve a Docker registry running on the build host (see
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
# registry, otherwise Docker will fail to pull images from it.
mkdir -p /etc/docker
cat >/etc/docker/daemon.json <<EOF
if $INIT_KUBERNETES; then
# Upstream kubelet looks in /opt/cni/bin, actual files are in
# /usr/libexec/cni from
# containernetworking-plugins-0.8.1-1.fc30.x86_64.
mkdir -p /opt/cni
ln -s /usr/libexec/cni /opt/cni/bin

# Testing may involve a Docker registry running on the build host (see
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
# registry, otherwise Docker will fail to pull images from it.
mkdir -p /etc/docker
cat >/etc/docker/daemon.json <<EOF
{ "insecure-registries": [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ] }
EOF

# Proxy settings for Docker.
mkdir -p /etc/systemd/system/docker.service.d/
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
# Proxy settings for Docker.
mkdir -p /etc/systemd/system/docker.service.d/
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
[Service]
Environment="HTTP_PROXY=$HTTP_PROXY" "HTTPS_PROXY=$HTTPS_PROXY" "NO_PROXY=$NO_PROXY"
EOF

# kubelet must start after the container runtime that it depends on.
mkdir -p /etc/systemd/system/kubelet.service.d
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
# kubelet must start after the container runtime that it depends on.
mkdir -p /etc/systemd/system/kubelet.service.d
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
[Unit]
After=docker.service
EOF

update-alternatives --set iptables /usr/sbin/iptables-legacy
systemctl daemon-reload
systemctl enable --now docker kubelet
update-alternatives --set iptables /usr/sbin/iptables-legacy
systemctl daemon-reload
systemctl enable --now docker kubelet
fi

function init_region(){
sudo ndctl disable-region region0
sudo ndctl init-labels nmem0
sudo ndctl enable-region region0
}

if $TEST_INIT_REGION; then
init_region
fi
14 changes: 0 additions & 14 deletions test/setup-kubernetes.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ set -x
set -o errexit # TODO: replace with explicit error checking and messages
set -o pipefail

: ${TEST_INIT_REGION:=true}
: ${TEST_CREATE_REGISTRY:=false}

function error_handler(){
Expand Down Expand Up @@ -175,19 +174,6 @@ ${TEST_CONFIGURE_POST_ALL}

}


function init_region(){
trap 'error_handler ${LINENO}' ERR
sudo ndctl disable-region region0
sudo ndctl init-labels nmem0
sudo ndctl enable-region region0

}

if $TEST_INIT_REGION; then
init_region
fi

if [[ "$HOSTNAME" == *"master"* ]]; then
setup_kubernetes_master
if $TEST_CREATE_REGISTRY; then
Expand Down
Loading

0 comments on commit 45f21a6

Please sign in to comment.