diff --git a/cluster-up/cluster/k3d-1.25-sriov/provider.sh b/cluster-up/cluster/k3d-1.25-sriov/provider.sh index de5d10bd2d..1bf73409ba 100755 --- a/cluster-up/cluster/k3d-1.25-sriov/provider.sh +++ b/cluster-up/cluster/k3d-1.25-sriov/provider.sh @@ -2,67 +2,38 @@ set -e -DEFAULT_CLUSTER_NAME="sriov" -DEFAULT_HOST_PORT=5000 -ALTERNATE_HOST_PORT=5001 -export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME} - -if [ $CLUSTER_NAME == $DEFAULT_CLUSTER_NAME ]; then - export HOST_PORT=$DEFAULT_HOST_PORT -else - export HOST_PORT=$ALTERNATE_HOST_PORT -fi +export CLUSTER_NAME="sriov" +export HOST_PORT=5000 function print_sriov_data() { - nodes=$(_kubectl get nodes -o=custom-columns=:.metadata.name --no-headers) + nodes=$(_get_agent_nodes) + echo "STEP: Print SR-IOV data" for node in $nodes; do - if [[ ! "$node" =~ .*"server".* ]]; then - echo "Node: $node" - echo "VFs:" - ${CRI_BIN} exec $node /bin/sh -c "ls -l /sys/class/net/*/device/virtfn*" - echo "PFs PCI Addresses:" - ${CRI_BIN} exec $node /bin/sh -c "grep PCI_SLOT_NAME /sys/class/net/*/device/uevent" - fi + echo "Node: $node" + echo "VFs:" + ${CRI_BIN} exec $node /bin/sh -c "ls -l /sys/class/net/*/device/virtfn*" + echo "PFs PCI Addresses:" + ${CRI_BIN} exec $node /bin/sh -c "grep PCI_SLOT_NAME /sys/class/net/*/device/uevent" done -} - -# ADAPT -function configure_registry_proxy() { - [ "$CI" != "true" ] && return - - echo "Configuring cluster nodes to work with CI mirror-proxy..." - - local -r ci_proxy_hostname="docker-mirror-proxy.kubevirt-prow.svc" - local -r kind_binary_path="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kind" - local -r configure_registry_proxy_script="${KUBEVIRTCI_PATH}/cluster/kind/configure-registry-proxy.sh" - - KIND_BIN="$kind_binary_path" PROXY_HOSTNAME="$ci_proxy_hostname" $configure_registry_proxy_script + echo } function print_sriov_info() { + echo 'STEP: Available NICs' # print hardware info for easier debugging based on logs - echo 'Available NICs' ${CRI_BIN} run --rm --cap-add=SYS_RAWIO quay.io/phoracek/lspci@sha256:0f3cacf7098202ef284308c64e3fc0ba441871a846022bb87d65ff130c79adb1 sh -c "lspci | egrep -i 'network|ethernet'" - echo "" + echo } function up() { print_sriov_info - k3d_up - # REMOVE - # echo BYE - # exit 0 - - # TODO add vfio mount - # add machine-id per server - # add workers maybe - ${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/config_sriov_cluster.sh print_sriov_data - echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready" + version=$(_kubectl get node k3d-sriov-server-0 -o=custom-columns=VERSION:.status.nodeInfo.kubeletVersion --no-headers) + echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready ($version)" } source ${KUBEVIRTCI_PATH}/cluster/k3d/common.sh diff --git a/cluster-up/cluster/k3d/common.sh b/cluster-up/cluster/k3d/common.sh index bd2c864961..9f1d84e201 100755 --- a/cluster-up/cluster/k3d/common.sh +++ b/cluster-up/cluster/k3d/common.sh @@ -2,6 +2,9 @@ set -e +# See https://github.com/k3d-io/k3d/releases +K3D_TAG=v5.4.7 + function detect_cri() { if podman ps >/dev/null 2>&1; then echo podman; elif docker ps >/dev/null 2>&1; then echo docker; fi } @@ -20,19 +23,11 @@ KUBERNETES_SERVICE_HOST=127.0.0.1 KUBERNETES_SERVICE_PORT=6443 function _ssh_into_node() { - # examples: - # ./cluster-up/ssh.sh k3d-sriov-server-0 ls - # ./cluster-up/ssh.sh k3d-sriov-server-0 /bin/sh ${CRI_BIN} exec -it "$@" } -function _install_cnis { - echo "STEP: Install cnis" - _install_cni_plugins -} - function _install_cni_plugins { - # check CPU arch + echo "STEP: Install cnis" PLATFORM=$(uname -m) case ${PLATFORM} in x86_64* | i?86_64* | amd64*) @@ -58,14 +53,14 @@ function _install_cni_plugins { curl -sSL -o ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$CNI_ARCHIVE $CNI_URL fi - for node in $(_get_nodes | awk '{print $1}'); do + for node in $(_get_nodes); do ${CRI_BIN} cp "${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$CNI_ARCHIVE" $node:/ ${CRI_BIN} exec $node /bin/sh -c "mkdir -p /opt/cni/bin && tar -xvzf $CNI_ARCHIVE -C /opt/cni/bin" > /dev/null done } -function _prepare_config() { - echo "STEP: Prepare config" +function _prepare_provider_config() { + echo "STEP: Prepare provider config" cat >$BASE_PATH/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh < ${BASE_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig } -function k3d_up() { - setup_k3d +function _label_agents() { + echo "STEP: label agents" + for node in $(_get_nodes); do + if [[ "$node" =~ .*"agent".* ]]; then + _kubectl label node $node node-role.kubernetes.io/worker=worker + fi + done +} +function _create_cluser() { + echo "STEP: Create cluster" id1=${BASE_PATH}/$KUBEVIRT_PROVIDER/machine-id-1 id2=${BASE_PATH}/$KUBEVIRT_PROVIDER/machine-id-2 id3=${BASE_PATH}/$KUBEVIRT_PROVIDER/machine-id-3 - echo 11111111111111111111111111111111 > ${id1} - echo 22222222222222222222222222222222 > ${id2} - echo 33333333333333333333333333333333 > ${id3} + printf '%.0s1' {1..32} > ${id1} + printf '%.0s2' {1..32} > ${id2} + printf '%.0s3' {1..32} > ${id3} if [ $CRI_BIN == podman ]; then - NETWORK=podman - podman pull docker.io/rancher/k3s:v1.25.6-k3s1 # TODO need to update according TAG + NETWORK=podman + # TODO need to update according K3D_TAG or fix with a nicer solution + podman pull docker.io/rancher/k3s:v1.25.6-k3s1 else - NETWORK=bridge + NETWORK=bridge fi k3d registry create --default-network $NETWORK $REGISTRY_NAME --port $REGISTRY_HOST:$HOST_PORT @@ -131,20 +136,22 @@ function k3d_up() { --k3s-arg "--kubelet-arg=cpu-manager-policy=static@agent:*" \ --k3s-arg "--kubelet-arg=kube-reserved=cpu=500m@agent:*" \ --k3s-arg "--kubelet-arg=system-reserved=cpu=500m@agent:*" \ - --volume "$(pwd)/cluster-up/cluster/k3d/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml@server:0" \ + --volume "$(pwd)/cluster-up/cluster/k3d/manifests/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml@server:0" \ -v /dev/vfio:/dev/vfio@agent:* \ -v /lib/modules:/lib/modules@agent:* \ -v ${id1}:/etc/machine-id@server:0 \ -v ${id2}:/etc/machine-id@agent:0 \ -v ${id3}:/etc/machine-id@agent:1 +} - _print_kubeconfig +function k3d_up() { + _install_k3d + _create_cluser + _extract_kubeconfig _prepare_nodes - _install_cnis - _prepare_config - - kubectl label node k3d-sriov-agent-0 node-role.kubernetes.io/worker=worker - kubectl label node k3d-sriov-agent-1 node-role.kubernetes.io/worker=worker + _install_cni_plugins + _prepare_provider_config + _label_agents } function _kubectl() { @@ -153,6 +160,16 @@ function _kubectl() { } function down() { + set +e + trap "set -e" RETURN + + for agent_node in $(_get_agent_nodes); do + if ip netns exec $agent_node ip -d a | grep "vf 0" -B 2 > /dev/null; then + iface=$(ip netns exec $agent_node ip -d a | grep "vf 0" -B 2 | grep "UP" | awk -F": " '{print $2}') + ip netns exec $agent_node ip link set $iface netns 1 && echo "gracefully detached $iface from $agent_node" + fi + done + k3d cluster delete $CLUSTER_NAME - ${CRI_BIN} rm --force $REGISTRY_NAME + ${CRI_BIN} rm --force $REGISTRY_NAME > /dev/null && echo "$REGISTRY_NAME deleted" } diff --git a/cluster-up/cluster/k3d/calico.yaml b/cluster-up/cluster/k3d/manifests/calico.yaml similarity index 100% rename from cluster-up/cluster/k3d/calico.yaml rename to cluster-up/cluster/k3d/manifests/calico.yaml