Skip to content

Commit

Permalink
vpgu: Bump kind to v0.18.0, k8s-1.27.1 (#1026)
Browse files Browse the repository at this point in the history
* kind-1.27-vgpu: Duplicate previous version as is

Signed-off-by: Or Shoval <[email protected]>

* vpgu: Bump kind to v0.18.0, k8s-1.27.1

Signed-off-by: Or Shoval <[email protected]>

* vgpu: Update readme

Change location, name and remove version from the readme,
so we won't need to update it and won't confuse the reader.

Signed-off-by: Or Shoval <[email protected]>

---------

Signed-off-by: Or Shoval <[email protected]>
  • Loading branch information
oshoval authored Jun 13, 2023
1 parent f91a57e commit e29c9f6
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 4 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# K8S 1.23.3 with mdev support in a Kind cluster
# K8S with mdev support in a Kind cluster

Provides a pre-deployed k8s cluster with version 1.23.3 that runs using [kind](https://github.com/kubernetes-sigs/kind) The cluster is completely ephemeral and is recreated on every cluster restart.
Provides a pre-deployed k8s cluster that runs using [kind](https://github.com/kubernetes-sigs/kind) The cluster is completely ephemeral and is recreated on every cluster restart.
The KubeVirt containers are built on the local machine and are then pushed to a registry which is exposed at
`localhost:5000`.

Expand All @@ -9,7 +9,7 @@ The KubeVirt containers are built on the local machine and are then pushed to a
The following needs to be executed as root.

```bash
export KUBEVIRT_PROVIDER=kind-1.23-vgpu
export KUBEVIRT_PROVIDER=kind-1.x-vgpu
make cluster-up
```

Expand All @@ -18,7 +18,7 @@ The cluster can be accessed as usual:
```bash
$ cluster-up/kubectl.sh get nodes
NAME STATUS ROLES AGE VERSION
vgpu-control-plane Ready master 6m14s v1.23.3
vgpu-control-plane Ready master 6m14s v1.x.y
```

## Bringing the cluster down
Expand Down
20 changes: 20 additions & 0 deletions cluster-up/cluster/kind-1.27-vgpu/config_vgpu_cluster.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

[ $(id -u) -ne 0 ] && echo "FATAL: this script requires sudo privileges" >&2 && exit 1

set -xe

SCRIPT_PATH=$(dirname "$(realpath "$0")")

source ${SCRIPT_PATH}/vgpu-node/node.sh
echo "_kubectl: " ${_kubectl}
echo "KUBECTL_PATH: " $KUBECTL_PATH
echo "KUBEVIRTCI_PATH: " ${KUBEVIRTCI_PATH}
source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh
echo "_kubectl: " ${_kubectl}

nodes=($(_kubectl get nodes -o custom-columns=:.metadata.name --no-headers))
node::remount_sysfs "${nodes[*]}"
node::discover_host_gpus

_kubectl get nodes
47 changes: 47 additions & 0 deletions cluster-up/cluster/kind-1.27-vgpu/conformance.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"Description": "DEFAULT",
"UUID": "",
"Version": "v0.56.9",
"ResultsDir": "/tmp/sonobuoy/results",
"Resources": null,
"Filters": {
"Namespaces": ".*",
"LabelSelector": ""
},
"Limits": {
"PodLogs": {
"Namespaces": "kube-system",
"SonobuoyNamespace": true,
"FieldSelectors": [],
"LabelSelector": "",
"Previous": false,
"SinceSeconds": null,
"SinceTime": null,
"Timestamps": false,
"TailLines": null,
"LimitBytes": null
}
},
"QPS": 30,
"Burst": 50,
"Server": {
"bindaddress": "0.0.0.0",
"bindport": 8080,
"advertiseaddress": "",
"timeoutseconds": 21600
},
"Plugins": null,
"PluginSearchPath": [
"./plugins.d",
"/etc/sonobuoy/plugins.d",
"~/sonobuoy/plugins.d"
],
"Namespace": "sonobuoy",
"WorkerImage": "sonobuoy/sonobuoy:v0.56.9",
"ImagePullPolicy": "IfNotPresent",
"ImagePullSecrets": "",
"AggregatorPermissions": "clusterAdmin",
"ServiceAccountName": "sonobuoy-serviceaccount",
"ProgressUpdatesPort": "8099",
"SecurityContextMode": "nonroot"
}
59 changes: 59 additions & 0 deletions cluster-up/cluster/kind-1.27-vgpu/provider.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash

set -e

DEFAULT_CLUSTER_NAME="vgpu"
DEFAULT_HOST_PORT=5000
ALTERNATE_HOST_PORT=5001
export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME}

if [ $CLUSTER_NAME == $DEFAULT_CLUSTER_NAME ]; then
export HOST_PORT=$DEFAULT_HOST_PORT
else
export HOST_PORT=$ALTERNATE_HOST_PORT
fi

function set_kind_params() {
export KIND_VERSION="${KIND_VERSION:-0.18.0}"
export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.27.1@sha256:9915f5629ef4d29f35b478e819249e89cfaffcbfeebda4324e5c01d53d937b09}"
export KUBECTL_PATH="${KUBECTL_PATH:-/bin/kubectl}"
}

function configure_registry_proxy() {
[ "$CI" != "true" ] && return

echo "Configuring cluster nodes to work with CI mirror-proxy..."

local -r ci_proxy_hostname="docker-mirror-proxy.kubevirt-prow.svc"
local -r kind_binary_path="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kind"
local -r configure_registry_proxy_script="${KUBEVIRTCI_PATH}/cluster/kind/configure-registry-proxy.sh"

KIND_BIN="$kind_binary_path" PROXY_HOSTNAME="$ci_proxy_hostname" $configure_registry_proxy_script
}

function up() {
# load the vfio_mdev module
/usr/sbin/modprobe vfio_mdev

# print hardware info for easier debugging based on logs
echo 'Available cards'
${CRI_BIN} run --rm --cap-add=SYS_RAWIO quay.io/phoracek/lspci@sha256:0f3cacf7098202ef284308c64e3fc0ba441871a846022bb87d65ff130c79adb1 sh -c "lspci -k | grep -EA2 'VGA|3D'"
echo ""

cp $KIND_MANIFESTS_DIR/kind.yaml ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml
_add_extra_mounts
kind_up

configure_registry_proxy

# remove the rancher.io kind default storageClass
_kubectl delete sc standard

${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/config_vgpu_cluster.sh

echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready"
}

set_kind_params

source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh
32 changes: 32 additions & 0 deletions cluster-up/cluster/kind-1.27-vgpu/vgpu-node/node.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

function node::discover_host_gpus() {
local -r gpu_types=( $(find /sys/class/mdev_bus/*/mdev_supported_types) )
[ "${#gpu_types[@]}" -eq 0 ] && echo "FATAL: Could not find available GPUs on host" >&2 && return 1

local gpu_addr
local gpu_addresses=()
for path in "${gpu_types}"; do
gpu_addr="${gpu_types#/sys/class/mdev_bus/}"
gpu_addr=${gpu_addr%/*}

gpu_addresses+=( $gpu_addr )
done

echo "${gpu_addresses[@]}"
}

function node::remount_sysfs() {
local -r nodes_array=($1)
local node_exec

for node in "${nodes_array[@]}"; do

# KIND mounts sysfs as read-only by default, remount as R/W"
node_exec="${CRI_BIN} exec $node"
$node_exec mount -o remount,rw /sys
$node_exec chmod 666 /dev/vfio/vfio

done
}

0 comments on commit e29c9f6

Please sign in to comment.