WIP

Signed-off-by: Or Shoval <[email protected]>
oshoval · Mar 6, 2023 · f0aa327 · f0aa327
1 parent e9b40c3
commit f0aa327
Show file tree

Hide file tree

Showing 2 changed files with 84 additions and 25 deletions.
diff --git a/cluster-up/cluster/k3d/common.sh b/cluster-up/cluster/k3d/common.sh
@@ -10,16 +10,16 @@ function detect_cri() {
 }
 
 export CRI_BIN=${CRI_BIN:-$(detect_cri)}
-KUBEVIRT_NUM_SERVERS=${KUBEVIRT_NUM_SERVERS:-1}
-KUBEVIRT_NUM_AGENTS=${KUBEVIRT_NUM_AGENTS:-2}
+export KUBEVIRT_NUM_SERVERS=1
+export KUBEVIRT_NUM_AGENTS=2
 
 export KUBEVIRTCI_PATH
 export KUBEVIRTCI_CONFIG_PATH
 
-REGISTRY_NAME=registry
+export REGISTRY_NAME=registry
 REGISTRY_HOST=127.0.0.1
-KUBERNETES_SERVICE_HOST=127.0.0.1
-KUBERNETES_SERVICE_PORT=6443
+#export KUBERNETES_SERVICE_HOST=127.0.0.1
+#export KUBERNETES_SERVICE_PORT="6443"
 
 function _ssh_into_node() {
     ${CRI_BIN} exec -it "$@"
@@ -119,34 +119,37 @@ function _label_agents() {
 function _create_cluser() {
     echo "STEP: Create cluster"
 
-    id1=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-1
-    id2=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-2
-    id3=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-3
+    export id1=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-1
+    export id2=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-2
+    export id3=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/machine-id-3
     printf '%.0s1' {1..32} > ${id1}
     printf '%.0s2' {1..32} > ${id2}
     printf '%.0s3' {1..32} > ${id3}
 
-    [ $CRI_BIN == podman ] && NETWORK=podman || NETWORK=bridge
+    [ $CRI_BIN == podman ] && export NETWORK=podman || export NETWORK=bridge
 
     k3d registry create --default-network $NETWORK $REGISTRY_NAME --port $REGISTRY_HOST:$HOST_PORT
     ${CRI_BIN} rename k3d-$REGISTRY_NAME $REGISTRY_NAME
 
-    k3d cluster create $CLUSTER_NAME --registry-use $REGISTRY_NAME \
-        --api-port $KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT \
-        --servers=$KUBEVIRT_NUM_SERVERS \
-        --agents=$KUBEVIRT_NUM_AGENTS \
-        --k3s-arg "--disable=traefik@server:0" \
-        --no-lb \
-        --k3s-arg "--flannel-backend=none@server:*" \
-        --k3s-arg "--kubelet-arg=cpu-manager-policy=static@agent:*" \
-        --k3s-arg "--kubelet-arg=kube-reserved=cpu=500m@agent:*" \
-        --k3s-arg "--kubelet-arg=system-reserved=cpu=500m@agent:*" \
-        --volume "$(pwd)/cluster-up/cluster/k3d/manifests/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml@server:0" \
-        -v /dev/vfio:/dev/vfio@agent:* \
-        -v /lib/modules:/lib/modules@agent:* \
-        -v ${id1}:/etc/machine-id@server:0 \
-        -v ${id2}:/etc/machine-id@agent:0 \
-        -v ${id3}:/etc/machine-id@agent:1
+    export CALICO=$(realpath cluster-up/cluster/k3d/manifests/calico.yaml)
+
+    k3d cluster create --config cluster-up/cluster/k3d/manifests/k3d.yaml
+    #k3d cluster create $CLUSTER_NAME --registry-use $REGISTRY_NAME \
+    #    --api-port $KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT \
+    #    --servers=$KUBEVIRT_NUM_SERVERS \
+        # --agents=$KUBEVIRT_NUM_AGENTS \
+        # --k3s-arg "--disable=traefik@server:0" \
+        # --no-lb \
+        # --k3s-arg "--flannel-backend=none@server:*" \
+        # --k3s-arg "--kubelet-arg=cpu-manager-policy=static@agent:*" \
+        # --k3s-arg "--kubelet-arg=kube-reserved=cpu=500m@agent:*" \
+        # --k3s-arg "--kubelet-arg=system-reserved=cpu=500m@agent:*" \
+        # --volume "$(pwd)/cluster-up/cluster/k3d/manifests/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml@server:0" \
+        # -v /dev/vfio:/dev/vfio@agent:* \
+        # -v /lib/modules:/lib/modules@agent:* \
+        # -v ${id1}:/etc/machine-id@server:0 \
+        # -v ${id2}:/etc/machine-id@agent:0 \
+        # -v ${id3}:/etc/machine-id@agent:1
 }
 
 function k3d_up() {

diff --git a/cluster-up/cluster/k3d/manifests/k3d.yaml b/cluster-up/cluster/k3d/manifests/k3d.yaml
@@ -0,0 +1,56 @@
+apiVersion: k3d.io/v1alpha4
+kind: Simple
+metadata:
+  name: $CLUSTER_NAME
+servers: $KUBEVIRT_NUM_SERVERS
+agents: $KUBEVIRT_NUM_AGENTS
+kubeAPI:
+  hostIP: "127.0.0.1"
+  hostPort: "6443"
+network: $NETWORK
+volumes:
+  - volume: ${CALICO}:/var/lib/rancher/k3s/server/manifests/calico.yaml
+    nodeFilters:
+      - server:0
+  - volume: /dev/vfio:/dev/vfio
+    nodeFilters:
+      - agent:*
+  - volume: /lib/modules:/lib/modules
+    nodeFilters:
+      - agent:*
+  - volume: ${id1}:/etc/machine-id
+    nodeFilters:
+      - server:0
+  - volume: ${id2}:/etc/machine-id
+    nodeFilters:
+      - agent:0
+  - volume: ${id3}:/etc/machine-id
+    nodeFilters:
+      - agent:1
+registries:
+  use:
+    - $REGISTRY_NAME:$HOST_PORT
+options:
+  k3d:
+    wait: false
+    disableLoadbalancer: true
+  k3s:
+    extraArgs:
+      - arg: "--disable=traefik"
+        nodeFilters:
+          - server:0
+      - arg: "--flannel-backend=none"
+        nodeFilters:
+          - server:*
+      - arg: "--kubelet-arg=cpu-manager-policy=static"
+        nodeFilters:
+          - agent:*
+      - arg: "--kubelet-arg=kube-reserved=cpu=500m"
+        nodeFilters:
+          - agent:*
+      - arg: "--kubelet-arg=system-reserved=cpu=500m"
+        nodeFilters:
+          - agent:*
+      - arg: "--debug"
+        nodeFilters:
+          - "agent:*"