Merge pull request #127 from stuggi/network_isolation

Deploy CRC with network isolation

Makefile

@@ -173,6 +173,9 @@ DATAPLANE_BRANCH     ?= main
 # Ceph
 CEPH_IMG       ?= quay.io/ceph/demo:latest
 
+# NNCP
+NNCP_INTERFACE ?= enp6s0
+
 # target vars for generic operator install info 1: target name , 2: operator name
 define vars
 ${1}: export NAMESPACE=${NAMESPACE}
@@ -1036,3 +1039,89 @@ ceph_cleanup: ## deletes the ceph pod
 	$(eval $(call vars,$@,ceph))
 	oc kustomize ${DEPLOY_DIR} | oc delete --ignore-not-found=true -f -
 	rm -Rf ${DEPLOY_DIR}
+
+##@ NMSTATE
+.PHONY: nmstate
+nmstate: export NAMESPACE=openshift-nmstate
+nmstate: ## installs nmstate operator in the openshift-nmstate namespace
+	$(eval $(call vars,$@,nmstate))
+	bash scripts/gen-namespace.sh
+	oc apply -f ${OUT}/${NAMESPACE}/namespace.yaml
+	sleep 2
+	bash scripts/gen-olm-nmstate.sh
+	oc apply -f ${OPERATOR_DIR}
+	while ! (oc get pod --no-headers=true -l app=kubernetes-nmstate-operator -n ${NAMESPACE}| grep "nmstate-operator"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} --for condition=Ready -l app=kubernetes-nmstate-operator --timeout=300s
+	oc apply -f ${DEPLOY_DIR}
+	while ! (oc get pod --no-headers=true -l component=kubernetes-nmstate-handler -n ${NAMESPACE}| grep "nmstate-handler"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} -l component=kubernetes-nmstate-handler --for condition=Ready --timeout=300s
+	while ! (oc get pod --no-headers=true -l component=kubernetes-nmstate-webhook -n ${NAMESPACE}| grep "nmstate-webhook"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} -l component=kubernetes-nmstate-webhook --for condition=Ready --timeout=300s
+
+.PHONY: nncp
+nncp: export INTERFACE=${NNCP_INTERFACE}
+nncp: ## installs the nncp resources to configure the interface connected to the edpm node, right now only single nic vlan. Interface referenced via NNCP_INTERFACE
+	$(eval $(call vars,$@,nncp))
+	WORKERS=$(shell oc get nodes -l node-role.kubernetes.io/worker -o jsonpath="{.items[*].metadata.name}") \
+	bash scripts/gen-nncp.sh
+	oc apply -f ${DEPLOY_DIR}/
+	oc wait nncp -l osp/interface=${NNCP_INTERFACE} --for condition=available --timeout=120s
+
+.PHONY: nncp_cleanup
+nncp_cleanup: export INTERFACE=${NNCP_INTERFACE}
+nncp_cleanup: ## unconfigured nncp configuration on worker node and deletes the nncp resource
+	$(eval $(call vars,$@,nncp))
+	sed -i 's/state: up/state: absent/' ${DEPLOY_DIR}/*_nncp.yaml
+	oc apply -f ${DEPLOY_DIR}/
+	oc wait nncp -l osp/interface=${NNCP_INTERFACE} --for condition=available --timeout=120s
+	oc delete --ignore-not-found=true -f ${DEPLOY_DIR}/
+	rm -Rf ${DEPLOY_DIR}
+
+.PHONY: netattach
+netattach: export INTERFACE=${NNCP_INTERFACE}
+netattach: namespace ## Creates network-attachment-definitions for the networks the workers are attached via nncp
+	$(eval $(call vars,$@,netattach))
+	bash scripts/gen-netatt.sh
+	oc apply -f ${DEPLOY_DIR}/
+
+.PHONY: netattach_cleanup
+netattach_cleanup: ## Deletes the network-attachment-definitions
+	$(eval $(call vars,$@,netattach))
+	oc delete --ignore-not-found=true -f ${DEPLOY_DIR}/
+	rm -Rf ${DEPLOY_DIR}
+
+##@ METALLB
+.PHONY: metallb
+metallb: export NAMESPACE=metallb-system
+metallb: export INTERFACE=${NNCP_INTERFACE}
+metallb: ## installs metallb operator in the metallb-system namespace
+	$(eval $(call vars,$@,metallb))
+	bash scripts/gen-namespace.sh
+	oc apply -f ${OUT}/${NAMESPACE}/namespace.yaml
+	sleep 2
+	bash scripts/gen-olm-metallb.sh
+	oc apply -f ${OPERATOR_DIR}
+	while ! (oc get pod --no-headers=true -l control-plane=controller-manager -n ${NAMESPACE}| grep "metallb-operator-controller"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} --for condition=Ready -l control-plane=controller-manager --timeout=300s
+	while ! (oc get pod --no-headers=true -l component=webhook-server -n ${NAMESPACE}| grep "metallb-operator-webhook"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} --for condition=Ready -l component=webhook-server --timeout=300s
+	oc apply -f ${DEPLOY_DIR}/deploy_operator.yaml
+	while ! (oc get pod --no-headers=true -l component=speaker -n ${NAMESPACE} | grep "speaker"); do sleep 10; done
+	oc wait pod -n ${NAMESPACE} -l component=speaker --for condition=Ready --timeout=300s
+
+.PHONY: metallb_config
+metallb_config: export NAMESPACE=metallb-system
+metallb_config: export INTERFACE=${NNCP_INTERFACE}
+metallb_config: ## creates the IPAddressPools and l2advertisement resources
+	$(eval $(call vars,$@,metallb))
+	bash scripts/gen-olm-metallb.sh
+	oc apply -f ${DEPLOY_DIR}/ipaddresspools.yaml
+	oc apply -f ${DEPLOY_DIR}/l2advertisement.yaml
+
+.PHONY: metallb_config_cleanup
+metallb_config_cleanup: export NAMESPACE=metallb-system
+metallb_config_cleanup: ## deletes the IPAddressPools and l2advertisement resources
+	$(eval $(call vars,$@,metallb))
+	oc delete --ignore-not-found=true -f ${DEPLOY_DIR}/ipaddresspools.yaml
+	oc delete --ignore-not-found=true -f ${DEPLOY_DIR}/l2advertisement.yaml
+	rm -f ${DEPLOY_DIR}/ipaddresspools.yaml ${DEPLOY_DIR}/l2advertisement.yaml

devsetup/Makefile

@@ -1,11 +1,13 @@
 CRC_URL ?= 'https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz'
 KUBEADMIN_PWD ?= 12345678
 PULL_SECRET  ?= ${PWD}/pull-secret.txt
+CRC_DEFAULT_NETWORK_IP ?= 192.168.122.10
 EDPM_COMPUTE_SUFFIX ?= 0
 EDPM_COMPUTE_IP ?= 192.168.122.100
 OPENSTACK_RUNNER_IMG ?= quay.io/openstack-k8s-operators/openstack-ansibleee-runner:latest
-EDPM_NETWORK_CONFIG_TEMPLATE ?= templates/net_config_bridge.j2
+EDPM_NETWORK_CONFIG_TEMPLATE ?= templates/single_nic_vlans/single_nic_vlans.j2
 EDPM_SSHD_ALLOWED_RANGES ?= ['192.168.122.0/24']
+EDPM_CHRONY_NTP_SERVER ?= clock.redhat.com
 
 ##@ General
 
@@ -24,6 +26,13 @@ EDPM_SSHD_ALLOWED_RANGES ?= ['192.168.122.0/24']
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Download required tools and versions
+.PHONY: download_tools
+download_tools: ## Runs an ansible playbook to install required tools with the versions to develop the service operators. The tools get installed in ~/bin and go in /usr/local/go (alternatives get used to set it as the system wide go version)
+	ANSIBLE_FORCE_COLOR=true ansible-playbook \
+	-v -i hosts \
+	download_tools.yaml
+
 ##@ CRC
 .PHONY: crc
 crc: ## Deploys CRC using CRC_URL to download and install CRC, KUBEADMIN_PWD as the password which defaults to 12345678 and PULL_SECRET to specify the file containing the pull secret, defaults to ${PWD}/pull-secret.txt. To change the default memory and/or cpus for the VM use `CPUS=X MEMORY=Y DISK=Z make crc`.
@@ -36,35 +45,58 @@ crc_cleanup: ## Destroys the CRC env, but does NOT clear ( --clear-cache ) the c
 	sudo rm -f /etc/pki/ca-trust/source/anchors/crc-router-ca.pem
 	sudo update-ca-trust
 
-##@ Download required tools and versions
-.PHONY: download_tools
-download_tools: ## Runs an ansible playbook to install required tools with the versions to develop the service operators. The tools get installed in ~/bin and go in /usr/local/go (alternatives get used to set it as the system wide go version)
-	ANSIBLE_FORCE_COLOR=true ansible-playbook \
-	-v -i hosts \
-	download_tools.yaml
-
 .PHONY: crc_attach_default_interface
-crc_attach_default_interface:
-	sudo virsh attach-interface crc --source default --type network --model virtio --config --persistent
+crc_attach_default_interface: crc_attach_default_interface_cleanup ## Attach default libvirt network to CRC
+	MAC_ADDRESS=$(shell echo -n 52:54:00; dd bs=1 count=3 if=/dev/random 2>/dev/null | hexdump -v -e '/1 "-%02X"' | tr '-' ':'); \
+	sudo virsh net-update default add-last ip-dhcp-host --xml "<host mac='$$MAC_ADDRESS' name='crc' ip='${CRC_DEFAULT_NETWORK_IP}'/>" --config --live; \
+	sudo virsh attach-interface crc --source default --type network --model virtio --mac $$MAC_ADDRESS --config --persistent; \
+	sleep 10; \
+	WORKER=$(shell oc get nodes -l node-role.kubernetes.io/worker -o jsonpath="{.items[*].metadata.name}"); \
+	oc debug node/$$WORKER -- ip -o link | awk 'toupper($$0) ~ /ETHER $$MAC_ADDRESS/{print $$2}'  | awk -F: '{print $$1}'
+
+.PHONY: crc_attach_default_interface_cleanup
+crc_attach_default_interface_cleanup: ## Detach default libvirt network from CRC
+	-MAC_ADDRESS=$(shell sudo virsh net-dumpxml default | grep crc | sed -e "s/.*mac='\(.*\)' name.*/\1/"); \
+	sudo virsh detach-interface crc network --mac "$$MAC_ADDRESS"
+	-sudo virsh net-update default delete ip-dhcp-host "<host name='crc'/>" --config --live
+	sleep 5
 
+##@ EDPM
 .PHONY: edpm_compute
-edpm_compute:
+edpm_compute: ## Create EDPM compute VM
 	scripts/gen-ansibleee-ssh-key.sh
 	scripts/gen-edpm-compute-node.sh ${EDPM_COMPUTE_SUFFIX}
 
 .PHONY: edpm_compute_cleanup
-edpm_compute_cleanup:
+edpm_compute_cleanup: ## Delete EDPM compute VM
 	scripts/edpm-compute-cleanup.sh ${EDPM_COMPUTE_SUFFIX}
 
 .PHONY: edpm_play
-edpm_play:
+edpm_play: export EDPM_OVN_METADATA_AGENT_TRANSPORT_URL=$(shell oc get secret rabbitmq-transport-url-neutron-neutron-transport -o json | jq -r .data.transport_url | base64 -d)
+edpm_play: export EDPM_OVN_METADATA_AGENT_SB_CONNECTION=$(shell oc get ovndbcluster ovndbcluster-sb -o json | jq -r .status.dbAddress)
+edpm_play: export EDPM_OVN_DBS=$(shell oc get ovndbcluster ovndbcluster-sb -o json | jq -r '.status.networkAttachments."openstack/internalapi"[0]')
+edpm_play: export EDPM_OVN_METADATA_AGENT_NOVA_METADATA_HOST=127.0.0.1
+edpm_play: export EDPM_OVN_METADATA_AGENT_PROXY_SHARED_SECRET=12345678
+edpm_play: export EDPM_OVN_METADATA_AGENT_BIND_HOST=127.0.0.1
+edpm_play: export EDPM_NOVA_NOTIFICATIONS_TRANSPORT_URL=$(shell oc get secret rabbitmq-transport-url-nova-api-transport -o json | jq -r .data.transport_url | base64 -d)
+edpm_play: export EDPM_NOVA_COMPUTE_TRANSPORT_URL=$(shell oc get secret rabbitmq-transport-url-cell1-transport -o json | jq -r .data.transport_url | base64 -d)
+edpm_play: ## Deploy EDPM node using openstackansibleee resource
 	scripts/gen-ansibleee-ssh-key.sh
 	sed -e "s|_COMPUTE_IP_|${EDPM_COMPUTE_IP}|g" \
 	    -e "s|_OPENSTACK_RUNNER_IMG_|${OPENSTACK_RUNNER_IMG}|g" \
 	    -e "s|_EDPM_NETWORK_CONFIG_TEMPLATE_|${EDPM_NETWORK_CONFIG_TEMPLATE}|g" \
 	    -e "s|_EDPM_SSHD_ALLOWED_RANGES_|${EDPM_SSHD_ALLOWED_RANGES}|g" \
+	    -e "s|_EDPM_CHRONY_NTP_SERVER_|${EDPM_CHRONY_NTP_SERVER}|g" \
+	    -e "s|_EDPM_OVN_METADATA_AGENT_TRANSPORT_URL_|${EDPM_OVN_METADATA_AGENT_TRANSPORT_URL}|g" \
+	    -e "s|_EDPM_OVN_METADATA_AGENT_SB_CONNECTION_|${EDPM_OVN_METADATA_AGENT_SB_CONNECTION}|g" \
+	    -e "s|_EDPM_OVN_DBS_|${EDPM_OVN_DBS}|g" \
+	    -e "s|_EDPM_OVN_METADATA_AGENT_NOVA_METADATA_HOST_|${EDPM_OVN_METADATA_AGENT_NOVA_METADATA_HOST}|g" \
+	    -e "s|_EDPM_OVN_METADATA_AGENT_PROXY_SHARED_SECRET_|${EDPM_OVN_METADATA_AGENT_PROXY_SHARED_SECRET}|g" \
+	    -e "s|_EDPM_OVN_METADATA_AGENT_BIND_HOST_|${EDPM_OVN_METADATA_AGENT_BIND_HOST}|g" \
+	    -e "s|_EDPM_NOVA_COMPUTE_TRANSPORT_URL_|${EDPM_NOVA_COMPUTE_TRANSPORT_URL}|g" \
+	    -e "s|_EDPM_NOVA_NOTIFICATIONS_TRANSPORT_URL_|${EDPM_NOVA_NOTIFICATIONS_TRANSPORT_URL}|g" \
 	    edpm/edpm-play.yaml | oc create -f -
 
 .PHONY: edpm_play_cleanup
-edpm_play_cleanup:
+edpm_play_cleanup: ## Cleanup EDPM openstackansibleee resource
 	-oc delete openstackansibleee deploy-external-dataplane-compute