feat(unmanaged-sriov): add udev & networkd support for unmanaged devices

- Add udev rules to mark SR-IOV devices which are transparently bonded to hv_netvsc devices in 10-azure-unmanaged-sriov.rules. - Add initramfs hooks in case udev rules need to be available in initramfs. Packaging can drop them if desired. - Add networkd configuration to configured these devices as unmanaged, if marked. - Add networkd.cmake for networkd-related cmake configuration. - Move udev-related cmake configuration into new udev.cmake. Signed-off-by: Chris Patterson <[email protected]> Signed-off-by: Haiyang Zhang <[email protected]>
cjp256 · Sep 17, 2024 · a6ffc03 · a6ffc03
1 parent 0f55fd7
commit a6ffc03
Show file tree

Hide file tree

Showing 9 changed files with 84 additions and 14 deletions.
diff --git a/.github/workflows/debs.yml b/.github/workflows/debs.yml
@@ -42,7 +42,10 @@ jobs:
         set -x
         dpkg -L azure-vm-utils
         test -f /usr/share/initramfs-tools/hooks/azure-disk
+        test -f /usr/share/initramfs-tools/hooks/azure-unmanaged-sriov
         test -f /usr/share/man/man8/azure-nvme-id.8.gz
         test -f /usr/sbin/azure-nvme-id
+        test -f /lib/systemd/network/10-azure-unmanaged-sriov.network
+        test -f /lib/udev/rules.d/10-azure-unmanaged-sriov.rules
         test -f /lib/udev/rules.d/80-azure-disk.rules
         azure-nvme-id --version
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -35,9 +35,12 @@ jobs:
         sudo make -C build install
     - name: Verify installation
       run: |
+        set -x
+        test -f /usr/local/lib/udev/rules.d/10-azure-unmanaged-sriov.rules
+        test -f /usr/local/lib/udev/rules.d/80-azure-disk.rules
         test -f /usr/local/sbin/azure-nvme-id
         test -f /usr/local/share/man/man8/azure-nvme-id.8
-        test -f /usr/local/lib/udev/rules.d/80-azure-disk.rules
+        test -f /usr/local/systemd/network/10-azure-unmanaged-sriov.network
         azure-nvme-id --version
     - name: Verify manpages are not generated by default
       run: |

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,6 +13,8 @@ endif()
 include(cmake/cppcheck.cmake)
 include(cmake/clangformat.cmake)
 include(cmake/doc.cmake)
+include(cmake/networkd.cmake)
+include(cmake/udev.cmake)
 
 include(CTest)
 enable_testing()
@@ -21,9 +23,8 @@ add_compile_options(-Wextra -Wall $<$<CONFIG:Debug>:-Werror> -std=gnu11 -D_GNU_S
 add_executable(azure-nvme-id src/main.c)
 
 set(AZURE_NVME_ID_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/sbin")
-set(DRACUT_MODULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/dracut/modules.d/97azure-disk" CACHE PATH "dracut modules installation directory")
+set(DRACUT_MODULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/dracut/modules.d" CACHE PATH "dracut modules installation directory")
 set(INITRAMFS_HOOKS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/share/initramfs-tools/hooks" CACHE PATH "initramfs-tools hooks installation directory")
-set(UDEV_RULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/udev/rules.d" CACHE PATH "udev rules.d installation directory")
 
 configure_file(
     "${CMAKE_SOURCE_DIR}/src/version.h.in"
@@ -33,24 +34,20 @@ configure_file(
 
 install(TARGETS azure-nvme-id DESTINATION ${AZURE_NVME_ID_INSTALL_DIR})
 
-# Install udev rules after updating "/usr/sbin/azure-nvme-id" with installed path $AZURE_NVME_ID_INSTALL_DIR.
-file(READ ${CMAKE_SOURCE_DIR}/udev/80-azure-disk.rules CONTENT)
-string(REPLACE "/usr/sbin/azure-nvme-id" "${AZURE_NVME_ID_INSTALL_DIR}/azure-nvme-id" CONTENT "${CONTENT}")
-file(WRITE ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules ${CONTENT})
-install(FILES ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})
-
 find_program(INITRAMFS_TOOLS update-initramfs)
 find_program(DRACUT dracut)
 if(INITRAMFS_TOOLS AND INITRAMFS_HOOKS_INSTALL_DIR)
-    message(STATUS "initramfs-tools found, installing initramfs hook")
-    set(HOOK_SCRIPT ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-disk)
-    install(FILES ${HOOK_SCRIPT}
+    message(STATUS "initramfs-tools found, installing initramfs hooks")
+    install(FILES ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-disk ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-unmanaged-sriov
         DESTINATION ${INITRAMFS_HOOKS_INSTALL_DIR}
         PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
 elseif(DRACUT AND DRACUT_MODULES_INSTALL_DIR)
-    message(STATUS "dracut found, installing dracut module")
+    message(STATUS "dracut found, installing dracut modules")
     install(FILES ${CMAKE_SOURCE_DIR}/initramfs/dracut/modules.d/97azure-disk/module-setup.sh
-        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}
+        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}/97azure-disk
+        PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
+    install(FILES ${CMAKE_SOURCE_DIR}/initramfs/dracut/modules.d/97azure-unmanaged-sriov/module-setup.sh
+        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}/97azure-unmanaged-sriov
         PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
 else()
     message(STATUS "initramfs-tools and dracut not found, skipping installation of azure-disk hook")

diff --git a/cmake/networkd.cmake b/cmake/networkd.cmake
@@ -0,0 +1,3 @@
+set(NETWORKD_CONFIGS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/systemd/network" CACHE PATH "networkd configs installation directory")
+
+install(FILES ${CMAKE_SOURCE_DIR}/networkd/10-azure-unmanaged-sriov.network DESTINATION ${NETWORKD_CONFIGS_INSTALL_DIR})
diff --git a/cmake/udev.cmake b/cmake/udev.cmake
@@ -0,0 +1,10 @@
+set(UDEV_RULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/udev/rules.d" CACHE PATH "udev rules.d installation directory")
+
+# Install Azure unmanaged SR-IOV rules and configuration.
+install(FILES ${CMAKE_SOURCE_DIR}/udev/10-azure-unmanaged-sriov.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})
+
+# Install udev rules after updating "/usr/sbin/azure-nvme-id" with installed path $AZURE_NVME_ID_INSTALL_DIR.
+file(READ ${CMAKE_SOURCE_DIR}/udev/80-azure-disk.rules CONTENT)
+string(REPLACE "/usr/sbin/azure-nvme-id" "${AZURE_NVME_ID_INSTALL_DIR}/azure-nvme-id" CONTENT "${CONTENT}")
+file(WRITE ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules ${CONTENT})
+install(FILES ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})
diff --git a/initramfs/dracut/modules.d/97azure-unmanaged-sriov/module-setup.sh b/initramfs/dracut/modules.d/97azure-unmanaged-sriov/module-setup.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+# called by dracut
+check() {
+    return 0
+}
+
+# called by dracut
+depends() {
+    return 0
+}
+
+# called by dracut
+install() {
+    inst_rules 10-azure-unmanaged-sriov.rules
+}
diff --git a/initramfs/initramfs-tools/hooks/azure-unmanaged-sriov b/initramfs/initramfs-tools/hooks/azure-unmanaged-sriov
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+PREREQ="udev"
+
+prereqs()
+{
+    echo "${PREREQ}"
+}
+
+case $1 in
+prereqs)
+    prereqs
+    exit 0
+    ;;
+esac
+
+. /usr/share/initramfs-tools/hook-functions
+
+DESTINATION="${DESTDIR}/lib/udev/rules.d/"
+RULES_FILE="/usr/lib/udev/rules.d/10-azure-unmanaged-sriov.rules"
+
+if [ -e "${RULES_FILE}" ]; then
+    mkdir -p "${DESTINATION}"
+    cp -p "${RULES_FILE}" "${DESTINATION}"
+fi
diff --git a/networkd/10-azure-unmanaged-sriov.network b/networkd/10-azure-unmanaged-sriov.network
@@ -0,0 +1,10 @@
+# Azure VMs with accelerated networking may have MANA, mlx4, or mlx5 SR-IOV
+# devices which are transparently bonded to a synthetic hv_netvsc device.
+# 10-azure-unmanaged-sriov.rules will mark these devices with
+# AZURE_UNMANAGED_SRIOV=1 so this can configure the devices as unmanaged.
+
+[Match]
+Property=AZURE_UNMANAGED_SRIOV=1
+
+[Network]
+Unmanaged=yes
diff --git a/udev/10-azure-unmanaged-sriov.rules b/udev/10-azure-unmanaged-sriov.rules
@@ -0,0 +1,4 @@
+# Azure VMs with accelerated networking may have MANA, mlx4, or mlx5 SR-IOV devices which are transparently bonded to a synthetic
+# hv_netvsc device.  Mark these as unmanaged devices (NM_UNMANAGED=1 for NetworkManager, AZURE_UNMANAGED_SRIOV=1 for networkd, etc.).
+SUBSYSTEM=="net", ACTION!="remove", DRIVERS=="mana", ATTR{flags}=="0x?[89ABCDEF]??", ENV{NM_UNMANAGED}="1", ENV{AZURE_UNMANAGED_SRIOV}="1"
+SUBSYSTEM=="net", ACTION!="remove", DRIVERS=="hv_pci", DRIVERS=="mlx4_core|mlx5_core", ENV{NM_UNMANAGED}="1", ENV{AZURE_UNMANAGED_SRIOV}="1"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		set(NETWORKD_CONFIGS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/systemd/network" CACHE PATH "networkd configs installation directory")

		install(FILES ${CMAKE_SOURCE_DIR}/networkd/10-azure-unmanaged-sriov.network DESTINATION ${NETWORKD_CONFIGS_INSTALL_DIR})