feat(unmanaged-sriov): add udev & networkd support for unmanaged devices

- Add udev rules to mark SR-IOV devices which are transparently bonded to
  hv_netvsc devices in 10-azure-unmanaged-sriov.rules.

- Add initramfs hooks in case udev rules need to be available in
  initramfs.  Packaging can drop them if desired.

- Add networkd configuration to configured these devices as unmanaged,
  if marked.

- Add networkd.cmake for networkd-related cmake configuration.

- Move udev-related cmake configuration into new udev.cmake.

Signed-off-by: Chris Patterson <[email protected]>
Signed-off-by: Haiyang Zhang <[email protected]>

.github/workflows/debs.yml

@@ -42,7 +42,10 @@ jobs:
         set -x
         dpkg -L azure-vm-utils
         test -f /usr/share/initramfs-tools/hooks/azure-disk
+        test -f /usr/share/initramfs-tools/hooks/azure-unmanaged-sriov
         test -f /usr/share/man/man8/azure-nvme-id.8.gz
         test -f /usr/sbin/azure-nvme-id
+        test -f /lib/systemd/network/10-azure-unmanaged-sriov.network
+        test -f /lib/udev/rules.d/10-azure-unmanaged-sriov.rules
         test -f /lib/udev/rules.d/80-azure-disk.rules
         azure-nvme-id --version

.github/workflows/main.yml

@@ -35,9 +35,12 @@ jobs:
         sudo make -C build install
     - name: Verify installation
       run: |
+        set -x
+        test -f /usr/local/lib/systemd/network/10-azure-unmanaged-sriov.network
+        test -f /usr/local/lib/udev/rules.d/10-azure-unmanaged-sriov.rules
+        test -f /usr/local/lib/udev/rules.d/80-azure-disk.rules
         test -f /usr/local/sbin/azure-nvme-id
         test -f /usr/local/share/man/man8/azure-nvme-id.8
-        test -f /usr/local/lib/udev/rules.d/80-azure-disk.rules
         azure-nvme-id --version
     - name: Verify manpages are not generated by default
       run: |

CMakeLists.txt

@@ -13,6 +13,8 @@ endif()
 include(cmake/cppcheck.cmake)
 include(cmake/clangformat.cmake)
 include(cmake/doc.cmake)
+include(cmake/networkd.cmake)
+include(cmake/udev.cmake)
 
 include(CTest)
 enable_testing()
@@ -21,9 +23,8 @@ add_compile_options(-Wextra -Wall $<$<CONFIG:Debug>:-Werror> -std=gnu11 -D_GNU_S
 add_executable(azure-nvme-id src/main.c)
 
 set(AZURE_NVME_ID_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/sbin")
-set(DRACUT_MODULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/dracut/modules.d/97azure-disk" CACHE PATH "dracut modules installation directory")
+set(DRACUT_MODULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/dracut/modules.d" CACHE PATH "dracut modules installation directory")
 set(INITRAMFS_HOOKS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/share/initramfs-tools/hooks" CACHE PATH "initramfs-tools hooks installation directory")
-set(UDEV_RULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/udev/rules.d" CACHE PATH "udev rules.d installation directory")
 
 configure_file(
     "${CMAKE_SOURCE_DIR}/src/version.h.in"
@@ -33,24 +34,20 @@ configure_file(
 
 install(TARGETS azure-nvme-id DESTINATION ${AZURE_NVME_ID_INSTALL_DIR})
 
-# Install udev rules after updating "/usr/sbin/azure-nvme-id" with installed path $AZURE_NVME_ID_INSTALL_DIR.
-file(READ ${CMAKE_SOURCE_DIR}/udev/80-azure-disk.rules CONTENT)
-string(REPLACE "/usr/sbin/azure-nvme-id" "${AZURE_NVME_ID_INSTALL_DIR}/azure-nvme-id" CONTENT "${CONTENT}")
-file(WRITE ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules ${CONTENT})
-install(FILES ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})
-
 find_program(INITRAMFS_TOOLS update-initramfs)
 find_program(DRACUT dracut)
 if(INITRAMFS_TOOLS AND INITRAMFS_HOOKS_INSTALL_DIR)
-    message(STATUS "initramfs-tools found, installing initramfs hook")
-    set(HOOK_SCRIPT ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-disk)
-    install(FILES ${HOOK_SCRIPT}
+    message(STATUS "initramfs-tools found, installing initramfs hooks")
+    install(FILES ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-disk ${CMAKE_SOURCE_DIR}/initramfs/initramfs-tools/hooks/azure-unmanaged-sriov
         DESTINATION ${INITRAMFS_HOOKS_INSTALL_DIR}
         PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
 elseif(DRACUT AND DRACUT_MODULES_INSTALL_DIR)
-    message(STATUS "dracut found, installing dracut module")
+    message(STATUS "dracut found, installing dracut modules")
     install(FILES ${CMAKE_SOURCE_DIR}/initramfs/dracut/modules.d/97azure-disk/module-setup.sh
-        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}
+        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}/97azure-disk
+        PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
+    install(FILES ${CMAKE_SOURCE_DIR}/initramfs/dracut/modules.d/97azure-unmanaged-sriov/module-setup.sh
+        DESTINATION ${DRACUT_MODULES_INSTALL_DIR}/97azure-unmanaged-sriov
         PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
 else()
     message(STATUS "initramfs-tools and dracut not found, skipping installation of azure-disk hook")

cmake/networkd.cmake

@@ -0,0 +1,3 @@
+set(NETWORKD_CONFIGS_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/systemd/network" CACHE PATH "networkd configs installation directory")
+
+install(FILES ${CMAKE_SOURCE_DIR}/networkd/10-azure-unmanaged-sriov.network DESTINATION ${NETWORKD_CONFIGS_INSTALL_DIR})

cmake/udev.cmake

@@ -0,0 +1,10 @@
+set(UDEV_RULES_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/udev/rules.d" CACHE PATH "udev rules.d installation directory")
+
+# Install Azure unmanaged SR-IOV rules and configuration.
+install(FILES ${CMAKE_SOURCE_DIR}/udev/10-azure-unmanaged-sriov.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})
+
+# Install udev rules after updating "/usr/sbin/azure-nvme-id" with installed path $AZURE_NVME_ID_INSTALL_DIR.
+file(READ ${CMAKE_SOURCE_DIR}/udev/80-azure-disk.rules CONTENT)
+string(REPLACE "/usr/sbin/azure-nvme-id" "${AZURE_NVME_ID_INSTALL_DIR}/azure-nvme-id" CONTENT "${CONTENT}")
+file(WRITE ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules ${CONTENT})
+install(FILES ${CMAKE_BINARY_DIR}/udev/80-azure-disk.rules DESTINATION ${UDEV_RULES_INSTALL_DIR})

initramfs/dracut/modules.d/97azure-unmanaged-sriov/module-setup.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+# called by dracut
+check() {
+    return 0
+}
+
+# called by dracut
+depends() {
+    return 0
+}
+
+# called by dracut
+install() {
+    inst_rules 10-azure-unmanaged-sriov.rules
+}

initramfs/initramfs-tools/hooks/azure-unmanaged-sriov

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+PREREQ="udev"
+
+prereqs()
+{
+    echo "${PREREQ}"
+}
+
+case $1 in
+prereqs)
+    prereqs
+    exit 0
+    ;;
+esac
+
+. /usr/share/initramfs-tools/hook-functions
+
+DESTINATION="${DESTDIR}/lib/udev/rules.d/"
+RULES_FILE="/usr/lib/udev/rules.d/10-azure-unmanaged-sriov.rules"
+
+if [ -e "${RULES_FILE}" ]; then
+    mkdir -p "${DESTINATION}"
+    cp -p "${RULES_FILE}" "${DESTINATION}"
+fi

networkd/10-azure-unmanaged-sriov.network

@@ -0,0 +1,10 @@
+# Azure VMs with accelerated networking may have MANA, mlx4, or mlx5 SR-IOV
+# devices which are transparently bonded to a synthetic hv_netvsc device.
+# 10-azure-unmanaged-sriov.rules will mark these devices with
+# AZURE_UNMANAGED_SRIOV=1 so this can configure the devices as unmanaged.
+
+[Match]
+Property=AZURE_UNMANAGED_SRIOV=1
+
+[Link]
+Unmanaged=yes

packaging/fedora/azure-vm-utils.spec

@@ -28,8 +28,9 @@ experience on Azure.
 %ctest
 
 %files
-%{_exec_prefix}/lib/dracut/modules.d/97azure-disk/module-setup.sh
-%{_exec_prefix}/lib/udev/rules.d/80-azure-disk.rules
+%{_exec_prefix}/lib/dracut/*
+%{_exec_prefix}/lib/systemd/*
+%{_exec_prefix}/lib/udev/*
 %{_sbindir}/azure-nvme-id
 %{_mandir}/man8/azure-nvme-id.8.gz
 

packaging/mariner/azure-vm-utils.spec

@@ -31,7 +31,8 @@ experience on Azure.
 %ctest
 
 %files
-%{_libdir}/dracut/modules.d/97azure-disk/module-setup.sh
-%{_libdir}/udev/rules.d/80-azure-disk.rules
+%{_libdir}/dracut/*
+%{_libdir}/systemd/*
+%{_libdir}/udev/*
 %{_sbindir}/azure-nvme-id
 %{_mandir}/man8/azure-nvme-id.8.gz

udev/10-azure-unmanaged-sriov.rules

@@ -0,0 +1,6 @@
+# Azure VMs with accelerated networking may have MANA, mlx4, or mlx5 SR-IOV devices which are transparently bonded to a synthetic
+# hv_netvsc device.  Mark devices with the IFF_SLAVE bit set as unmanaged devices:
+#   AZURE_UNMANAGED_SRIOV=1 for 10-azure-unmanaged-sriov.network
+#   ID_NET_MANAGED_BY=unmanaged for systemd-networkd >= 255
+#   NM_UNMANAGED=1 for NetworkManager
+SUBSYSTEM=="net", ACTION!="remove", DRIVERS=="mana|mlx4_core|mlx5_core", ATTR{flags}=="0x?[89ABCDEF]??", ENV{AZURE_UNMANAGED_SRIOV}="1", ENV{ID_NET_MANAGED_BY}="unmanaged", ENV{NM_UNMANAGED}="1"