From 49e7d0475ea0c362035c32dfded38e0f55c0b560 Mon Sep 17 00:00:00 2001
From: HikariKnight <2557889+HikariKnight@users.noreply.github.com>
Date: Sun, 28 Apr 2024 19:38:06 +0200
Subject: [PATCH] feat: add kvmfr module (looking-glass) and vfio support
 (#1226)

Co-authored-by: m2 <69128853+m2Giles@users.noreply.github.com>
---
 Containerfile                                 |   6 +-
 build_files/dx/build-dx.sh                    |   1 +
 build_files/dx/copr-repos-dx.sh               |   5 +-
 build_files/dx/install-akmods-dx.sh           |   9 ++
 just/bluefin-system.just                      |  94 +++++++++++++++
 .../dx/usr/libexec/bluefin-dx-kvmfr-setup     | 108 ++++++++++++++++++
 6 files changed, 221 insertions(+), 2 deletions(-)
 create mode 100644 build_files/dx/install-akmods-dx.sh
 create mode 100755 system_files/dx/usr/libexec/bluefin-dx-kvmfr-setup

diff --git a/Containerfile b/Containerfile
index a3cc1ec50c5..05631c5b3a7 100644
--- a/Containerfile
+++ b/Containerfile
@@ -45,6 +45,7 @@ ARG IMAGE_NAME="${IMAGE_NAME}"
 ARG IMAGE_VENDOR="${IMAGE_VENDOR}"
 ARG BASE_IMAGE_NAME="${BASE_IMAGE_NAME}"
 ARG IMAGE_FLAVOR="${IMAGE_FLAVOR}"
+ARG AKMODS_FLAVOR="${AKMODS_FLAVOR}"
 ARG FEDORA_MAJOR_VERSION="${FEDORA_MAJOR_VERSION}"
 
 # dx specific files come from the dx directory in this repo
@@ -52,10 +53,13 @@ COPY build_files/dx build_files/shared /tmp/build/
 COPY system_files/dx /
 COPY packages.json /tmp/packages.json
 
+# Copy akmods-extra from ublue
+COPY --from=ghcr.io/ublue-os/akmods-extra:${AKMODS_FLAVOR}-${FEDORA_MAJOR_VERSION} /rpms /tmp/akmods-rpms
+
 # Build, Clean-up, Commit
 RUN bash -c ". /tmp/build/build-dx.sh"  && \
     fc-cache --system-only --really-force --verbose && \
     rm -rf /tmp/* /var/* && \
     mkdir -p /var/tmp && \
     chmod -R 1777 /var/tmp && \
-    ostree container commit
\ No newline at end of file
+    ostree container commit
diff --git a/build_files/dx/build-dx.sh b/build_files/dx/build-dx.sh
index 40c21e23e50..c6e43823a87 100644
--- a/build_files/dx/build-dx.sh
+++ b/build_files/dx/build-dx.sh
@@ -7,6 +7,7 @@ set -ouex pipefail
 sysctl -p
 
 . /tmp/build/copr-repos-dx.sh
+. /tmp/build/install-akmods-dx.sh
 . /tmp/build/packages-dx.sh
 . /tmp/build/image-info.sh
 . /tmp/build/fetch-install-dx.sh
diff --git a/build_files/dx/copr-repos-dx.sh b/build_files/dx/copr-repos-dx.sh
index 27b32cc86d2..b64dcac11c2 100644
--- a/build_files/dx/copr-repos-dx.sh
+++ b/build_files/dx/copr-repos-dx.sh
@@ -12,4 +12,7 @@ wget https://copr.fedorainfracloud.org/coprs/ublue-os/staging/repo/fedora-"${FED
 wget https://copr.fedorainfracloud.org/coprs/karmab/kcli/repo/fedora-"${FEDORA_MAJOR_VERSION}"/karmab-kcli-fedora-"${FEDORA_MAJOR_VERSION}".repo -O /etc/yum.repos.d/karmab-kcli-fedora-"${FEDORA_MAJOR_VERSION}".repo
 
 # Fonts
-wget https://copr.fedorainfracloud.org/coprs/atim/ubuntu-fonts/repo/fedora-"${FEDORA_MAJOR_VERSION}"/atim-ubuntu-fonts-fedora-"${FEDORA_MAJOR_VERSION}".repo -O /etc/yum.repos.d/atim-ubuntu-fonts-fedora-"${FEDORA_MAJOR_VERSION}".repo
\ No newline at end of file
+wget https://copr.fedorainfracloud.org/coprs/atim/ubuntu-fonts/repo/fedora-"${FEDORA_MAJOR_VERSION}"/atim-ubuntu-fonts-fedora-"${FEDORA_MAJOR_VERSION}".repo -O /etc/yum.repos.d/atim-ubuntu-fonts-fedora-"${FEDORA_MAJOR_VERSION}".repo
+
+# Kvmfr module
+wget https://copr.fedorainfracloud.org/coprs/hikariknight/looking-glass-kvmfr/repo/fedora-"${FEDORA_MAJOR_VERSION}"/hikariknight-looking-glass-kvmfr-fedora-"${FEDORA_MAJOR_VERSION}".repo -O /etc/yum.repos.d/hikariknight-looking-glass-kvmfr-fedora-"${FEDORA_MAJOR_VERSION}".repo
diff --git a/build_files/dx/install-akmods-dx.sh b/build_files/dx/install-akmods-dx.sh
new file mode 100644
index 00000000000..3949c139759
--- /dev/null
+++ b/build_files/dx/install-akmods-dx.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+set -ouex pipefail
+
+sed -i 's@enabled=0@enabled=1@g' /etc/yum.repos.d/_copr_ublue-os-akmods.repo
+if [[ "${FEDORA_MAJOR_VERSION}" -ge "39" ]]; then
+    rpm-ostree install \
+        /tmp/akmods-rpms/kmods/*kvmfr*.rpm
+fi
diff --git a/just/bluefin-system.just b/just/bluefin-system.just
index 981f13d6909..9041836fa78 100644
--- a/just/bluefin-system.just
+++ b/just/bluefin-system.just
@@ -161,6 +161,100 @@ dx-group:
     sudo usermod -aG libvirt $USER
     @echo "Logout to use docker, incus-admin, lxd, libvirt"
 
+# Configure system to use vfio and kvmfr
+configure-vfio ACTION="":
+    #!/usr/bin/bash
+    source /usr/lib/ujust/ujust.sh
+    CURRENT_IMAGE=$(rpm-ostree status -b --json | jq -r '.deployments[0]."container-image-reference"')
+    if grep -q "dx" <<< $CURRENT_IMAGE
+    then
+      DEVMODE="enabled"
+    else
+      DEVMODE="disabled"
+    fi
+    if [ "$DEVMODE" == "disabled" ]; then
+      echo 'Please run "ujust devmode" first'
+      exit 0
+    fi
+    OPTION={{ ACTION }}
+    if [ "$OPTION" == "help" ]; then
+      echo "Usage: ujust configure-vfio <option>"
+      echo "  <option>: Specify the quick option to skip the prompt"
+      echo "  Use 'vfio-on' to select Enable VFIO drivers"
+      echo "  Use 'vfio-off' to select Disable VFIO drivers"
+      echo "  Use 'kvmfr' to select Autocreate Looking-Glass shm"
+      exit 0
+    elif [ "$OPTION" == "" ]; then
+      echo "${bold}VFIO and kvmfr Configuration${normal}"
+      echo "This is only used for GPU passthrough of a secondary dGPU."
+      echo "It will enable vfio and configure kvmfr for use with $(Urllink "https://looking-glass.io" "Looking Glass")"
+      echo "If you do not plan to use any of this then press ESC."
+      echo "${bold}NOTE:${normal} Since this is a niche use case, support will be ${b}very limited${n}"
+      OPTION=$(Choose \
+        "Enable VFIO drivers" \
+        "Disable VFIO drivers" \
+        "Enable kvmfr module" \
+      )
+    fi
+    if [[ "${OPTION,,}" =~ (^enable[[:space:]]vfio|vfio-on) ]]; then
+      echo "Enabling VFIO..."
+      VIRT_TEST=$(rpm-ostree kargs)
+      CPU_VENDOR=$(grep "vendor_id" "/proc/cpuinfo" | uniq | awk -F": " '{ print $2 }')
+      VENDOR_KARG="unset"
+      if [[ ${VIRT_TEST} == *kvm.report_ignored_msrs* ]]; then
+        echo 'add_drivers+=" vfio vfio_iommu_type1 vfio-pci "' | sudo tee /etc/dracut.conf.d/vfio.conf
+        rpm-ostree initramfs --enable
+        if [[ ${CPU_VENDOR} == "AuthenticAMD" ]]; then
+          VENDOR_KARG="amd_iommu=on"
+        elif [[ ${CPU_VENDOR} == "GenuineIntel" ]]; then
+          VENDOR_KARG="intel_iommu=on"  
+        fi
+        if [[ ${VENDOR_KARG} == "unset" ]]; then
+          echo "Failed to get CPU vendor, exiting..."
+          exit 1
+        else
+          rpm-ostree kargs \
+            --append-if-missing="${VENDOR_KARG}" \
+            --append-if-missing="iommu=pt" \
+            --append-if-missing="rd.driver.pre=vfio_pci" \
+            --append-if-missing="vfio_pci.disable_vga=1"
+          echo "VFIO will be enabled on next boot, make sure you enable IOMMU, VT-d or AMD-v in your BIOS!"
+          echo "Please understand that since this is such a niche use case, support will be very limited!"
+          echo "To add your unused/second GPU device ids to the vfio driver by running"
+          echo 'rpm-ostree kargs --append-if-missing="vfio-pci.ids=xxxx:yyyy,xxxx:yyzz"'
+          echo "NOTE: Your second GPU will not be usable by the host after you do this!"
+        fi
+      fi
+    elif [[ "${OPTION,,}" =~ (^disable[[:space:]]vfio|vfio-off) ]]; then
+      echo ""
+      echo "Make sure you have ${b}disabled autostart of all VMs using VFIO${n} before continuing!"
+      CONFIRM=$(Choose Cancel Continue)
+      if [ "$CONFIRM" == "Continue" ]; then
+        echo "Disabling VFIO..."
+        VFIO_IDS="$(rpm-ostree kargs | sed -E 's/.+(vfio_pci.ids=.+\s)/\1/' | awk '{ print $1 }' | grep vfio_pci.ids)"
+        VFIO_IDS_KARG=""
+        if [ -n "$VFIO_IDS" ]; then
+          echo "Found VFIO ids in kargs, adding the below line to removal list"
+          echo "$VFIO_IDS"
+          VFIO_IDS_KARG="--delete-if-present=\"$VFIO_IDS\""
+        fi
+        echo "Removing dracut modules"
+        sudo rm /etc/dracut.conf.d/vfio.conf
+        rpm-ostree initramfs --enable
+        rpm-ostree kargs \
+        --delete-if-present="iommu=pt" \
+        --delete-if-present="iommu=on" \
+        --delete-if-present="amd_iommu=on" \
+        --delete-if-present="intel_iommu=on" \
+        --delete-if-present="rd.driver.pre=vfio_pci" \
+        --delete-if-present="vfio_pci.disable_vga=1" \
+        --delete-if-present="vfio_pci.disable_vga=0" \
+        $VFIO_IDS_KARG
+      fi
+    elif [[ "${OPTION,,}" =~ kvmfr ]]; then
+      sudo /usr/libexec/bluefin-dx-kvmfr-setup
+    fi
+
 # Install system flatpaks for rebasers
 [private]
 install-system-flatpaks:
diff --git a/system_files/dx/usr/libexec/bluefin-dx-kvmfr-setup b/system_files/dx/usr/libexec/bluefin-dx-kvmfr-setup
new file mode 100755
index 00000000000..f711cee379d
--- /dev/null
+++ b/system_files/dx/usr/libexec/bluefin-dx-kvmfr-setup
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+source /usr/lib/ujust/ujust.sh
+
+# Required disclaimer and where to report issues first
+echo "$(Urllink "https://looking-glass.io/docs/rc/ivshmem_kvmfr/#libvirt" "This module") along with $(Urllink "https://looking-glass.io" "Looking Glass") is very experimental and not recommended for production use!"
+echo "The ublue team packages the kvmfr module only because it has to be supplied with the system image while using an atomic desktop."
+echo "If you do plan to use Looking Glass, please $(Urllink "https://universal-blue.discourse.group/docs?topic=956" "follow the guide here") on how to compile it for your system."
+echo "To use the kvmfr module after enabling it, just add and edit the xml for libvirt from the documentation in the first link."
+echo "Since we package the kvmfr module please open kvmfr related issues you have on Bluefin or Aurora and tag me"
+echo "in the $(Urllink "https://discord.gg/WEu6BdFEtp" "Universal Blue Discord") or the $(Urllink "https://github.com/ublue-os/bluefin/issues" "Bluefin Github issue tracker")."
+echo "~ @HikariKnight"
+
+CONFIRM=$(Choose Ok Cancel)
+if [ "$CONFIRM" == "Cancel" ]; then
+    exit 0
+fi
+
+# Add kvmfr to dracut so that it's modprobe file can be used
+echo ""
+echo "Setting up kvmfr module so it loads next boot"
+sudo bash -c 'cat << KVMFR_DRACUT > /etc/dracut.conf.d/kvmfr.conf
+install_items+=" /etc/modprobe.d/kvmfr.conf "
+KVMFR_DRACUT'
+
+# Add kvmfr modprobe file following upstream documentation
+sudo bash -c "cat << KVMFR_MODPROBE > /etc/modprobe.d/kvmfr.conf
+options kvmfr static_size_mb=128
+KVMFR_MODPROBE"
+
+# Add upstream udev rule for kvmfr, adjusted for fedora systems
+echo "Adding udev rule for /dev/kvmfr0"
+sudo bash -c 'cat << KVMFR_UDEV > /etc/udev/rules.d/99-kvmfr.rules
+SUBSYSTEM=="kvmfr", OWNER="'$USER'", GROUP="qemu", MODE="0660"
+KVMFR_UDEV'
+
+# Add /dev/kvmfr0 to qemu cgroup device acl list
+echo "Adding /dev/kvmfr0 to qemu cgroup_device_acl"
+# This is not ideal and if someone has a better way to do this without perl, you are welcome to change it
+sudo perl -0777 -pi -e 's/
+#cgroup_device_acl = \[
+#    "\/dev\/null", "\/dev\/full", "\/dev\/zero",
+#    "\/dev\/random", "\/dev\/urandom",
+#    "\/dev\/ptmx", "\/dev\/kvm",
+#    "\/dev\/userfaultfd"
+#\]
+/
+cgroup_device_acl = \[
+    "\/dev\/null", "\/dev\/full", "\/dev\/zero",
+    "\/dev\/random", "\/dev\/urandom",
+    "\/dev\/ptmx", "\/dev\/kvm",
+    "\/dev\/userfaultfd", "\/dev\/kvmfr0"
+\]
+/' /etc/libvirt/qemu.conf
+
+# Add SELinux context record for /dev/kvmfr0 (for simplicity we use the same one that was used for the shm)
+echo "Adding SELinux context record for /dev/kvmfr0"
+sudo semanage fcontext -a -t svirt_tmpfs_t /dev/kvmfr0
+
+# Create type enforcement for /dev/kvmfr0 as there is no existing way to access kvmfr using virt context
+echo "Adding SELinux access rules for /dev/kvmfr0"
+if [ ! -d "$HOME/.config/selinux_te/mod" ]; then
+    mkdir -p "$HOME/.config/selinux_te/mod"
+fi
+if [ ! -d "$HOME/.config/selinux_te/pp" ]; then
+    mkdir -p "$HOME/.config/selinux_te/pp"
+fi
+bash -c "cat << KVMFR_SELINUX > $HOME/.config/selinux_te/kvmfr.te
+module kvmfr 1.0;
+require {
+    type device_t;
+    type svirt_t;
+    class chr_file { open read write map };
+}
+#============= svirt_t ==============
+allow svirt_t device_t:chr_file { open read write map };
+KVMFR_SELINUX"
+
+# Tell user what type enforcement we made and how it looks like
+echo "This is the type enforcement we wrote for SELinux and you can find it in $HOME/.config/selinux_te/kvmfr.te"
+echo "#======= start of kvmfr.te ======="
+cat "$HOME/.config/selinux_te/kvmfr.te"
+echo "#======== end of kvmfr.te ========"
+checkmodule -M -m -o "$HOME/.config/selinux_te/mod/kvmfr.mod" "$HOME/.config/selinux_te/kvmfr.te"
+semodule_package -o "$HOME/.config/selinux_te/pp/kvmfr.pp" -m "$HOME/.config/selinux_te/mod/kvmfr.mod"
+sudo semodule -i "$HOME/.config/selinux_te/pp/kvmfr.pp"
+
+# Load kvmfr module into currently booted system
+echo "Loading kvmfr module so you do not have to reboot to use it the first time"
+sudo modprobe kvmfr static_size_mb=128
+sudo chown $USER:qemu /dev/kvmfr0
+
+# Final message and regenerate initramfs so kvmfr loads next boot
+echo ""
+echo "Kvmfr0 $(Urllink "https://looking-glass.io/docs/rc/install_libvirt/#determining-memory" "static size is set to 128mb by default")"
+echo "this will work with up to 4K SDR resolutiion, as most dummy plugs go up to 4K"
+echo "some games will try use the adapters max resolution on first boot and cause issues if the value is too low."
+echo "Most ghost display adapters max out at 4k, hence the default value of 128mb."
+echo ""
+echo "If you need to change it to a different value"
+echo "you can do that in /etc/modprobe.d/kvmfr.conf"
+echo "$(Urllink "https://looking-glass.io/docs/rc/ivshmem_kvmfr/#libvirt" "Please read official documentation for kvmfr for how to use it")"
+echo ""
+echo "Press OK to start the process of regenerating your initramfs, this will take a long time"
+echo "and there is no good way to track progress for it, if anything is wrong it will error out."
+echo "${b}NOTE: You can start using kvmfr right now without rebooting, but you will need to regenerate initramfs for it to auto load next boot.${n}"
+
+CONFIRM=$(Choose OK)
+rpm-ostree initramfs --enable