feat: Add support for using nvidia GPUs inside containers (ublue-os#43)

* feat: Add support for using nvidia GPUs inside containers

* feat: Add container runtime config files into general add-ons package

* feat: Add config.toml for rootless container support

* fix: Keep backup of original nvidia container runtime config.toml

Containerfile

@@ -43,14 +43,24 @@ RUN NVIDIA_PACKAGE_NAME="$(cat /tmp/nvidia-package-name.txt)" \
     || \
         (cat /var/cache/akmods/${NVIDIA_PACKAGE_NAME}/${NVIDIA_VERSION}-for-${KERNEL_VERSION}.failed.log && exit 1)
 
-ADD akmods-nvidia-key.spec /tmp/akmods-nvidia-key/akmods-nvidia-key.spec
+ADD ublue-os-nvidia-addons.spec /tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec
 
-RUN install -D /etc/pki/akmods/certs/public_key.der /tmp/akmods-nvidia-key/rpmbuild/SOURCES/public_key.der
+ADD https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo \
+    /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo
+
+RUN sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo
+
+ADD files/etc/nvidia-container-runtime/config-rootless.toml \
+    /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/config-rootless.toml
+ADD https://raw.githubusercontent.com/NVIDIA/dgx-selinux/master/bin/RHEL9/nvidia-container.pp \
+    /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container.pp
+
+RUN install -D /etc/pki/akmods/certs/public_key.der /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/public_key.der
 
 RUN rpmbuild -ba \
-    --define '_topdir /tmp/akmods-nvidia-key/rpmbuild' \
+    --define '_topdir /tmp/ublue-os-nvidia-addons/rpmbuild' \
     --define '%_tmppath %{_topdir}/tmp' \
-    /tmp/akmods-nvidia-key/akmods-nvidia-key.spec
+    /tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec
 
 
 RUN cp /tmp/nvidia-package-name.txt /var/cache/akmods/nvidia-package-name.txt
@@ -60,8 +70,13 @@ RUN rpm -q "xorg-x11-drv-$(cat /tmp/nvidia-package-name.txt)" \
 
 FROM ${BASE_IMAGE}:${FEDORA_MAJOR_VERSION}
 
-COPY --from=builder /var/cache/akmods      /tmp/akmods
-COPY --from=builder /tmp/akmods-nvidia-key /tmp/akmods-nvidia-key
+COPY --from=builder /var/cache/akmods /tmp/akmods
+COPY --from=builder /tmp/ublue-os-nvidia-addons /tmp/ublue-os-nvidia-addons
+
+RUN sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo
+
+RUN install -D /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo \
+    /etc/yum.repos.d/nvidia-container-runtime.repo
 
 RUN KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')" \
     NVIDIA_FULL_VERSION="$(cat /tmp/akmods/nvidia-full-version.txt)" \
@@ -71,13 +86,18 @@ RUN KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}
             https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm \
             https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm \
     && \
-        sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/{fedora-{cisco-openh264,modular,updates-modular},rpmfusion-free{,-updates}}.repo \
+        sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-free{,-updates}.repo \
     && \
         rpm-ostree install \
             xorg-x11-drv-${NVIDIA_PACKAGE_NAME}-{,cuda-,devel-,kmodsrc-,power-}${NVIDIA_FULL_VERSION} \
-            kernel-devel-${KERNEL_VERSION} \
+            kernel-devel-${KERNEL_VERSION} nvidia-container-toolkit \
             "/tmp/akmods/${NVIDIA_PACKAGE_NAME}/kmod-${NVIDIA_PACKAGE_NAME}-${KERNEL_VERSION}-${NVIDIA_FULL_VERSION#*:}.rpm" \
-            /tmp/akmods-nvidia-key/rpmbuild/RPMS/noarch/akmods-nvidia-key-*.rpm \
+            /tmp/ublue-os-nvidia-addons/rpmbuild/RPMS/noarch/ublue-os-nvidia-addons-*.rpm \
+    && \
+        mv /etc/nvidia-container-runtime/config.toml{,.orig} && \
+        cp /etc/nvidia-container-runtime/config{-rootless,}.toml \
+    && \
+        semodule --verbose --install /usr/share/selinux/packages/nvidia-container.pp \
     && \
         sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-nonfree{,-updates}.repo \
     && \

README.md

@@ -112,6 +112,19 @@ $ podman build \
     --tag build-test:latest
 ```
 
+## Using Nvidia GPUs in containers
+
+[There is support for enabling Nvidia GPUs in containers](https://www.redhat.com/en/blog/how-use-gpus-containers-bare-metal-rhel-8). This can can be verified by running the following:
+
+```
+$ podman run \
+    --user 1000:1000 \
+    --security-opt=no-new-privileges \
+    --cap-drop=ALL \
+    --security-opt label=type:nvidia_container_t  \
+    docker.io/mirrorgooglecontainers/cuda-vector-add:v0.1
+```
+
 ## Acknowledgements
 
 Thanks to Alex Diaz for advice, and who got this working first, check out this repo:

files/etc/nvidia-container-runtime/config-rootless.toml

@@ -0,0 +1,34 @@
+disable-require = false
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+no-cgroups = true
+#user = "root:video"
+ldconfig = "@/sbin/ldconfig"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"
+debug = "~/.local/nvidia-container-runtime.log"
+log-level = "info"
+
+# Specify the runtimes to consider. This list is processed in order and the PATH
+# searched for matching executables unless the entry is an absolute path.
+runtimes = [
+    "docker-runc",
+    "runc",
+]
+
+mode = "auto"
+
+    [nvidia-container-runtime.modes.csv]
+
+    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

ublue-os-nvidia-addons.spec

@@ -0,0 +1,47 @@
+Name:           ublue-os-nvidia-addons
+Version:        0.2
+Release:        1%{?dist}
+Summary:        Additional files for nvidia driver support
+
+License:        MIT
+URL:            https://github.com/ublue-os/nvidia
+
+BuildArch:      noarch
+Supplements:    mokutil policycoreutils
+
+Source0:        public_key.der
+Source1:        nvidia-container-runtime.repo
+Source2:        config-rootless.toml
+Source3:        nvidia-container.pp
+
+%description
+Adds various runtime files for nvidia support. These include a key for importing with mokutil to enable secure boot for nvidia kernel modules
+
+%prep
+%setup -q -c -T
+
+
+%install
+# Have different name for *.der in case kmodgenca is needed for creating more keys
+install -Dm0644 %{SOURCE0} %{buildroot}%{_sysconfdir}/pki/akmods/certs/akmods-nvidia.der
+install -Dm0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
+install -Dm0644 %{SOURCE2} %{buildroot}%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
+install -Dm0644 %{SOURCE3} %{buildroot}%{_datadir}/selinux/packages/nvidia-container.pp
+
+sed -i 's@enabled=1@enabled=0@g' %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
+
+%files
+%attr(0644,root,root) %{_sysconfdir}/pki/akmods/certs/akmods-nvidia.der
+%attr(0644,root,root) %{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
+%attr(0644,root,root) %{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
+%attr(0644,root,root) %{_datadir}/selinux/packages/nvidia-container.pp
+
+%changelog
+* Thu Feb 16 2023 Joshua Stone <[email protected]> - 0.2
+- Add nvidia-container-runtime repo
+- Add nvidia-container-runtime selinux policy file
+- Re-purpose into a general-purpose add-on package
+- Update URL to point to ublue-os project
+
+* Fri Feb 03 2023 Joshua Stone <[email protected]> - 0.1
+- Add key for enrolling kernel modules in alpha builds