feat: nvgpu guest (#349)

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

build/Makefile

@@ -76,6 +76,13 @@ INFINITY_REV    ?= e6c85d841f25e21393a7ea47fdef286318915d8c
 INFINITY_SERIES := patches/cachyos/tuned-eevdf/$(KERNEL_REF:cachyos-%=%)
 INFINITY_WORK   := $(OUTPUT_DIR)/infinity-sched
 
+# The guest half of NVIDIA forwarding, built into the kernel. A branch, so each
+# kernel build takes the driver as it is now; set NVGPU_REF to a tag or commit
+# to pin it. The commit built is recorded as $(KERNEL_OUTPUT).nvgpu-rev.
+NVGPU_GIT  ?= https://github.com/nestrilabs/virtio-nvgpu.git
+NVGPU_REF  ?= dev
+NVGPU_WORK := $(OUTPUT_DIR)/virtio-nvgpu
+
 ifneq ($(KERNEL_INFINITY),)
 KERNEL_SRC    ?= $(OUTPUT_DIR)/kernel-infinity
 KERNEL_OUTPUT := $(OUTPUT_DIR)/vmlinux-infinity
@@ -136,6 +143,7 @@ kernel:
 		KERNEL_INFINITY=$(KERNEL_INFINITY) INFINITY_GIT=$(INFINITY_GIT) \
 		INFINITY_REV=$(INFINITY_REV) INFINITY_SERIES=$(INFINITY_SERIES) \
 		INFINITY_WORK=$(INFINITY_WORK) \
+		NVGPU_GIT=$(NVGPU_GIT) NVGPU_REF=$(NVGPU_REF) NVGPU_WORK=$(NVGPU_WORK) \
 		bash scripts/kernel-build.sh
 
 # Leaves $(PROTON_WORK) and the kernel alone: one is hours of build and ccache,
@@ -143,11 +151,11 @@ kernel:
 # proton-clean and kernel-clean are the ones that drop them.
 clean:
 	find $(OUTPUT_DIR) -mindepth 1 -maxdepth 1 ! -name proton ! -name 'kernel*' \
-		! -name 'vmlinux*' ! -name infinity-sched -exec rm -rf {} + 2>/dev/null || true
+		! -name 'vmlinux*' ! -name infinity-sched ! -name virtio-nvgpu -exec rm -rf {} + 2>/dev/null || true
 
 kernel-clean:
-	rm -rf $(OUTPUT_DIR)/kernel $(OUTPUT_DIR)/kernel-infinity $(INFINITY_WORK) \
-		$(OUTPUT_DIR)/vmlinux $(OUTPUT_DIR)/vmlinux-infinity
+	rm -rf $(OUTPUT_DIR)/kernel $(OUTPUT_DIR)/kernel-infinity $(INFINITY_WORK) $(NVGPU_WORK) \
+		$(OUTPUT_DIR)/vmlinux $(OUTPUT_DIR)/vmlinux-infinity $(OUTPUT_DIR)/vmlinux*.nvgpu-rev
 
 proton-clean:
 	rm -rf $(PROTON_WORK)

build/README.md

@@ -120,6 +120,33 @@ and this guest has `CONFIG_MODULES` off and no `/lib/modules` at all.
 - The tree is off the pinned ref (a bisect, a local patch)? The build warns
   and builds what is there rather than checking the ref out over your work.
 
+### NVIDIA: the forwarding driver
+
+```sh
+make kernel                          # fetches virtio-nvgpu's dev branch and builds its driver in
+make NVGPU_REF=<tag-or-commit> kernel
+```
+
+On an NVIDIA host the guest has no GPU of its own. It gets a virtio device
+that carries the NVIDIA driver's ioctls to the host, and it runs NVIDIA's own
+user-mode libraries against it. The guest half of that is a kernel driver from
+[virtio-nvgpu](https://github.com/nestrilabs/virtio-nvgpu), which every build
+fetches into `output/virtio-nvgpu` and builds into the kernel as
+`CONFIG_VIRTIO_GPU_NV`. It has to be built in, because this kernel cannot load
+modules. The driver's parameters therefore go on the command line as
+`virtio_gpu_nv.<name>=`.
+
+- **`NVGPU_REF` is a branch by default**, so a build takes the driver as it is
+  that day. The commit that went in is written to `output/vmlinux.nvgpu-rev`,
+  because a branch name does not tell you which driver a given `vmlinux`
+  contains. If the fetch fails, the build uses the checkout it already has and
+  warns that it is doing so.
+- An AMD or Intel host is unaffected. The driver binds only to the forwarding
+  device, and nothing offers that device to those guests.
+- The NVIDIA libraries are **not** part of the image. They must be the same
+  build as the host's kernel module, so the host shares its own copy with the
+  guest at run time.
+
 ### Experimental: the Infinity scheduler
 
 ```sh

build/kernel/nestri.fragment

@@ -106,6 +106,7 @@ CONFIG_VIRTIO_VSOCKETS=y
 CONFIG_VIRTIO_BALLOON=y
 CONFIG_DRM=y
 CONFIG_DRM_VIRTIO_GPU=y
+CONFIG_VIRTIO_GPU_NV=y       # NVIDIA hosts: the ioctl forwarder, from virtio-nvgpu
 
 # ── Filesystems ──────────────────────────────────────────
 CONFIG_EXT4_FS=y             # output/rootfs.ext4, and a box's writable install layer

build/scripts/kernel-build.sh

@@ -13,6 +13,9 @@ set -euo pipefail
 : "${KERNEL_REF:?}"
 : "${KERNEL_SRC:?}"
 : "${KERNEL_OUTPUT:?}"
+: "${NVGPU_GIT:?}"
+: "${NVGPU_REF:?}"
+: "${NVGPU_WORK:?}"
 
 JOBS="${JOBS:-$(nproc)}"
 KERNEL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../kernel" && pwd)"
@@ -22,6 +25,8 @@ SEED="${KERNEL_DIR}/base.config"
 # Resolved now, because everything below runs from inside the tree.
 mkdir -p "$(dirname "${KERNEL_OUTPUT}")"
 KERNEL_OUTPUT="$(cd "$(dirname "${KERNEL_OUTPUT}")" && pwd)/$(basename "${KERNEL_OUTPUT}")"
+mkdir -p "${NVGPU_WORK}"
+NVGPU_WORK="$(cd "${NVGPU_WORK}" && pwd)"
 
 # ── Source ──────────────────────────────────────────────
 if [[ ! -f "${KERNEL_SRC}/Makefile" ]]; then
@@ -93,6 +98,78 @@ if [[ -n "${KERNEL_INFINITY:-}" ]]; then
     fi
 fi
 
+# ── NVIDIA forwarding driver ────────────────────────────
+# An NVIDIA host gives the guest no GPU, only a virtio device that carries the
+# NVIDIA driver's own ioctls to the host; the guest half of that is a kernel
+# driver from virtio-nvgpu. This kernel has CONFIG_MODULES off, so the driver
+# is built in rather than loaded, and its module parameters become
+# `virtio_gpu_nv.<name>=` on the command line.
+#
+# It follows NVGPU_REF, a branch by default, so every build takes the driver as
+# it is now. The commit actually built is written beside the kernel, because a
+# branch name says nothing about which driver a given vmlinux carries.
+if [[ ! -d "${NVGPU_WORK}/.git" ]]; then
+    git init -q "${NVGPU_WORK}"
+fi
+if git -C "${NVGPU_WORK}" fetch -q --depth=1 "${NVGPU_GIT}" "${NVGPU_REF}"; then
+    git -C "${NVGPU_WORK}" checkout -q --detach FETCH_HEAD
+elif git -C "${NVGPU_WORK}" rev-parse -q --verify HEAD >/dev/null; then
+    # Not fatal: a kernel should still build offline. It is loud because the
+    # driver built is then older than NVGPU_REF, and nothing else says so.
+    echo "kernel: could not fetch ${NVGPU_REF} from ${NVGPU_GIT}; building the driver already checked out" >&2
+else
+    echo "kernel: could not fetch ${NVGPU_REF} from ${NVGPU_GIT}, and there is no earlier checkout" >&2
+    exit 1
+fi
+nvgpu_rev="$(git -C "${NVGPU_WORK}" rev-parse HEAD)"
+echo "kernel: virtio-nvgpu driver at ${nvgpu_rev}"
+
+# Copied in, not linked: kbuild writes its objects beside the sources, and a
+# symlinked directory would put them in the checkout. A file is only copied
+# when it differs, so an unchanged driver does not relink the kernel.
+nvgpu_src="${NVGPU_WORK}/driver"
+# Under drm/, not virtio/, although it binds a virtio device. Built in, a
+# driver initialises in link order, and drivers/virtio links before
+# drivers/gpu: there, its probe runs before the DRM core exists and its render
+# node fails with "DRM core is not initialized". Loaded as a module, the order
+# never came up.
+nvgpu_dst="drivers/gpu/drm/nvgpu"
+mkdir -p "${nvgpu_dst}/gen"
+for dst in "${nvgpu_dst}"/*.[ch] "${nvgpu_dst}"/gen/*.h; do
+    [[ -e "${dst}" && ! -e "${nvgpu_src}/${dst#"${nvgpu_dst}"/}" ]] && rm -f "${dst}"
+done
+for src in "${nvgpu_src}"/*.[ch] "${nvgpu_src}"/gen/*.h "${nvgpu_src}/Kconfig"; do
+    dst="${nvgpu_dst}/${src#"${nvgpu_src}"/}"
+    cmp -s "${src}" "${dst}" || cp "${src}" "${dst}"
+done
+
+# The driver's Makefile also builds out of tree, against a KDIR. Kbuild prefers
+# a Kbuild file over a Makefile, so the in-tree half is taken from it alone:
+# the object list, not the out-of-tree rules around it.
+kbuild="$(grep -E '^obj-\$\(CONFIG_VIRTIO_GPU_NV\)' "${nvgpu_src}/Makefile" || true)"
+if [[ -z "${kbuild}" ]]; then
+    echo "kernel: ${nvgpu_src}/Makefile has no obj-\$(CONFIG_VIRTIO_GPU_NV) line to build it in tree by" >&2
+    exit 1
+fi
+[[ "$(cat "${nvgpu_dst}/Kbuild" 2>/dev/null)" == "${kbuild}" ]] || printf '%s\n' "${kbuild}" > "${nvgpu_dst}/Kbuild"
+
+# A tree this script hooked into drivers/virtio before the move above still
+# has that hook, and building both would define the driver twice.
+sed -i '/^source "drivers\/virtio\/nvgpu\/Kconfig"$/d' drivers/virtio/Kconfig
+sed -i '/^obj-\$(CONFIG_VIRTIO_GPU_NV) += nvgpu\/$/d' drivers/virtio/Makefile
+rm -rf drivers/virtio/nvgpu
+
+# Hooked in beside DRM's own virtio-gpu, once. Inside the DRM menu's `if DRM`,
+# which is also what makes it depend on DRM. If this is ever skipped, the
+# fragment check below catches it: CONFIG_VIRTIO_GPU_NV cannot be set without
+# its Kconfig.
+grep -qx 'source "drivers/gpu/drm/nvgpu/Kconfig"' drivers/gpu/drm/Kconfig \
+    || sed -i '\|^source "drivers/gpu/drm/virtio/Kconfig"$|a source "drivers/gpu/drm/nvgpu/Kconfig"' \
+        drivers/gpu/drm/Kconfig
+grep -qx 'obj-$(CONFIG_VIRTIO_GPU_NV) += nvgpu/' drivers/gpu/drm/Makefile \
+    || sed -i '/^obj-\$(CONFIG_DRM_VIRTIO_GPU) += virtio\/$/a obj-$(CONFIG_VIRTIO_GPU_NV) += nvgpu/' \
+        drivers/gpu/drm/Makefile
+
 # ── Config ──────────────────────────────────────────────
 # A fresh tree has no .config. The seed is a known-good minimal config that
 # olddefconfig migrates to whatever version the tree is at; it only saves a
@@ -152,4 +229,5 @@ fi
 make -j"${JOBS}" "${make_args[@]}" vmlinux
 
 cp vmlinux "${KERNEL_OUTPUT}"
+echo "${nvgpu_rev}" > "${KERNEL_OUTPUT}.nvgpu-rev"
 echo "kernel: installed ${KERNEL_OUTPUT} ($(numfmt --to=iec "$(stat -c %s "${KERNEL_OUTPUT}")"))"