From 291fabd145f73db81ffdb8a134744745099eb656 Mon Sep 17 00:00:00 2001 From: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:17:00 +0300 Subject: [PATCH] feat: nvgpu guest (#349) Co-authored-by: DatCaptainHorse Co-authored-by: Claude Opus 5.5 --- build/Makefile | 14 +++++-- build/README.md | 27 ++++++++++++ build/kernel/nestri.fragment | 1 + build/scripts/kernel-build.sh | 78 +++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 3 deletions(-) diff --git a/build/Makefile b/build/Makefile index b6a42ff0..237ecae2 100644 --- a/build/Makefile +++ b/build/Makefile @@ -76,6 +76,13 @@ INFINITY_REV ?= e6c85d841f25e21393a7ea47fdef286318915d8c INFINITY_SERIES := patches/cachyos/tuned-eevdf/$(KERNEL_REF:cachyos-%=%) INFINITY_WORK := $(OUTPUT_DIR)/infinity-sched +# The guest half of NVIDIA forwarding, built into the kernel. A branch, so each +# kernel build takes the driver as it is now; set NVGPU_REF to a tag or commit +# to pin it. The commit built is recorded as $(KERNEL_OUTPUT).nvgpu-rev. +NVGPU_GIT ?= https://github.com/nestrilabs/virtio-nvgpu.git +NVGPU_REF ?= dev +NVGPU_WORK := $(OUTPUT_DIR)/virtio-nvgpu + ifneq ($(KERNEL_INFINITY),) KERNEL_SRC ?= $(OUTPUT_DIR)/kernel-infinity KERNEL_OUTPUT := $(OUTPUT_DIR)/vmlinux-infinity @@ -136,6 +143,7 @@ kernel: KERNEL_INFINITY=$(KERNEL_INFINITY) INFINITY_GIT=$(INFINITY_GIT) \ INFINITY_REV=$(INFINITY_REV) INFINITY_SERIES=$(INFINITY_SERIES) \ INFINITY_WORK=$(INFINITY_WORK) \ + NVGPU_GIT=$(NVGPU_GIT) NVGPU_REF=$(NVGPU_REF) NVGPU_WORK=$(NVGPU_WORK) \ bash scripts/kernel-build.sh # Leaves $(PROTON_WORK) and the kernel alone: one is hours of build and ccache, @@ -143,11 +151,11 @@ kernel: # proton-clean and kernel-clean are the ones that drop them. clean: find $(OUTPUT_DIR) -mindepth 1 -maxdepth 1 ! -name proton ! -name 'kernel*' \ - ! -name 'vmlinux*' ! -name infinity-sched -exec rm -rf {} + 2>/dev/null || true + ! -name 'vmlinux*' ! -name infinity-sched ! -name virtio-nvgpu -exec rm -rf {} + 2>/dev/null || true kernel-clean: - rm -rf $(OUTPUT_DIR)/kernel $(OUTPUT_DIR)/kernel-infinity $(INFINITY_WORK) \ - $(OUTPUT_DIR)/vmlinux $(OUTPUT_DIR)/vmlinux-infinity + rm -rf $(OUTPUT_DIR)/kernel $(OUTPUT_DIR)/kernel-infinity $(INFINITY_WORK) $(NVGPU_WORK) \ + $(OUTPUT_DIR)/vmlinux $(OUTPUT_DIR)/vmlinux-infinity $(OUTPUT_DIR)/vmlinux*.nvgpu-rev proton-clean: rm -rf $(PROTON_WORK) diff --git a/build/README.md b/build/README.md index 561259b4..d569a1b0 100644 --- a/build/README.md +++ b/build/README.md @@ -120,6 +120,33 @@ and this guest has `CONFIG_MODULES` off and no `/lib/modules` at all. - The tree is off the pinned ref (a bisect, a local patch)? The build warns and builds what is there rather than checking the ref out over your work. +### NVIDIA: the forwarding driver + +```sh +make kernel # fetches virtio-nvgpu's dev branch and builds its driver in +make NVGPU_REF= kernel +``` + +On an NVIDIA host the guest has no GPU of its own. It gets a virtio device +that carries the NVIDIA driver's ioctls to the host, and it runs NVIDIA's own +user-mode libraries against it. The guest half of that is a kernel driver from +[virtio-nvgpu](https://github.com/nestrilabs/virtio-nvgpu), which every build +fetches into `output/virtio-nvgpu` and builds into the kernel as +`CONFIG_VIRTIO_GPU_NV`. It has to be built in, because this kernel cannot load +modules. The driver's parameters therefore go on the command line as +`virtio_gpu_nv.=`. + +- **`NVGPU_REF` is a branch by default**, so a build takes the driver as it is + that day. The commit that went in is written to `output/vmlinux.nvgpu-rev`, + because a branch name does not tell you which driver a given `vmlinux` + contains. If the fetch fails, the build uses the checkout it already has and + warns that it is doing so. +- An AMD or Intel host is unaffected. The driver binds only to the forwarding + device, and nothing offers that device to those guests. +- The NVIDIA libraries are **not** part of the image. They must be the same + build as the host's kernel module, so the host shares its own copy with the + guest at run time. + ### Experimental: the Infinity scheduler ```sh diff --git a/build/kernel/nestri.fragment b/build/kernel/nestri.fragment index 2e488794..5b6b573a 100644 --- a/build/kernel/nestri.fragment +++ b/build/kernel/nestri.fragment @@ -106,6 +106,7 @@ CONFIG_VIRTIO_VSOCKETS=y CONFIG_VIRTIO_BALLOON=y CONFIG_DRM=y CONFIG_DRM_VIRTIO_GPU=y +CONFIG_VIRTIO_GPU_NV=y # NVIDIA hosts: the ioctl forwarder, from virtio-nvgpu # ── Filesystems ────────────────────────────────────────── CONFIG_EXT4_FS=y # output/rootfs.ext4, and a box's writable install layer diff --git a/build/scripts/kernel-build.sh b/build/scripts/kernel-build.sh index 36e31ebe..7c0ee9c2 100644 --- a/build/scripts/kernel-build.sh +++ b/build/scripts/kernel-build.sh @@ -13,6 +13,9 @@ set -euo pipefail : "${KERNEL_REF:?}" : "${KERNEL_SRC:?}" : "${KERNEL_OUTPUT:?}" +: "${NVGPU_GIT:?}" +: "${NVGPU_REF:?}" +: "${NVGPU_WORK:?}" JOBS="${JOBS:-$(nproc)}" KERNEL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../kernel" && pwd)" @@ -22,6 +25,8 @@ SEED="${KERNEL_DIR}/base.config" # Resolved now, because everything below runs from inside the tree. mkdir -p "$(dirname "${KERNEL_OUTPUT}")" KERNEL_OUTPUT="$(cd "$(dirname "${KERNEL_OUTPUT}")" && pwd)/$(basename "${KERNEL_OUTPUT}")" +mkdir -p "${NVGPU_WORK}" +NVGPU_WORK="$(cd "${NVGPU_WORK}" && pwd)" # ── Source ────────────────────────────────────────────── if [[ ! -f "${KERNEL_SRC}/Makefile" ]]; then @@ -93,6 +98,78 @@ if [[ -n "${KERNEL_INFINITY:-}" ]]; then fi fi +# ── NVIDIA forwarding driver ──────────────────────────── +# An NVIDIA host gives the guest no GPU, only a virtio device that carries the +# NVIDIA driver's own ioctls to the host; the guest half of that is a kernel +# driver from virtio-nvgpu. This kernel has CONFIG_MODULES off, so the driver +# is built in rather than loaded, and its module parameters become +# `virtio_gpu_nv.=` on the command line. +# +# It follows NVGPU_REF, a branch by default, so every build takes the driver as +# it is now. The commit actually built is written beside the kernel, because a +# branch name says nothing about which driver a given vmlinux carries. +if [[ ! -d "${NVGPU_WORK}/.git" ]]; then + git init -q "${NVGPU_WORK}" +fi +if git -C "${NVGPU_WORK}" fetch -q --depth=1 "${NVGPU_GIT}" "${NVGPU_REF}"; then + git -C "${NVGPU_WORK}" checkout -q --detach FETCH_HEAD +elif git -C "${NVGPU_WORK}" rev-parse -q --verify HEAD >/dev/null; then + # Not fatal: a kernel should still build offline. It is loud because the + # driver built is then older than NVGPU_REF, and nothing else says so. + echo "kernel: could not fetch ${NVGPU_REF} from ${NVGPU_GIT}; building the driver already checked out" >&2 +else + echo "kernel: could not fetch ${NVGPU_REF} from ${NVGPU_GIT}, and there is no earlier checkout" >&2 + exit 1 +fi +nvgpu_rev="$(git -C "${NVGPU_WORK}" rev-parse HEAD)" +echo "kernel: virtio-nvgpu driver at ${nvgpu_rev}" + +# Copied in, not linked: kbuild writes its objects beside the sources, and a +# symlinked directory would put them in the checkout. A file is only copied +# when it differs, so an unchanged driver does not relink the kernel. +nvgpu_src="${NVGPU_WORK}/driver" +# Under drm/, not virtio/, although it binds a virtio device. Built in, a +# driver initialises in link order, and drivers/virtio links before +# drivers/gpu: there, its probe runs before the DRM core exists and its render +# node fails with "DRM core is not initialized". Loaded as a module, the order +# never came up. +nvgpu_dst="drivers/gpu/drm/nvgpu" +mkdir -p "${nvgpu_dst}/gen" +for dst in "${nvgpu_dst}"/*.[ch] "${nvgpu_dst}"/gen/*.h; do + [[ -e "${dst}" && ! -e "${nvgpu_src}/${dst#"${nvgpu_dst}"/}" ]] && rm -f "${dst}" +done +for src in "${nvgpu_src}"/*.[ch] "${nvgpu_src}"/gen/*.h "${nvgpu_src}/Kconfig"; do + dst="${nvgpu_dst}/${src#"${nvgpu_src}"/}" + cmp -s "${src}" "${dst}" || cp "${src}" "${dst}" +done + +# The driver's Makefile also builds out of tree, against a KDIR. Kbuild prefers +# a Kbuild file over a Makefile, so the in-tree half is taken from it alone: +# the object list, not the out-of-tree rules around it. +kbuild="$(grep -E '^obj-\$\(CONFIG_VIRTIO_GPU_NV\)' "${nvgpu_src}/Makefile" || true)" +if [[ -z "${kbuild}" ]]; then + echo "kernel: ${nvgpu_src}/Makefile has no obj-\$(CONFIG_VIRTIO_GPU_NV) line to build it in tree by" >&2 + exit 1 +fi +[[ "$(cat "${nvgpu_dst}/Kbuild" 2>/dev/null)" == "${kbuild}" ]] || printf '%s\n' "${kbuild}" > "${nvgpu_dst}/Kbuild" + +# A tree this script hooked into drivers/virtio before the move above still +# has that hook, and building both would define the driver twice. +sed -i '/^source "drivers\/virtio\/nvgpu\/Kconfig"$/d' drivers/virtio/Kconfig +sed -i '/^obj-\$(CONFIG_VIRTIO_GPU_NV) += nvgpu\/$/d' drivers/virtio/Makefile +rm -rf drivers/virtio/nvgpu + +# Hooked in beside DRM's own virtio-gpu, once. Inside the DRM menu's `if DRM`, +# which is also what makes it depend on DRM. If this is ever skipped, the +# fragment check below catches it: CONFIG_VIRTIO_GPU_NV cannot be set without +# its Kconfig. +grep -qx 'source "drivers/gpu/drm/nvgpu/Kconfig"' drivers/gpu/drm/Kconfig \ + || sed -i '\|^source "drivers/gpu/drm/virtio/Kconfig"$|a source "drivers/gpu/drm/nvgpu/Kconfig"' \ + drivers/gpu/drm/Kconfig +grep -qx 'obj-$(CONFIG_VIRTIO_GPU_NV) += nvgpu/' drivers/gpu/drm/Makefile \ + || sed -i '/^obj-\$(CONFIG_DRM_VIRTIO_GPU) += virtio\/$/a obj-$(CONFIG_VIRTIO_GPU_NV) += nvgpu/' \ + drivers/gpu/drm/Makefile + # ── Config ────────────────────────────────────────────── # A fresh tree has no .config. The seed is a known-good minimal config that # olddefconfig migrates to whatever version the tree is at; it only saves a @@ -152,4 +229,5 @@ fi make -j"${JOBS}" "${make_args[@]}" vmlinux cp vmlinux "${KERNEL_OUTPUT}" +echo "${nvgpu_rev}" > "${KERNEL_OUTPUT}.nvgpu-rev" echo "kernel: installed ${KERNEL_OUTPUT} ($(numfmt --to=iec "$(stat -c %s "${KERNEL_OUTPUT}")"))"