From 01d6f2302d28713ff7b84ead7c050d0a42f0e051 Mon Sep 17 00:00:00 2001 From: Wanjohi Date: Fri, 2 Oct 2026 22:43:03 +0300 Subject: [PATCH] build: put virtio-nvgpu's Vulkan layer into the guest rootfs On an NVIDIA host the guest's Vulkan driver offers ray tracing, optical flow and a few others it cannot create a device with, and an engine that enables them does not start. virtio-nvgpu's layer hides them; it acts only on a forwarded NVIDIA device, so AMD and Intel boxes are unaffected. Built from virtio-nvgpu at NVGPU_REF, the ref the kernel build already takes the guest driver from, and recorded in the image. --- build/Containerfile | 41 +++++++++++++++++++++++++++++++++++++++-- build/Makefile | 10 +++++++--- build/README.md | 6 ++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/build/Containerfile b/build/Containerfile index f87397f4..374be2af 100644 --- a/build/Containerfile +++ b/build/Containerfile @@ -176,6 +176,40 @@ RUN --mount=type=cache,target=/root/.cargo/registry \ find /artifacts/nestri -type f -printf '/%P\n' > /artifacts/nestri/.manifest +# ─────────────────────────────────────────────────────────── +# virtio-nvgpu's Vulkan layer — for guests on an NVIDIA host +# +# On an NVIDIA host the guest runs NVIDIA's own Vulkan driver over a +# forwarded device, and that driver offers ray tracing, optical flow and a +# few others it then cannot create a device with: vkCreateDevice fails, and +# an engine that enables what it is offered does not start. The layer hides +# them. It acts only on an NVIDIA device in a guest with the virtio-nvgpu +# driver, so on an AMD or Intel box it is loaded and does nothing. +# +# Built from virtio-nvgpu's own tree at NVGPU_REF, the same ref the kernel +# build takes the guest driver from (build/Makefile), because the layer's job +# is to say what that driver and its backend can do. The commit is recorded +# in /usr/share/nestri/virtio-nvgpu.rev. +# ─────────────────────────────────────────────────────────── +FROM builder AS nvgpu-layer-build +ARG NVGPU_GIT=https://github.com/nestrilabs/virtio-nvgpu.git +ARG NVGPU_REF=dev +RUN --mount=type=cache,target=/root/.cargo/registry \ + git init -q /build/virtio-nvgpu && cd /build/virtio-nvgpu && \ + git fetch -q --depth=1 "${NVGPU_GIT}" "${NVGPU_REF}" && git checkout -q FETCH_HEAD && \ + cargo build --release -p nvgpu-vklayer && \ + install -Dm755 target/release/libVkLayer_nvgpu.so \ + /artifacts/nvgpu/usr/lib/libVkLayer_nvgpu.so && \ + mkdir -p /artifacts/nvgpu/usr/share/vulkan/implicit_layer.d /artifacts/nvgpu/usr/share/nestri && \ + sed 's|"./libVkLayer_nvgpu.so"|"libVkLayer_nvgpu.so"|' vklayer/VkLayer_nvgpu.json \ + > /artifacts/nvgpu/usr/share/vulkan/implicit_layer.d/VkLayer_nvgpu.json && \ + grep -q '"library_path": "libVkLayer_nvgpu.so"' \ + /artifacts/nvgpu/usr/share/vulkan/implicit_layer.d/VkLayer_nvgpu.json && \ + git rev-parse HEAD > /artifacts/nvgpu/usr/share/nestri/virtio-nvgpu.rev && \ + cd / && rm -rf /build/virtio-nvgpu && \ + find /artifacts/nvgpu -type f -printf '/%P\n' > /artifacts/nvgpu/.manifest + + # ─────────────────────────────────────────────────────────── # umu-launcher — built here, not installed from the distribution # @@ -429,10 +463,12 @@ LABEL org.opencontainers.image.source="https://github.com/nestrilabs/nestri" # purpose, not an oversight. COPY --from=mesa-build /artifacts/mesa/.manifest /tmp/mesa.manifest COPY --from=nestri-build /artifacts/nestri/.manifest /tmp/nestri.manifest -RUN cat /tmp/mesa.manifest /tmp/nestri.manifest > /tmp/.strip-manifest && \ - rm -f /tmp/mesa.manifest /tmp/nestri.manifest +COPY --from=nvgpu-layer-build /artifacts/nvgpu/.manifest /tmp/nvgpu.manifest +RUN cat /tmp/mesa.manifest /tmp/nestri.manifest /tmp/nvgpu.manifest > /tmp/.strip-manifest && \ + rm -f /tmp/mesa.manifest /tmp/nestri.manifest /tmp/nvgpu.manifest COPY --from=mesa-build /artifacts/mesa / COPY --from=nestri-build /artifacts/nestri / +COPY --from=nvgpu-layer-build /artifacts/nvgpu / COPY --from=umu-build /artifacts/umu / # **No Proton here.** It is attached to a box at run time, mounted at @@ -555,6 +591,7 @@ RUN failed=0; \ : > /tmp/missing-libs; \ for f in /usr/bin/nesinit /usr/bin/nescope /usr/bin/neshub /usr/bin/neswire \ /usr/bin/nesgamepad /usr/lib/libnescapture_layer.so /usr/lib/libSDL2-2.0.so.0 \ + /usr/lib/libVkLayer_nvgpu.so \ /usr/bin/dbus-daemon /usr/bin/pipewire \ /usr/bin/wireplumber /usr/bin/ip \ /usr/lib/libgallium-*.so /usr/lib/libEGL_mesa.so.0 \ diff --git a/build/Makefile b/build/Makefile index 02fa199f..bc01159e 100644 --- a/build/Makefile +++ b/build/Makefile @@ -76,9 +76,11 @@ INFINITY_REV ?= e6c85d841f25e21393a7ea47fdef286318915d8c INFINITY_SERIES := patches/cachyos/tuned-eevdf/$(KERNEL_REF:cachyos-%=%) INFINITY_WORK := $(OUTPUT_DIR)/infinity-sched -# The guest half of NVIDIA forwarding, built into the kernel. A branch, so each -# kernel build takes the driver as it is now; set NVGPU_REF to a tag or commit -# to pin it. The commit built is recorded as $(KERNEL_OUTPUT).nvgpu-rev. +# The guest half of NVIDIA forwarding: the driver, built into the kernel, and +# its Vulkan layer, built into the rootfs. A branch, so each build takes them +# as they are now; set NVGPU_REF to a tag or commit to pin it. The commit built +# is recorded as $(KERNEL_OUTPUT).nvgpu-rev and, in the image, as +# /usr/share/nestri/virtio-nvgpu.rev. Build both from the same ref. NVGPU_GIT ?= https://github.com/nestrilabs/virtio-nvgpu.git NVGPU_REF ?= dev NVGPU_WORK := $(OUTPUT_DIR)/virtio-nvgpu @@ -94,11 +96,13 @@ endif build: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ --build-arg PROTON_IMAGE=$(PROTON_REF) \ + --build-arg NVGPU_GIT=$(NVGPU_GIT) --build-arg NVGPU_REF=$(NVGPU_REF) \ -f Containerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT) build-debug: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ --build-arg PROTON_IMAGE=$(PROTON_REF) \ + --build-arg NVGPU_GIT=$(NVGPU_GIT) --build-arg NVGPU_REF=$(NVGPU_REF) \ -f Containerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT) # Hours, and only when PROTON_TAG moves. Two steps: the build runs on the host, diff --git a/build/README.md b/build/README.md index ebbdacf3..796e7a06 100644 --- a/build/README.md +++ b/build/README.md @@ -166,6 +166,12 @@ modules. The driver's parameters therefore go on the command line as warns that it is doing so. - An AMD or Intel host is unaffected. The driver binds only to the forwarding device, and nothing offers that device to those guests. +- **The rootfs carries virtio-nvgpu's Vulkan layer**, built from the same + `NVGPU_REF`. NVIDIA's driver in a guest offers ray tracing and a few other + extensions it cannot then create a device with; the layer hides them. It + acts only on an NVIDIA device in a guest with the forwarding driver, so AMD + and Intel boxes are unaffected. Build the kernel and the image from the same + ref, or the layer describes a different driver than the one booted. - The NVIDIA libraries are **not** part of the image. They must be the same build as the host's kernel module, so the host shares its own copy with the guest at run time.