feat(nescapture): open the capture layer

A Vulkan implicit layer that captures frames from inside the workload's own
process and encodes them on the GPU they were drawn on. Fourth and last of this
batch, imported as a tree from `nestrilabs/nescapture` on the same terms.

Filed under `apps/` rather than `crates/` despite building a cdylib. The rule
here is what a thing *is*, not what it compiles to: this is a finished artefact
that gets installed into an image beside its layer manifest, not a library
another crate in this tree depends on. `crates/` is for the latter, and putting
this there would make the distinction useless the first time someone looked.

Wired to the workspace, `nesprotocol` by path. Its description named the
transport component; that reads better as what it actually is — where the frames
go — so it says that instead.

Whole workspace builds and tests: 21 across four members.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

apps/nescapture/src/lib.rs

@@ -0,0 +1,353 @@
+// ─────────────────────────────────────────────────────────────────────────────
+//  lib.rs — nescapture Vulkan implicit layer
+//
+//  Entry points:
+//    vkNegotiateLoaderLayerInterfaceVersion
+//    vkGetInstanceProcAddr
+//    vkGetDeviceProcAddr
+//
+//  To add a new hook:
+//    1. Implement it in the relevant module.
+//    2. Add it to `match_device_fn` below.
+//    3. Export it with #[unsafe(no_mangle)] in the module.
+// ─────────────────────────────────────────────────────────────────────────────
+
+#![allow(
+    non_snake_case,
+    non_camel_case_types,
+    dead_code,
+    unused_variables,
+    clippy::missing_safety_doc,
+    clippy::too_many_arguments
+)]
+
+mod capture;
+mod commands;
+mod config;
+mod device;
+mod discovery;
+mod dispatch;
+mod dmabuf_import;
+mod encode;
+mod framebuffer;
+mod instance;
+mod pipeline;
+mod present;
+mod shader;
+mod state;
+mod swapchain;
+
+use commands::{
+    vkCmdBeginRenderPass, vkCmdBeginRenderingKHR, vkCmdBindPipeline, vkCmdDraw, vkCmdDrawIndexed,
+    vkCmdDrawIndexedIndirect, vkCmdDrawIndexedIndirectCount, vkCmdDrawIndexedIndirectCountKHR,
+    vkCmdDrawIndirect, vkCmdDrawIndirectCount, vkCmdDrawIndirectCountKHR, vkCmdEndRenderPass,
+    vkCmdEndRenderingKHR,
+};
+use device::{vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue};
+use framebuffer::{
+    vkAllocateCommandBuffers, vkCreateFramebuffer, vkCreateImageView, vkDestroyFramebuffer,
+    vkDestroyImageView, vkFreeCommandBuffers,
+};
+use instance::{vkCreateInstance, vkDestroyInstance};
+use pipeline::{vkCreateGraphicsPipelines, vkDestroyPipeline};
+use present::vkQueuePresentKHR;
+use shader::{vkCreateShaderModule, vkDestroyShaderModule};
+use swapchain::{vkCreateSwapchainKHR, vkDestroySwapchainKHR, vkGetSwapchainImagesKHR};
+
+use dispatch::{PFN_vkGetDeviceProcAddr, PFN_vkGetInstanceProcAddr, RawFn};
+use state::{DEVICE_STATE, INSTANCE_STATE};
+
+use ash::vk::{self, Handle};
+use once_cell::sync::OnceCell;
+use std::ffi::CStr;
+use std::os::raw::{c_char, c_void};
+
+const ENABLE_ENV: &str = "NESCAPTURE_ENABLE";
+
+pub(crate) fn enabled() -> bool {
+    std::env::var(ENABLE_ENV).map(|v| v == "1").unwrap_or(false)
+}
+
+static LOGGER: OnceCell<()> = OnceCell::new();
+
+pub(crate) fn init_logger() {
+    LOGGER.get_or_init(|| {
+        env_logger::Builder::from_default_env().init();
+    });
+}
+
+// ── Vulkan loader structs (not in ash) ────────────────────────────────────────
+
+#[repr(C)]
+pub(crate) struct VkLayerInstanceLink {
+    pNext: *mut VkLayerInstanceLink,
+    pfnNextGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
+    pfnNextGetPhysicalDeviceProcAddr: Option<RawFn>,
+}
+
+#[repr(C)]
+pub(crate) struct VkLayerDeviceLink {
+    pNext: *mut VkLayerDeviceLink,
+    pfnNextGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
+    pfnNextGetDeviceProcAddr: Option<PFN_vkGetDeviceProcAddr>,
+}
+
+#[repr(C)]
+pub(crate) union VkLayerCreateInfoU {
+    pub pLayerInfo: *mut VkLayerInstanceLink,
+    pub pDeviceLayerInfo: *mut VkLayerDeviceLink,
+}
+
+const VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO: i32 = 47;
+const VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO: i32 = 48;
+
+#[repr(C)]
+pub(crate) struct VkLayerInstanceCreateInfo {
+    pub sType: i32,
+    pub pNext: *const c_void,
+    pub function: u32,
+    pub u: VkLayerCreateInfoU,
+}
+
+pub(crate) type VkLayerDeviceCreateInfo = VkLayerInstanceCreateInfo;
+
+pub(crate) unsafe fn dispatch_key(handle: *const c_void) -> usize {
+    unsafe { *(handle as *const usize) }
+}
+
+pub(crate) unsafe fn find_layer_link<T>(p_next: *const c_void, function: u32) -> Option<*mut T> {
+    let mut current = p_next;
+    while !current.is_null() {
+        let header = current as *const VkLayerInstanceCreateInfo;
+        let s = unsafe { (*header).sType };
+        if (s == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO
+            || s == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO)
+            && unsafe { (*header).function } == function
+        {
+            return Some(header as *mut T);
+        }
+        current = unsafe { (*header).pNext };
+    }
+    None
+}
+
+pub(crate) unsafe fn load_device_fn<T>(
+    get: PFN_vkGetDeviceProcAddr,
+    device: vk::Device,
+    name: &[u8],
+) -> T {
+    let raw = unsafe {
+        get(device, name.as_ptr() as *const c_char)
+            .unwrap_or_else(|| panic!("missing device fn: {}", core::str::from_utf8(name).unwrap()))
+    };
+    unsafe { std::mem::transmute_copy(&raw) }
+}
+
+pub(crate) unsafe fn try_load_device_fn<T>(
+    get: PFN_vkGetDeviceProcAddr,
+    device: vk::Device,
+    name: &[u8],
+) -> Option<T> {
+    let raw = unsafe { get(device, name.as_ptr() as *const c_char) }?;
+    Some(unsafe { std::mem::transmute_copy(&raw) })
+}
+
+pub(crate) unsafe fn load_instance_fn<T>(
+    get: PFN_vkGetInstanceProcAddr,
+    instance: vk::Instance,
+    name: &[u8],
+) -> T {
+    let raw = unsafe {
+        get(instance, name.as_ptr() as *const c_char).unwrap_or_else(|| {
+            panic!(
+                "missing instance fn: {}",
+                core::str::from_utf8(name).unwrap()
+            )
+        })
+    };
+    unsafe { std::mem::transmute_copy(&raw) }
+}
+
+#[inline]
+pub(crate) unsafe fn to_raw(addr: usize) -> RawFn {
+    unsafe { std::mem::transmute(addr) }
+}
+
+macro_rules! layer_fn {
+    ($f:ident) => {
+        return Some(unsafe { to_raw($f as *const () as usize) })
+    };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+//  vkGetInstanceProcAddr
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[unsafe(no_mangle)]
+pub unsafe extern "system" fn vkGetInstanceProcAddr(
+    instance: vk::Instance,
+    p_name: *const c_char,
+) -> Option<RawFn> {
+    let name = unsafe { CStr::from_ptr(p_name).to_bytes() };
+
+    match name {
+        b"vkGetInstanceProcAddr" => layer_fn!(vkGetInstanceProcAddr),
+        b"vkCreateInstance" => layer_fn!(vkCreateInstance),
+        b"vkDestroyInstance" => layer_fn!(vkDestroyInstance),
+        b"vkCreateDevice" => layer_fn!(vkCreateDevice),
+        _ => {}
+    }
+
+    if let Some(f) = unsafe { match_device_fn(name) } {
+        return Some(f);
+    }
+
+    if instance.as_raw() == 0 {
+        return None;
+    }
+
+    let ikey = unsafe { dispatch_key(instance.as_raw() as *const c_void) };
+    if let Some(istate) = INSTANCE_STATE.get(&ikey) {
+        return unsafe { (istate.get_instance_proc_addr)(instance, p_name) };
+    }
+    None
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+//  vkGetDeviceProcAddr
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[unsafe(no_mangle)]
+pub unsafe extern "system" fn vkGetDeviceProcAddr(
+    device: vk::Device,
+    p_name: *const c_char,
+) -> Option<RawFn> {
+    let name = unsafe { CStr::from_ptr(p_name).to_bytes() };
+
+    if let Some(f) = unsafe { match_device_fn(name) } {
+        return Some(f);
+    }
+
+    let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
+    if let Some(ds) = DEVICE_STATE.get(&key) {
+        return unsafe { (ds.fp.get_device_proc_addr)(device, p_name) };
+    }
+    None
+}
+
+unsafe fn match_device_fn(name: &[u8]) -> Option<RawFn> {
+    let name_str = std::str::from_utf8(name).unwrap_or("<non-utf8>");
+    // Only log swapchain/acquire/present functions to keep noise down
+    let log_this = name_str.contains("Swapchain")
+        || name_str.contains("AcquireNextImage")
+        || name_str.contains("QueuePresent")
+        || name_str.contains("CreateDevice")
+        || name_str.contains("GetSwapchain");
+    if log_this {
+        log::debug!("gdpa: {}", name_str);
+    }
+    unsafe {
+        match name {
+            b"vkGetDeviceProcAddr" => Some(to_raw(vkGetDeviceProcAddr as *const () as usize)),
+            b"vkCreateDevice" => Some(to_raw(vkCreateDevice as *const () as usize)),
+            b"vkDestroyDevice" => Some(to_raw(vkDestroyDevice as *const () as usize)),
+            b"vkGetDeviceQueue" => Some(to_raw(vkGetDeviceQueue as *const () as usize)),
+            b"vkQueuePresentKHR" => Some(to_raw(vkQueuePresentKHR as *const () as usize)),
+
+            b"vkCreateShaderModule" => Some(to_raw(vkCreateShaderModule as *const () as usize)),
+            b"vkDestroyShaderModule" => Some(to_raw(vkDestroyShaderModule as *const () as usize)),
+            b"vkCreateGraphicsPipelines" => {
+                Some(to_raw(vkCreateGraphicsPipelines as *const () as usize))
+            }
+            b"vkDestroyPipeline" => Some(to_raw(vkDestroyPipeline as *const () as usize)),
+
+            b"vkCreateImageView" => Some(to_raw(vkCreateImageView as *const () as usize)),
+            b"vkDestroyImageView" => Some(to_raw(vkDestroyImageView as *const () as usize)),
+            b"vkCreateFramebuffer" => Some(to_raw(vkCreateFramebuffer as *const () as usize)),
+            b"vkDestroyFramebuffer" => Some(to_raw(vkDestroyFramebuffer as *const () as usize)),
+            b"vkAllocateCommandBuffers" => {
+                Some(to_raw(vkAllocateCommandBuffers as *const () as usize))
+            }
+            b"vkFreeCommandBuffers" => Some(to_raw(vkFreeCommandBuffers as *const () as usize)),
+
+            b"vkCmdBindPipeline" => Some(to_raw(vkCmdBindPipeline as *const () as usize)),
+            b"vkCmdBeginRenderPass" => Some(to_raw(vkCmdBeginRenderPass as *const () as usize)),
+            b"vkCmdEndRenderPass" => Some(to_raw(vkCmdEndRenderPass as *const () as usize)),
+            b"vkCmdBeginRenderingKHR" => Some(to_raw(vkCmdBeginRenderingKHR as *const () as usize)),
+            b"vkCmdEndRenderingKHR" => Some(to_raw(vkCmdEndRenderingKHR as *const () as usize)),
+
+            b"vkCmdDraw" => Some(to_raw(vkCmdDraw as *const () as usize)),
+            b"vkCmdDrawIndexed" => Some(to_raw(vkCmdDrawIndexed as *const () as usize)),
+            b"vkCmdDrawIndirect" => Some(to_raw(vkCmdDrawIndirect as *const () as usize)),
+            b"vkCmdDrawIndexedIndirect" => {
+                Some(to_raw(vkCmdDrawIndexedIndirect as *const () as usize))
+            }
+            b"vkCmdDrawIndirectCount" => Some(to_raw(vkCmdDrawIndirectCount as *const () as usize)),
+            b"vkCmdDrawIndexedIndirectCount" => {
+                Some(to_raw(vkCmdDrawIndexedIndirectCount as *const () as usize))
+            }
+            b"vkCmdDrawIndirectCountKHR" => {
+                Some(to_raw(vkCmdDrawIndirectCountKHR as *const () as usize))
+            }
+            b"vkCmdDrawIndexedIndirectCountKHR" => Some(to_raw(
+                vkCmdDrawIndexedIndirectCountKHR as *const () as usize,
+            )),
+
+            b"vkCreateSwapchainKHR" => {
+                if log_this {
+                    log::debug!("gdpa: → our vkCreateSwapchainKHR");
+                }
+                Some(to_raw(vkCreateSwapchainKHR as *const () as usize))
+            }
+            b"vkDestroySwapchainKHR" => {
+                if log_this {
+                    log::debug!("gdpa: → our vkDestroySwapchainKHR");
+                }
+                Some(to_raw(vkDestroySwapchainKHR as *const () as usize))
+            }
+            b"vkGetSwapchainImagesKHR" => {
+                if log_this {
+                    log::debug!("gdpa: → our vkGetSwapchainImagesKHR");
+                }
+                Some(to_raw(vkGetSwapchainImagesKHR as *const () as usize))
+            }
+
+            _ => {
+                if log_this {
+                    log::debug!("gdpa: → passthrough (not intercepted)");
+                }
+                None
+            }
+        }
+    }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+//  vkNegotiateLoaderLayerInterfaceVersion
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[repr(C)]
+pub struct VkNegotiateLayerInterface {
+    pub sType: u32,
+    pub pNext: *mut c_void,
+    pub loaderLayerInterfaceVersion: u32,
+    pub pfnGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
+    pub pfnGetDeviceProcAddr: Option<PFN_vkGetDeviceProcAddr>,
+    pub pfnGetPhysicalDeviceProcAddr:
+        Option<unsafe extern "system" fn(vk::Instance, *const c_char) -> Option<RawFn>>,
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "system" fn vkNegotiateLoaderLayerInterfaceVersion(
+    p_version_struct: *mut VkNegotiateLayerInterface,
+) -> vk::Result {
+    init_logger();
+    let s = unsafe { &mut *p_version_struct };
+    if s.loaderLayerInterfaceVersion > 2 {
+        s.loaderLayerInterfaceVersion = 2;
+    }
+    s.pfnGetInstanceProcAddr = Some(vkGetInstanceProcAddr);
+    s.pfnGetDeviceProcAddr = Some(vkGetDeviceProcAddr);
+    s.pfnGetPhysicalDeviceProcAddr = None;
+    vk::Result::SUCCESS
+}