github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2026-09-19 17:25:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: nescapture capture improvements and drive mounts (#337)

Browse Source 
Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kristian Ollikainen
2026-09-18 22:58:22 +03:00
committed by GitHub
parent ebc0242b49
commit 6811c93d51
32 changed files with 3677 additions and 691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

624
apps/nescapture/src/capture.rs
View File

@@ -5,8 +5,11 @@
// so that after the GPU blit we can export an fd and import it into pixelforge's
// separate VkDevice for zero-copy hardware encoding via DmaBufImporter.
//
// The row stride comes from the image's SubresourceLayout, queried once at
// allocation; the encoder needs it to import the LINEAR image correctly.
// The ring is allocated tiled where the driver offers a single-plane DRM
// format modifier, and linear where it does not. The stride and the chosen
// modifier come from the image itself, queried once at allocation, and both
// travel with every frame: the importer creates its side with that exact
// modifier, and a wrong value there is a correctly sized frame of nonsense.
// ─────────────────────────────────────────────────────────────────────────────
use crate::state::{CB_STATE, CAPTURE_SLOTS, CaptureRing, CaptureSlot, DEVICE_STATE};
@@ -51,20 +54,89 @@ macro_rules! image_barrier {
// ── Memory helper ─────────────────────────────────────────────────────────────
unsafe fn find_host_coherent_mt(ds: &crate::state::DeviceState, bits: u32) -> u32 {
unsafe fn find_memory_type(
ds: &crate::state::DeviceState,
bits: u32,
want: crate::memory::Want,
) -> Option<u32> {
let mut mp = vk::PhysicalDeviceMemoryProperties::default();
let k = unsafe { crate::dispatch_key(ds.physical_device.as_raw() as *const std::ffi::c_void) };
if let Some(i) = crate::state::INSTANCE_STATE.get(&k) {
unsafe { (i.get_physical_device_memory_properties)(ds.physical_device, &mut mp) };
}
(0..mp.memory_type_count)
.find(|&i| {
(bits & (1 << i)) != 0
&& mp.memory_types[i as usize].property_flags.contains(
vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT,
)
let types: Vec<crate::memory::MemoryType> = mp.memory_types[..mp.memory_type_count as usize]
.iter()
.map(|t| crate::memory::MemoryType {
flags: t.property_flags,
})
.unwrap_or(0)
.collect();
crate::memory::pick_memory_type(&types, bits, want)
}
/// Modifiers the device can both receive a transfer into and have sampled from,
/// for `fmt`.
///
/// Both feature bits matter and for different sides: the layer writes the image
/// with `vkCmdCopyImage`, and pixelforge samples it in the colour-conversion
/// compute shader after importing it. A modifier that supports only one of
/// those is no use to this ring.
unsafe fn supported_modifiers(
ds: &crate::state::DeviceState,
fmt: vk::Format,
) -> Vec<crate::modifiers::ModifierProps> {
let k = unsafe { crate::dispatch_key(ds.physical_device.as_raw() as *const std::ffi::c_void) };
let Some(istate) = crate::state::INSTANCE_STATE.get(&k) else {
return Vec::new();
};
let Some(get_props2) = istate.get_physical_device_format_properties2 else {
return Vec::new();
};
// Two calls: the first to learn the count, the second to fill the list.
let mut list = vk::DrmFormatModifierPropertiesListEXT::default();
let mut props2 = vk::FormatProperties2 {
p_next: &mut list as *mut _ as *mut std::ffi::c_void,
..Default::default()
};
unsafe { get_props2(ds.physical_device, fmt, &mut props2) };
let count = list.drm_format_modifier_count as usize;
if count == 0 {
return Vec::new();
}
let mut entries = vec![vk::DrmFormatModifierPropertiesEXT::default(); count];
list.p_drm_format_modifier_properties = entries.as_mut_ptr();
let mut props2 = vk::FormatProperties2 {
p_next: &mut list as *mut _ as *mut std::ffi::c_void,
..Default::default()
};
unsafe { get_props2(ds.physical_device, fmt, &mut props2) };
let needed =
vk::FormatFeatureFlags::TRANSFER_DST | vk::FormatFeatureFlags::SAMPLED_IMAGE;
entries
.iter()
.filter(|e| e.drm_format_modifier_tiling_features.contains(needed))
.map(|e| crate::modifiers::ModifierProps {
modifier: e.drm_format_modifier,
plane_count: e.drm_format_modifier_plane_count,
})
.collect()
}
/// Which modifier the driver actually gave an image.
///
/// The image is created from a list of acceptable modifiers and the driver
/// chooses; the importer needs the one it chose, not the list. `None` when the
/// extension is absent or the call fails, which sends the caller back to the
/// linear path rather than letting it guess.
unsafe fn image_modifier(ds: &crate::state::DeviceState, image: vk::Image) -> Option<u64> {
let get = ds.fp.get_image_drm_format_modifier_properties_ext?;
let mut props = vk::ImageDrmFormatModifierPropertiesEXT::default();
if unsafe { get(ds.raw, image, &mut props) } != vk::Result::SUCCESS {
return None;
}
Some(props.drm_format_modifier)
}
// ── Image allocators ──────────────────────────────────────────────────────────
@@ -113,7 +185,90 @@ unsafe fn allocate_dmabuf_image(
h: u32,
fmt: vk::Format,
label: &str,
) -> Option<(vk::Image, vk::DeviceMemory)> {
) -> Option<(vk::Image, vk::DeviceMemory, u64)> {
let export_ai = vk::ExportMemoryAllocateInfo {
s_type: vk::StructureType::EXPORT_MEMORY_ALLOCATE_INFO,
p_next: std::ptr::null_mut(),
handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
// Tiled first. A linear destination means the copy detiles a whole frame on
// the way in and the encoder samples a linear image on the way out; the
// importer has always been able to take a tiled buffer, and only this side
// was ever linear.
let candidates = unsafe { supported_modifiers(ds, fmt) };
if let Some(chosen) = crate::modifiers::pick_modifier(&candidates)
&& chosen.modifier != crate::modifiers::LINEAR
{
let mut ext_img = vk::ExternalMemoryImageCreateInfo {
s_type: vk::StructureType::EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
p_next: std::ptr::null_mut(),
handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
let modifiers = [chosen.modifier];
let mut mod_list = vk::ImageDrmFormatModifierListCreateInfoEXT::default()
.drm_format_modifiers(&modifiers);
mod_list.p_next = &mut ext_img as *mut _ as *mut std::ffi::c_void;
let ci = vk::ImageCreateInfo {
s_type: vk::StructureType::IMAGE_CREATE_INFO,
p_next: &mod_list as *const _ as *const _,
flags: vk::ImageCreateFlags::empty(),
image_type: vk::ImageType::TYPE_2D,
format: fmt,
extent: vk::Extent3D {
width: w,
height: h,
depth: 1,
},
mip_levels: 1,
array_layers: 1,
samples: vk::SampleCountFlags::TYPE_1,
tiling: vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT,
usage: vk::ImageUsageFlags::TRANSFER_DST,
sharing_mode: vk::SharingMode::EXCLUSIVE,
queue_family_index_count: 0,
p_queue_family_indices: std::ptr::null(),
initial_layout: vk::ImageLayout::UNDEFINED,
_marker: std::marker::PhantomData,
};
if let Some((image, memory)) = unsafe { alloc_image(ds, &ci, Some(&export_ai), label) } {
// Ask which one it took rather than assuming the one offered: the
// importer is given an explicit modifier and a wrong value there is
// a correctly sized frame full of nonsense.
match unsafe { image_modifier(ds, image) } {
Some(actual) => {
log::info!(
"capture '{label}': tiled, modifier {actual:#018x} \
(offered {:#018x}, {} candidate(s))",
chosen.modifier,
candidates.len()
);
return Some((image, memory, actual));
}
None => {
log::warn!(
"capture '{label}': the driver would not report the modifier it \
chose — falling back to linear rather than importing a guess"
);
unsafe {
(ds.fp.destroy_image)(ds.raw, image, std::ptr::null());
(ds.fp.free_memory)(ds.raw, memory, std::ptr::null());
}
}
}
} else {
log::warn!("capture '{label}': tiled allocation refused — falling back to linear");
}
} else {
log::info!(
"capture '{label}': no tiled modifier offered ({} candidate(s)) — linear",
candidates.len()
);
}
let ext_img = vk::ExternalMemoryImageCreateInfo {
s_type: vk::StructureType::EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
p_next: std::ptr::null_mut(),
@@ -142,14 +297,8 @@ unsafe fn allocate_dmabuf_image(
initial_layout: vk::ImageLayout::UNDEFINED,
_marker: std::marker::PhantomData,
};
let export_ai = vk::ExportMemoryAllocateInfo {
s_type: vk::StructureType::EXPORT_MEMORY_ALLOCATE_INFO,
p_next: std::ptr::null_mut(),
handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
if let Some(r) = unsafe { alloc_image(ds, &ci, Some(&export_ai), label) } {
return Some(r);
if let Some((image, memory)) = unsafe { alloc_image(ds, &ci, Some(&export_ai), label) } {
return Some((image, memory, crate::modifiers::LINEAR));
}
log::warn!(
"DMA-BUF alloc failed for '{}' — using plain host image. \
@@ -157,6 +306,7 @@ unsafe fn allocate_dmabuf_image(
label
);
unsafe { allocate_host_image(ds, w, h, fmt, label) }
.map(|(i, m)| (i, m, crate::modifiers::LINEAR))
}
unsafe fn alloc_image(
@@ -177,7 +327,24 @@ unsafe fn alloc_image(
memory_type_bits: 0,
};
unsafe { (ds.fp.get_image_memory_requirements)(ds.raw, image, &mut mr) };
let mt = unsafe { find_host_coherent_mt(ds, mr.memory_type_bits) };
// An exported image is written by this device and read by pixelforge's,
// both on the same GPU. Nothing maps it, so host-visible memory buys
// nothing and on a discrete card costs a full frame across the bus each
// way. Only the readback fallback has to be mappable.
let want = match export {
Some(_) => crate::memory::Want::DeviceLocal,
None => crate::memory::Want::HostCoherent,
};
let mt = match unsafe { find_memory_type(ds, mr.memory_type_bits, want) } {
Some(mt) => mt,
None => {
// Index zero used to be the fallback here, which binds the image
// to a memory type its own requirements may forbid.
log::warn!("no {want:?} memory type for '{label}' - not allocating");
unsafe { (ds.fp.destroy_image)(ds.raw, image, std::ptr::null()) };
return None;
}
};
let p_next: *const _ = match export {
Some(e) => e as *const _ as *const _,
None => std::ptr::null(),
@@ -215,20 +382,23 @@ unsafe fn alloc_image(
// ── Stride query ──────────────────────────────────────────────────────────────
/// Row stride in bytes of a LINEAR image, or 0 on failure.
pub unsafe fn query_stride(ds: &crate::state::DeviceState, image: vk::Image) -> u32 {
let subresource = vk::ImageSubresource {
aspect_mask: vk::ImageAspectFlags::COLOR,
pub unsafe fn query_stride(ds: &crate::state::DeviceState, image: vk::Image, modifier: u64) -> u32 {
// A DRM_FORMAT_MODIFIER image is laid out in memory planes, not colour
// planes, and asking it for COLOR is invalid — the aspect has to name the
// memory plane. Single-plane is all `pick_modifier` will accept, so plane
// zero is the whole image.
let aspect_mask = if modifier == crate::modifiers::LINEAR {
vk::ImageAspectFlags::COLOR
} else {
vk::ImageAspectFlags::MEMORY_PLANE_0_EXT
};
let sub = vk::ImageSubresource {
aspect_mask,
mip_level: 0,
array_layer: 0,
};
let mut layout = vk::SubresourceLayout {
offset: 0,
size: 0,
row_pitch: 0,
array_pitch: 0,
depth_pitch: 0,
};
unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &subresource, &mut layout) };
let mut layout = vk::SubresourceLayout::default();
unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &sub, &mut layout) };
layout.row_pitch as u32
}
@@ -304,10 +474,20 @@ unsafe fn ensure_capture_ring(
h: u32,
f: vk::Format,
queue_family: u32,
image_count: usize,
) -> bool {
if let Some(existing) = ring.as_ref() {
let (ew, eh, ef) = existing.size;
if ew >= w && eh >= h && ef == f && existing.queue_family == queue_family {
// `image_count` joins the identity because the blit buffers are
// allocated one per (image, slot) pair. A swapchain that gained an
// image needs more of them, and a ring that kept the old count would
// silently stop capturing whenever that image came round.
if ew >= w
&& eh >= h
&& ef == f
&& existing.queue_family == queue_family
&& existing.image_count == image_count
{
return true;
}
if !ds.capture_slots.all_free() {
@@ -317,7 +497,7 @@ unsafe fn ensure_capture_ring(
unsafe { destroy_capture_ring(ds, old) };
}
}
match unsafe { create_capture_ring(ds, w, h, f, queue_family) } {
match unsafe { create_capture_ring(ds, w, h, f, queue_family, image_count) } {
Some(fresh) => {
*ring = Some(fresh);
true
@@ -332,6 +512,7 @@ unsafe fn create_capture_ring(
h: u32,
f: vk::Format,
queue_family: u32,
image_count: usize,
) -> Option<CaptureRing> {
let pci = vk::CommandPoolCreateInfo {
s_type: vk::StructureType::COMMAND_POOL_CREATE_INFO,
@@ -349,16 +530,20 @@ unsafe fn create_capture_ring(
return None;
}
// One per (swapchain image, slot) pair, so each can be recorded once and
// re-submitted. Typically twelve to sixteen buffers; they hold a barrier
// pair and a copy each and are never re-recorded in steady state.
let blit_count = image_count.max(1) * CAPTURE_SLOTS;
let ai = vk::CommandBufferAllocateInfo {
s_type: vk::StructureType::COMMAND_BUFFER_ALLOCATE_INFO,
p_next: std::ptr::null(),
command_pool,
level: vk::CommandBufferLevel::PRIMARY,
command_buffer_count: CAPTURE_SLOTS as u32,
command_buffer_count: blit_count as u32,
_marker: std::marker::PhantomData,
};
let mut cbs = [vk::CommandBuffer::null(); CAPTURE_SLOTS];
if unsafe { (ds.fp.allocate_command_buffers)(ds.raw, &ai, cbs.as_mut_ptr()) }
let mut blits = vec![vk::CommandBuffer::null(); blit_count];
if unsafe { (ds.fp.allocate_command_buffers)(ds.raw, &ai, blits.as_mut_ptr()) }
!= vk::Result::SUCCESS
{
unsafe { (ds.fp.destroy_command_pool)(ds.raw, command_pool, std::ptr::null()) };
@@ -373,9 +558,13 @@ unsafe fn create_capture_ring(
_marker: std::marker::PhantomData,
};
let (timestamp_pool, timestamp_period) =
unsafe { create_timestamp_pool(ds, queue_family) };
let mut slots = Vec::with_capacity(CAPTURE_SLOTS);
for (i, &command_buffer) in cbs.iter().enumerate() {
let Some((image, memory)) = (unsafe { allocate_dmabuf_image(ds, w, h, f, "capture") })
for i in 0..CAPTURE_SLOTS {
let Some((image, memory, modifier)) =
(unsafe { allocate_dmabuf_image(ds, w, h, f, "capture") })
else {
unsafe { destroy_partial_ring(ds, command_pool, slots) };
return None;
@@ -389,7 +578,7 @@ unsafe fn create_capture_ring(
unsafe { destroy_partial_ring(ds, command_pool, slots) };
return None;
}
let stride = unsafe { query_stride(ds, image) };
let stride = unsafe { query_stride(ds, image, modifier) };
// Export once. Each frame hands the encoder a dup of this fd, which
// costs a file-descriptor clone instead of a kernel export per frame.
let dmabuf_fd = unsafe { get_dmabuf_fd(ds, memory) }.unwrap_or(-1);
@@ -401,7 +590,7 @@ unsafe fn create_capture_ring(
memory,
dmabuf_fd,
stride,
command_buffer,
modifier,
fence,
});
}
@@ -413,8 +602,20 @@ unsafe fn create_capture_ring(
Some(CaptureRing {
command_pool,
slots,
blits,
blits_recorded: vec![false; blit_count],
image_count: image_count.max(1),
timestamp_pool,
timestamp_period,
size: (w, h, f),
queue_family,
// Zero so the first frame always records: no real extent equals it, so
// the invalidation check in `capture_present_frame` fires once and then
// never again until something actually changes.
blit_extent: vk::Extent2D {
width: 0,
height: 0,
},
present_wait: Vec::new(),
retired: Vec::new(),
})
@@ -462,6 +663,13 @@ pub unsafe fn destroy_capture_ring(ds: &crate::state::DeviceState, ring: Capture
}
}
}
unsafe {
if !ring.timestamp_pool.is_null()
&& let Some(destroy) = ds.fp.destroy_query_pool
{
destroy(ds.raw, ring.timestamp_pool, std::ptr::null());
}
}
unsafe { destroy_partial_ring(ds, ring.command_pool, ring.slots) };
}
@@ -608,85 +816,161 @@ pub struct CaptureSubmission {
pub present_wait: vk::Semaphore,
}
/// Blit the presented swapchain image into a ring slot, ahead of the present.
/// Create the blit timestamp pool, or a null handle where it cannot be used.
///
/// Two orderings have to hold and neither did before.
///
/// The blit must not read the swapchain image before the game has finished
/// rendering into it. The game signals that with the semaphores it attached to
/// `VkPresentInfoKHR`, so the blit waits on exactly those.
///
/// The game must not render into that image again before the blit has read it.
/// Presentation is what releases the image back to the application, so the
/// present is made to wait on a semaphore the blit signals. The old code
/// submitted the blit from a worker thread after `vkQueuePresentKHR` had
/// already returned, which guaranteed neither.
///
/// Returns `None` when the frame cannot be captured, in which case the caller
/// must present unmodified — the application's semaphores have not been touched.
pub unsafe fn capture_present_frame(
/// Null is the normal, expected outcome on some hardware — RADV's video-encode
/// family reports `timestampValidBits == 0`, and a graphics family could too —
/// and it costs nothing but the measurement. `vkCmdWriteTimestamp` on a family
/// reporting zero is a validation error
/// (VUID-vkCmdWriteTimestamp-timestampValidBits-00829), so it has to be asked
/// rather than assumed.
unsafe fn create_timestamp_pool(
ds: &crate::state::DeviceState,
queue: vk::Queue,
si: vk::Image,
fmt: vk::Format,
ext: vk::Extent2D,
image_index: usize,
app_waits: &[vk::Semaphore],
) -> Option<CaptureSubmission> {
if ext.width == 0 || ext.height == 0 {
return None;
}
if !ds
.swapchain_transfer_src
.load(std::sync::atomic::Ordering::Relaxed)
{
return None;
queue_family: u32,
) -> (vk::QueryPool, f32) {
let none = (vk::QueryPool::null(), 0.0);
let (Some(create), Some(_), Some(_), Some(_)) = (
ds.fp.create_query_pool,
ds.fp.cmd_reset_query_pool,
ds.fp.cmd_write_timestamp,
ds.fp.get_query_pool_results,
) else {
return none;
};
let k = unsafe { crate::dispatch_key(ds.physical_device.as_raw() as *const std::ffi::c_void) };
let Some(istate) = crate::state::INSTANCE_STATE.get(&k) else {
return none;
};
let (Some(get_props), Some(get_families)) = (
istate.get_physical_device_properties,
istate.get_physical_device_queue_family_properties,
) else {
return none;
};
let mut props = vk::PhysicalDeviceProperties::default();
unsafe { get_props(ds.physical_device, &mut props) };
let period = props.limits.timestamp_period;
if period <= 0.0 {
return none;
}
// A command buffer may only be submitted to the family its pool was created
// for. An unknown queue means one the layer never saw through
// vkGetDeviceQueue, so there is nothing safe to assume about it.
let queue_family = *crate::state::QUEUE_TO_FAMILY.get(&queue.as_raw())?;
let mut ring_guard = ds.capture_ring.lock().ok()?;
if !unsafe {
ensure_capture_ring(ds, &mut ring_guard, ext.width, ext.height, fmt, queue_family)
} {
return None;
}
let ring = ring_guard.as_mut()?;
let present_wait = unsafe { ensure_present_semaphore(ds, ring, image_index) }?;
// Never blocks: a frame with no free slot is one the encoder has not caught
// up with, and stalling the game's present to wait for it would be worse
// than skipping it.
let guard = ds.capture_slots.try_acquire()?;
let slot = ring.slots.get(guard.index())?;
let cb = slot.command_buffer;
let fence = slot.fence;
let fi = slot.image;
// A free slot's fence is already signalled — the capture worker waits on it
// before the encoder ever sees the frame. This covers the paths that
// abandon a frame and return the slot without that wait.
unsafe {
if (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 2_000_000) != vk::Result::SUCCESS {
return None;
let mut count = 0u32;
unsafe { get_families(ds.physical_device, &mut count, std::ptr::null_mut()) };
let mut families = vec![vk::QueueFamilyProperties::default(); count as usize];
unsafe { get_families(ds.physical_device, &mut count, families.as_mut_ptr()) };
match families.get(queue_family as usize) {
Some(f) if f.timestamp_valid_bits > 0 => {}
_ => {
log::info!(
"queue family {queue_family} reports timestampValidBits=0; \
blit GPU timing disabled"
);
return none;
}
let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence);
let _ = (ds.fp.reset_command_buffer)(cb, vk::CommandBufferResetFlags::empty());
}
let bi = vk::CommandBufferBeginInfo {
let ci = vk::QueryPoolCreateInfo {
s_type: vk::StructureType::QUERY_POOL_CREATE_INFO,
p_next: std::ptr::null(),
flags: vk::QueryPoolCreateFlags::empty(),
query_type: vk::QueryType::TIMESTAMP,
// Two per slot. Per slot and not per (image, slot) pair because only
// one blit per slot is ever in flight.
query_count: (CAPTURE_SLOTS * 2) as u32,
pipeline_statistics: vk::QueryPipelineStatisticFlags::empty(),
_marker: std::marker::PhantomData,
};
let mut pool = vk::QueryPool::null();
if unsafe { create(ds.raw, &ci, std::ptr::null(), &mut pool) } != vk::Result::SUCCESS {
log::warn!("blit timestamp pool could not be created; GPU timing disabled");
return none;
}
(pool, period)
}
/// GPU nanoseconds the last blit into `slot` took.
///
/// Call only after that slot's fence has signalled, so the results are there
/// and the `WAIT` flag returns immediately. `None` when timing is off, when the
/// driver refuses the results, or when the counter wrapped between the pair.
pub unsafe fn blit_gpu_time_ns(ds: &crate::state::DeviceState, slot: usize) -> Option<u64> {
let ring_guard = ds.capture_ring.lock().ok()?;
let ring = ring_guard.as_ref()?;
if ring.timestamp_pool.is_null() {
return None;
}
let get = ds.fp.get_query_pool_results?;
let mut ticks = [0u64; 2];
let result = unsafe {
get(
ds.raw,
ring.timestamp_pool,
(slot * 2) as u32,
2,
std::mem::size_of_val(&ticks),
ticks.as_mut_ptr() as *mut std::ffi::c_void,
std::mem::size_of::<u64>() as vk::DeviceSize,
vk::QueryResultFlags::WAIT | vk::QueryResultFlags::TYPE_64,
)
};
if result != vk::Result::SUCCESS {
return None;
}
let elapsed = ticks[1].checked_sub(ticks[0])?;
Some((elapsed as f64 * f64::from(ring.timestamp_period)) as u64)
}
/// Record the blit from one swapchain image into one ring slot.
///
/// Called once per (image, slot) pair and then never again while the swapchain
/// and the extent hold. No `ONE_TIME_SUBMIT`: this buffer is submitted many
/// times. It is never submitted twice concurrently, because the slot it writes
/// is held by a `SlotGuard` for the whole life of the frame, so the previous
/// submission has completed before that slot is handed out again.
unsafe fn record_blit(
ds: &crate::state::DeviceState,
cb: vk::CommandBuffer,
si: vk::Image,
fi: vk::Image,
ext: vk::Extent2D,
timestamp_pool: vk::QueryPool,
slot_index: usize,
) -> bool {
let begin = vk::CommandBufferBeginInfo {
s_type: vk::StructureType::COMMAND_BUFFER_BEGIN_INFO,
p_next: std::ptr::null(),
flags: vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT,
// The pool carries RESET_COMMAND_BUFFER, so beginning an already
// recorded buffer implicitly resets it. That is the re-record path,
// taken after a swapchain recreation or an extent change.
flags: vk::CommandBufferUsageFlags::empty(),
p_inheritance_info: std::ptr::null(),
_marker: std::marker::PhantomData,
};
if unsafe { (ds.fp.begin_command_buffer)(cb, &bi) } != vk::Result::SUCCESS {
return None;
if unsafe { (ds.fp.begin_command_buffer)(cb, &begin) } != vk::Result::SUCCESS {
return false;
}
// Bracket the barriers as well as the copy: the layout transitions on the
// swapchain image are part of what this costs the GPU, and the first of
// them is a full flush. Recorded once with the rest of the buffer; the
// reset runs on every submission, which is what makes the pair reusable.
let timed = !timestamp_pool.is_null()
&& ds.fp.cmd_reset_query_pool.is_some()
&& ds.fp.cmd_write_timestamp.is_some();
if timed {
let first = (slot_index * 2) as u32;
unsafe {
(ds.fp.cmd_reset_query_pool.unwrap())(cb, timestamp_pool, first, 2);
(ds.fp.cmd_write_timestamp.unwrap())(
cb,
vk::PipelineStageFlags::TOP_OF_PIPE,
timestamp_pool,
first,
);
}
}
let b1 = image_barrier!(
@@ -775,9 +1059,120 @@ pub unsafe fn capture_present_frame(
);
}
if timed {
unsafe {
(ds.fp.cmd_write_timestamp.unwrap())(
cb,
vk::PipelineStageFlags::BOTTOM_OF_PIPE,
timestamp_pool,
(slot_index * 2 + 1) as u32,
);
}
}
if unsafe { (ds.fp.end_command_buffer)(cb) } != vk::Result::SUCCESS {
return false;
}
true
}
/// Blit the presented swapchain image into a ring slot, ahead of the present.
///
/// Two orderings have to hold and neither did before.
///
/// The blit must not read the swapchain image before the game has finished
/// rendering into it. The game signals that with the semaphores it attached to
/// `VkPresentInfoKHR`, so the blit waits on exactly those.
///
/// The game must not render into that image again before the blit has read it.
/// Presentation is what releases the image back to the application, so the
/// present is made to wait on a semaphore the blit signals. The old code
/// submitted the blit from a worker thread after `vkQueuePresentKHR` had
/// already returned, which guaranteed neither.
///
/// Returns `None` when the frame cannot be captured, in which case the caller
/// must present unmodified — the application's semaphores have not been touched.
pub unsafe fn capture_present_frame(
ds: &crate::state::DeviceState,
queue: vk::Queue,
si: vk::Image,
fmt: vk::Format,
ext: vk::Extent2D,
image_index: usize,
image_count: usize,
app_waits: &[vk::Semaphore],
) -> Option<CaptureSubmission> {
if ext.width == 0 || ext.height == 0 {
return None;
}
if !ds
.swapchain_transfer_src
.load(std::sync::atomic::Ordering::Relaxed)
{
return None;
}
// A command buffer may only be submitted to the family its pool was created
// for. An unknown queue means one the layer never saw through
// vkGetDeviceQueue, so there is nothing safe to assume about it.
let queue_family = *crate::state::QUEUE_TO_FAMILY.get(&queue.as_raw())?;
let mut ring_guard = ds.capture_ring.lock().ok()?;
if !unsafe {
ensure_capture_ring(
ds,
&mut ring_guard,
ext.width,
ext.height,
fmt,
queue_family,
image_count,
)
} {
return None;
}
let ring = ring_guard.as_mut()?;
let present_wait = unsafe { ensure_present_semaphore(ds, ring, image_index) }?;
// Never blocks: a frame with no free slot is one the encoder has not caught
// up with, and stalling the game's present to wait for it would be worse
// than skipping it.
let guard = ds.capture_slots.try_acquire()?;
let slot_index = guard.index();
let slot = ring.slots.get(slot_index)?;
let fence = slot.fence;
let fi = slot.image;
// A free slot's fence is already signalled — the encoder side waits on it
// before it ever reads the slot. This covers the paths that abandon a frame
// and return the slot without that wait.
unsafe {
if (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 2_000_000) != vk::Result::SUCCESS {
return None;
}
let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence);
}
// The recording depends on the source image, the destination image and the
// extent. A ring survives the swapchain shrinking, so the extent can move
// under recordings whose images are still valid — invalidate on it here
// rather than trusting every caller to have noticed.
if ring.blit_extent != ext {
ring.blits_recorded.iter_mut().for_each(|r| *r = false);
ring.blit_extent = ext;
}
let blit = crate::state::blit_index(image_index, slot_index, ring.image_count)?;
let cb = *ring.blits.get(blit)?;
if !ring.blits_recorded[blit] {
let pool = ring.timestamp_pool;
if !unsafe { record_blit(ds, cb, si, fi, ext, pool, slot_index) } {
return None;
}
ring.blits_recorded[blit] = true;
}
let wait_stages = vec![vk::PipelineStageFlags::TRANSFER; app_waits.len()];
let subi = vk::SubmitInfo {
@@ -840,6 +1235,21 @@ pub fn retire_present_semaphore(ds: &crate::state::DeviceState, image_index: usi
}
/// Set aside every per-image semaphore, for a swapchain that is going away.
/// Mark every recorded blit as needing re-recording.
///
/// Called when the swapchain is recreated. Each recording names a specific
/// source `VkImage`, and a recreated swapchain's images are new objects even
/// when the indices and the extent are unchanged — so submitting a recording
/// made against the old ones reads destroyed images.
pub fn invalidate_recorded_blits(ds: &crate::state::DeviceState) {
let Ok(mut ring_guard) = ds.capture_ring.lock() else {
return;
};
if let Some(ring) = ring_guard.as_mut() {
ring.blits_recorded.iter_mut().for_each(|r| *r = false);
}
}
pub fn retire_all_present_semaphores(ds: &crate::state::DeviceState) {
let Ok(mut ring_guard) = ds.capture_ring.lock() else {
return;

19
apps/nescapture/src/device.rs
View File

@@ -75,11 +75,16 @@ pub unsafe extern "system" fn vkCreateDevice(
const EXT_EXTERNAL_MEMORY: &[u8] = b"VK_KHR_external_memory\0";
const EXT_EXTERNAL_MEMORY_FD: &[u8] = b"VK_KHR_external_memory_fd\0";
const EXT_EXTERNAL_MEMORY_DMABUF: &[u8] = b"VK_EXT_external_memory_dma_buf\0";
// Lets the capture ring be allocated tiled. The importer has always created
// its side with DRM_FORMAT_MODIFIER_EXT tiling; without this the producer
// can only offer it a linear buffer.
const EXT_IMAGE_DRM_FORMAT_MODIFIER: &[u8] = b"VK_EXT_image_drm_format_modifier\0";
let needed: &[&[u8]] = &[
EXT_EXTERNAL_MEMORY,
EXT_EXTERNAL_MEMORY_FD,
EXT_EXTERNAL_MEMORY_DMABUF,
EXT_IMAGE_DRM_FORMAT_MODIFIER,
];
// Build extended list: original + any of ours not already present.
@@ -185,6 +190,14 @@ pub unsafe extern "system" fn vkCreateDevice(
cmd_copy_image: load!(b"vkCmdCopyImage\0"),
get_image_subresource_layout: load!(b"vkGetImageSubresourceLayout\0"),
get_memory_fd_khr: try_load!(b"vkGetMemoryFdKHR\0"),
get_image_drm_format_modifier_properties_ext: try_load!(
b"vkGetImageDrmFormatModifierPropertiesEXT\0"
),
create_query_pool: try_load!(b"vkCreateQueryPool\0"),
destroy_query_pool: try_load!(b"vkDestroyQueryPool\0"),
cmd_reset_query_pool: try_load!(b"vkCmdResetQueryPool\0"),
cmd_write_timestamp: try_load!(b"vkCmdWriteTimestamp\0"),
get_query_pool_results: try_load!(b"vkGetQueryPoolResults\0"),
// Phase 4 — synchronisation
create_fence: load!(b"vkCreateFence\0"),
@@ -205,6 +218,8 @@ pub unsafe extern "system" fn vkCreateDevice(
create_swapchain_khr: try_load!(b"vkCreateSwapchainKHR\0"),
destroy_swapchain_khr: try_load!(b"vkDestroySwapchainKHR\0"),
get_swapchain_images_khr: try_load!(b"vkGetSwapchainImagesKHR\0"),
acquire_next_image_khr: try_load!(b"vkAcquireNextImageKHR\0"),
acquire_next_image2_khr: try_load!(b"vkAcquireNextImage2KHR\0"),
// Phase 6 — draw commands
cmd_draw: load!(b"vkCmdDraw\0"),
@@ -274,7 +289,9 @@ pub unsafe extern "system" fn vkCreateDevice(
frame_gate: std::sync::Mutex::new(crate::pacing::FrameGate::from_env()),
capture_tx: std::sync::Mutex::new(None),
frame_pacer: std::sync::Mutex::new(crate::pacing::FramePacer::from_env()),
last_present_return: std::sync::Mutex::new(None),
encoder_starting: std::sync::atomic::AtomicBool::new(false),
});
DEVICE_STATE.insert(key, dev_state);

99
apps/nescapture/src/dispatch.rs
View File

@@ -190,6 +190,62 @@ pub type PFN_vkGetImageSubresourceLayout = unsafe extern "system" fn(
*mut vk::SubresourceLayout,
);
// DRM format modifiers. Both optional: without them the capture ring stays
// linear, which is what it was before it could be anything else.
pub type PFN_vkGetPhysicalDeviceFormatProperties2 = unsafe extern "system" fn(
vk::PhysicalDevice,
vk::Format,
*mut vk::FormatProperties2<'_>,
);
pub type PFN_vkGetImageDrmFormatModifierPropertiesEXT = unsafe extern "system" fn(
vk::Device,
vk::Image,
*mut vk::ImageDrmFormatModifierPropertiesEXT<'_>,
) -> vk::Result;
// Timestamp queries around the capture blit. All optional: without them the
// blit's GPU cost is simply not reported.
pub type PFN_vkGetPhysicalDeviceProperties =
unsafe extern "system" fn(vk::PhysicalDevice, *mut vk::PhysicalDeviceProperties);
pub type PFN_vkGetPhysicalDeviceQueueFamilyProperties =
unsafe extern "system" fn(vk::PhysicalDevice, *mut u32, *mut vk::QueueFamilyProperties);
pub type PFN_vkCreateQueryPool = unsafe extern "system" fn(
vk::Device,
*const vk::QueryPoolCreateInfo<'_>,
*const vk::AllocationCallbacks,
*mut vk::QueryPool,
) -> vk::Result;
pub type PFN_vkDestroyQueryPool = unsafe extern "system" fn(
vk::Device,
vk::QueryPool,
*const vk::AllocationCallbacks,
);
pub type PFN_vkCmdResetQueryPool =
unsafe extern "system" fn(vk::CommandBuffer, vk::QueryPool, u32, u32);
pub type PFN_vkCmdWriteTimestamp = unsafe extern "system" fn(
vk::CommandBuffer,
vk::PipelineStageFlags,
vk::QueryPool,
u32,
);
pub type PFN_vkGetQueryPoolResults = unsafe extern "system" fn(
vk::Device,
vk::QueryPool,
u32,
u32,
usize,
*mut std::ffi::c_void,
vk::DeviceSize,
vk::QueryResultFlags,
) -> vk::Result;
// DMA-BUF fd export (used to share final_image with pixelforge zero-copy)
pub type PFN_vkGetMemoryFdKHR = unsafe extern "system" fn(
vk::Device,
@@ -261,6 +317,21 @@ pub type PFN_vkDestroySwapchainKHR =
pub type PFN_vkGetSwapchainImagesKHR =
unsafe extern "system" fn(vk::Device, vk::SwapchainKHR, *mut u32, *mut vk::Image) -> vk::Result;
pub type PFN_vkAcquireNextImageKHR = unsafe extern "system" fn(
vk::Device,
vk::SwapchainKHR,
u64,
vk::Semaphore,
vk::Fence,
*mut u32,
) -> vk::Result;
pub type PFN_vkAcquireNextImage2KHR = unsafe extern "system" fn(
vk::Device,
*const vk::AcquireNextImageInfoKHR<'_>,
*mut u32,
) -> vk::Result;
// ── Phase 6: Draw commands ───────────────────────────────────────────────────
pub type PFN_vkCmdDraw = unsafe extern "system" fn(vk::CommandBuffer, u32, u32, u32, u32);
@@ -298,6 +369,16 @@ pub struct NextInstanceFn {
pub get_instance_proc_addr: PFN_vkGetInstanceProcAddr,
pub destroy_instance: PFN_vkDestroyInstance,
pub get_physical_device_memory_properties: PFN_vkGetPhysicalDeviceMemoryProperties,
/// `None` on an instance below Vulkan 1.1 without
/// `VK_KHR_get_physical_device_properties2`. Without it the modifier list
/// cannot be queried and the capture ring stays linear.
pub get_physical_device_format_properties2: Option<PFN_vkGetPhysicalDeviceFormatProperties2>,
/// Needed for `timestampPeriod`, which turns device ticks into nanoseconds.
pub get_physical_device_properties: Option<PFN_vkGetPhysicalDeviceProperties>,
/// Needed for a queue family's `timestampValidBits`. A family reporting
/// zero makes `vkCmdWriteTimestamp` illegal on it, so it has to be asked.
pub get_physical_device_queue_family_properties:
Option<PFN_vkGetPhysicalDeviceQueueFamilyProperties>,
pub create_device: PFN_vkCreateDevice,
}
@@ -343,6 +424,20 @@ pub struct NextDeviceFn {
/// `None` when `VK_KHR_external_memory_fd` is unavailable.
/// Required for DMA-BUF export to pixelforge's VkDevice.
pub get_memory_fd_khr: Option<PFN_vkGetMemoryFdKHR>,
/// `None` when `VK_EXT_image_drm_format_modifier` was not enabled. The
/// driver picks the modifier from the list it is offered, so this is how
/// the layer learns which one it actually got — and the importer needs the
/// exact value, not the list.
pub get_image_drm_format_modifier_properties_ext:
Option<PFN_vkGetImageDrmFormatModifierPropertiesEXT>,
// Phase 4 — blit timing. All-or-nothing: the ring only times the blit when
// every one of these loaded and the presenting queue family can timestamp.
pub create_query_pool: Option<PFN_vkCreateQueryPool>,
pub destroy_query_pool: Option<PFN_vkDestroyQueryPool>,
pub cmd_reset_query_pool: Option<PFN_vkCmdResetQueryPool>,
pub cmd_write_timestamp: Option<PFN_vkCmdWriteTimestamp>,
pub get_query_pool_results: Option<PFN_vkGetQueryPoolResults>,
// Phase 4 — synchronisation
pub create_fence: PFN_vkCreateFence,
@@ -364,6 +459,10 @@ pub struct NextDeviceFn {
pub create_swapchain_khr: Option<PFN_vkCreateSwapchainKHR>,
pub destroy_swapchain_khr: Option<PFN_vkDestroySwapchainKHR>,
pub get_swapchain_images_khr: Option<PFN_vkGetSwapchainImagesKHR>,
/// Both acquire entry points, hooked only to time them. A game uses one or
/// the other and the layer must not care which.
pub acquire_next_image_khr: Option<PFN_vkAcquireNextImageKHR>,
pub acquire_next_image2_khr: Option<PFN_vkAcquireNextImage2KHR>,
// Phase 6 — draw commands
pub cmd_draw: PFN_vkCmdDraw,

291
apps/nescapture/src/encode.rs
View File

@@ -172,7 +172,12 @@ pub fn output_format(pixel_fmt: PixelFormat, bit_depth: EncodeBitDepth) -> Outpu
// ── Captured frame (sent from present.rs to encoder thread) ──────────────────
pub struct CapturedFrame {
pub source: FrameSource,
/// Which device this frame's slot belongs to.
///
/// The encoder thread needs it to reach the fence and the exported fd: it
/// now does the waiting that a separate capture thread used to do, and that
/// work is per-device.
pub ds_key: usize,
pub width: u32,
pub height: u32,
pub vk_format: u32,
@@ -355,6 +360,8 @@ pub struct PipelineHandle {
pub dropped_frames: Arc<AtomicU32>,
pub present_attempts: Arc<AtomicU32>,
pub capture_attempts: Arc<AtomicU32>,
/// Where each present's time went, split three ways. Diagnostic.
pub timing: Arc<crate::timing::PresentTiming>,
}
impl PipelineHandle {
@@ -362,7 +369,10 @@ impl PipelineHandle {
let (codec, ctx) = resolve_codec(config.codec_request.as_deref())
.ok_or_else(|| "no hardware video encoder found on this GPU".to_string())?;
let (frame_tx, frame_rx) = mpsc::sync_channel::<CapturedFrame>(2);
// One deep. The frame in it is now an unwaited blit rather than an
// exported buffer, and the ring's four slots are already the
// backpressure — a second layer of queue only adds latency.
let (frame_tx, frame_rx) = mpsc::sync_channel::<CapturedFrame>(1);
let (encoded_tx, encoded_rx) = mpsc::sync_channel::<EncodedFrame>(2);
let (reconfig_tx, reconfig_rx) = mpsc::channel::<EncodeSettingsChange>();
let shutdown = Arc::new(AtomicBool::new(false));
@@ -373,6 +383,7 @@ impl PipelineHandle {
let dropped_frames = Arc::new(AtomicU32::new(0));
let present_attempts = Arc::new(AtomicU32::new(0));
let capture_attempts = Arc::new(AtomicU32::new(0));
let timing = Arc::new(crate::timing::PresentTiming::default());
let current_codec = Arc::new(AtomicU8::new(codec.to_protocol_codec()));
let needs_reconfig_flag = Arc::new(AtomicBool::new(false));
@@ -395,6 +406,7 @@ impl PipelineHandle {
current_codec: current_codec.clone(),
wanted_depth_override: None,
needs_reconfig_flag: needs_reconfig_flag.clone(),
capture_ms: capture_ms.clone(),
};
thread::Builder::new()
.name("nescapture-encoder".into())
@@ -426,6 +438,7 @@ impl PipelineHandle {
let stats_shutdown = shutdown.clone();
let pa = present_attempts.clone();
let ca = capture_attempts.clone();
let stats_timing = timing.clone();
thread::Builder::new()
.name("nescapture-stats".into())
.spawn(move || {
@@ -436,6 +449,7 @@ impl PipelineHandle {
stats_drop,
pa,
ca,
stats_timing,
stats_ipc,
stats_shutdown,
)
@@ -543,16 +557,26 @@ impl PipelineHandle {
dropped_frames,
present_attempts,
capture_attempts,
timing,
})
}
pub fn push_frame(&self, frame: CapturedFrame) -> bool {
self.capture_fps.fetch_add(1, Ordering::Relaxed);
let ok = self.frame_tx.try_send(frame).is_ok();
if !ok {
self.dropped_frames.fetch_add(1, Ordering::Relaxed);
// Counted on success only. It used to be incremented before the send,
// so a frame the channel refused was reported both as captured and as
// dropped, and the capture rate read as the rate the ring offered
// rather than the rate the encoder accepted — which is the number
// anyone reading it wants.
match self.frame_tx.try_send(frame) {
Ok(()) => {
self.capture_fps.fetch_add(1, Ordering::Relaxed);
true
}
Err(_) => {
self.dropped_frames.fetch_add(1, Ordering::Relaxed);
false
}
}
ok
}
pub fn shutdown(&self) {
@@ -589,6 +613,9 @@ struct EncoderConfig {
current_codec: Arc<AtomicU8>,
wanted_depth_override: Option<EncodeBitDepth>,
needs_reconfig_flag: Arc<AtomicBool>,
/// Present-to-encoder latency in milliseconds, as `f32` bits. Written here
/// now that this thread is the one doing the waiting.
capture_ms: Arc<AtomicU32>,
}
struct EncodedPacket {
@@ -676,12 +703,30 @@ fn encoder_thread(
cfg.idr_requested.store(true, Ordering::Relaxed);
}
let mut raw = match frame_rx.recv_timeout(std::time::Duration::from_millis(100)) {
let raw = match frame_rx.recv_timeout(std::time::Duration::from_millis(100)) {
Ok(frame) => frame,
Err(mpsc::RecvTimeoutError::Timeout) => continue,
Err(mpsc::RecvTimeoutError::Disconnected) => break,
};
// Wait for the blit and export the buffer. This used to be a thread of
// its own between the present hook and here; it is cheaper on this one,
// because the blit it waits for was submitted a frame earlier and has
// already completed, and every frame saves a channel and a wakeup.
let Some(ds) = crate::state::DEVICE_STATE.get(&raw.ds_key).map(|s| s.clone()) else {
log::error!("encoder: device state gone");
break;
};
let Some(mut source) = crate::present::resolve_source(&ds, &raw) else {
continue;
};
// Measured from the game's present, not from the top of this iteration:
// the wait above is part of what capture costs.
let capture_elapsed = raw.present_time.elapsed().as_secs_f32() * 1000.0;
cfg.capture_ms
.store(capture_elapsed.to_bits(), Ordering::Relaxed);
let Some(input_fmt) = vk_format_to_input_format(raw.vk_format) else {
// Drop the frame rather than encode it wrongly. Logged once per
// format so a persistent mismatch says so without filling the log
@@ -756,7 +801,7 @@ fn encoder_thread(
// first, for every frame after it.
let buffer_index = raw.slot.as_ref().map(|s| s.index()).unwrap_or(0);
let result = match &mut raw.source {
let result = match &mut source {
FrameSource::DmaBuf {
fd,
stride,
@@ -791,7 +836,7 @@ fn encoder_thread(
FrameSource::Pixels(pixels) => cpu_encode_frame(
&ctx,
&mut state.encoder,
pixels,
pixels.as_slice(),
raw.width,
raw.height,
raw.vk_format,
@@ -1141,16 +1186,46 @@ fn ipc_send_thread(
let mut last_warn = Instant::now();
let mut error_count: u64 = 0;
// Where this loop's time goes, per second.
//
// `encode` in the rate line is the encoder's own GPU time for whichever
// frame happened to be last, which is not the same thing as how long a
// frame took to get out of here. This loop is serial — wait for the
// encoder, build the IPC frame, write the socket — so a spike in any of
// the three delays every frame behind it. A client measured video
// datagrams stopping for 43 ms at a time while the present path stayed
// under 30 ms and audio was untouched, which puts the missing 13 ms
// somewhere in here, and averages cannot show which part.
let mut last_out = Instant::now();
let mut worst_out_gap = std::time::Duration::ZERO;
let mut worst_queued = std::time::Duration::ZERO;
let mut worst_awaited = std::time::Duration::ZERO;
let mut worst_send = std::time::Duration::ZERO;
let mut worst_key_wait = std::time::Duration::ZERO;
let mut keyframes: u32 = 0;
let mut last_pace = Instant::now();
loop {
if shutdown.load(Ordering::Relaxed) {
break 'outer;
}
let (result, present_time) =
match encoded_rx.recv_timeout(std::time::Duration::from_millis(100)) {
Ok(p) => (pollster::block_on(p.future), p.present_time),
Err(mpsc::RecvTimeoutError::Timeout) => continue,
Err(mpsc::RecvTimeoutError::Disconnected) => break 'outer,
};
// Timed apart, because they are different faults with different
// fixes and one number cannot tell them apart. `queued` is this
// loop waiting for the capture side to submit anything at all;
// `awaited` is the encoder finishing work already submitted. A
// single timer around both reported 40 ms and named neither.
let recv_start = Instant::now();
let pending = match encoded_rx.recv_timeout(std::time::Duration::from_millis(100)) {
Ok(p) => p,
Err(mpsc::RecvTimeoutError::Timeout) => continue,
Err(mpsc::RecvTimeoutError::Disconnected) => break 'outer,
};
let queued = recv_start.elapsed();
let present_time = pending.present_time;
let encode_start = Instant::now();
let result = pollster::block_on(pending.future);
let awaited = encode_start.elapsed();
let waited = queued + awaited;
let pkt = match result {
Ok(p) => p,
Err(e) => {
@@ -1190,6 +1265,14 @@ fn ipc_send_thread(
&pkt.data,
);
if pkt.is_key_frame {
keyframes += 1;
worst_key_wait = worst_key_wait.max(waited);
}
worst_queued = worst_queued.max(queued);
worst_awaited = worst_awaited.max(awaited);
let send_start = Instant::now();
if let Err(e) = socket.send(&ipc_frame) {
error_count += 1;
if last_warn.elapsed() > std::time::Duration::from_secs(5) {
@@ -1205,6 +1288,32 @@ fn ipc_send_thread(
break;
}
worst_send = worst_send.max(send_start.elapsed());
let out = Instant::now();
worst_out_gap = worst_out_gap.max(out.duration_since(last_out));
last_out = out;
if last_pace.elapsed() >= std::time::Duration::from_secs(1) {
last_pace = Instant::now();
log::info!(
" ipc: worst gap between frames out {:.1}ms = worst wait for a \
submission {:.1}ms + worst wait for the encoder {:.1}ms, worst \
socket send {:.1}ms, {keyframes} keyframe(s) (worst wait on one \
{:.1}ms)",
worst_out_gap.as_secs_f64() * 1000.0,
worst_queued.as_secs_f64() * 1000.0,
worst_awaited.as_secs_f64() * 1000.0,
worst_send.as_secs_f64() * 1000.0,
worst_key_wait.as_secs_f64() * 1000.0,
);
worst_out_gap = std::time::Duration::ZERO;
worst_queued = std::time::Duration::ZERO;
worst_awaited = std::time::Duration::ZERO;
worst_send = std::time::Duration::ZERO;
worst_key_wait = std::time::Duration::ZERO;
keyframes = 0;
}
frame_count += 1;
if frame_count % 300 == 0 {
log::trace!("IPC sent {frame_count} frames");
@@ -1215,6 +1324,34 @@ fn ipc_send_thread(
log::info!("IPC thread exited ({frame_count} frames)");
}
/// Microseconds this box spent stalled on one resource, cumulative since boot.
///
/// `/proc/pressure/<kind>` reports two totals: `some` is time at least one task
/// was blocked on the resource, `full` is time *every* runnable task was. For a
/// stall a player sees, `some` is the one that matters -- the render thread is
/// one task, and it being blocked is enough.
///
/// Returns `None` when the kernel was built without `CONFIG_PSI` or it is off,
/// which is a thing to say once rather than to retry every second.
fn pressure_total_us(kind: &str) -> Option<(u64, u64)> {
let text = std::fs::read_to_string(format!("/proc/pressure/{kind}")).ok()?;
let mut some = None;
let mut full = None;
for line in text.lines() {
let total = line
.split_whitespace()
.find_map(|f| f.strip_prefix("total="))
.and_then(|v| v.parse::<u64>().ok());
if line.starts_with("some") {
some = total;
} else if line.starts_with("full") {
full = total;
}
}
// `cpu` has no `full` line on most kernels, so its absence is not a failure.
Some((some?, full.unwrap_or(0)))
}
fn stats_sender_thread(
capture_fps: Arc<AtomicU32>,
encode_avg_ms: Arc<AtomicU32>,
@@ -1222,22 +1359,42 @@ fn stats_sender_thread(
dropped_frames: Arc<AtomicU32>,
present_attempts: Arc<AtomicU32>,
capture_attempts: Arc<AtomicU32>,
timing: Arc<crate::timing::PresentTiming>,
ipc_path: std::path::PathBuf,
shutdown: Arc<AtomicBool>,
) {
// Optional, where it used to end the thread. The socket only exists when a
// hub is listening, and this thread now also writes the per-second rate line
// that says where frames are going — which is wanted most in exactly the
// bare runs that have no hub.
let socket = match std::os::unix::net::UnixDatagram::unbound() {
Ok(s) => s,
Ok(s) if s.connect(&ipc_path).is_ok() => {
log::info!("stats sender → {}", ipc_path.display());
Some(s)
}
Ok(_) => {
log::warn!(
"stats socket connect failed; rates are logged but not sent to the hub"
);
None
}
Err(e) => {
log::error!("stats socket create: {e}");
return;
log::error!("stats socket create: {e}; rates are logged only");
None
}
};
if socket.connect(&ipc_path).is_err() {
log::warn!("stats socket connect failed, stats unavailable");
return;
}
log::info!("stats sender → {}", ipc_path.display());
// What the box itself was stalled on, second by second.
//
// Every stage of the pipeline has now been measured and is clean; what is
// left is the game's own frame time, which spikes to 29 ms about once a
// second in one title and never in another on the identical path. That is
// no longer a question about this code, and guessing at it from outside is
// how a week goes. The kernel already knows: PSI attributes a stall to cpu,
// io or memory, and the answer decides whether to look at the host's
// scheduling, the guest's storage, or its memory sizing.
let mut prev_pressure: Option<[(u64, u64); 3]> = None;
let mut pressure_said_missing = false;
loop {
if shutdown.load(Ordering::Relaxed) {
@@ -1255,9 +1412,89 @@ fn stats_sender_thread(
let pa = present_attempts.swap(0, Ordering::Relaxed);
let ca = capture_attempts.swap(0, Ordering::Relaxed);
let mut buf = Vec::with_capacity(22);
nesprotocol::stats::encode_hudless_stats(&mut buf, fps, enc_ms, dropped, pa, ca, cap_ms);
let _ = socket.send(&buf);
let starved = crate::slots::SLOT_STARVED.swap(0, Ordering::Relaxed);
// Logged as well as sent, because the socket goes to the desktop app
// and the question this answers is asked from inside the container.
// The three rates are the whole diagnosis: `present` is what the game
// produced, `admitted` is what the gate let through, and `encoded` is
// what reached the encoder. `present` below target means the game is
// the bottleneck and nothing here can help it; `admitted` above
// `encoded` with `starved` non-zero means the encoder is not returning
// slots fast enough and the capture rate follows it down.
// Where the present path's time went, for the second just ended. A
// hitch lands in exactly one of these three and that names its owner:
// `gap` is the game's own frame time with this layer excluded, `layer`
// is this layer's code on both sides of the down-call, `down` is the
// driver, WSI and compositor.
let (gap_avg, gap_max) = timing.gap.take();
let (layer_avg, layer_max) = timing.layer.take();
let (down_avg, down_max) = timing.down.take();
let (blit_avg, blit_max) = timing.blit.take();
let (acq_avg, acq_max) = timing.acquire.take();
let (hold_avg, hold_max) = timing.hold.take();
let long_gaps = timing.take_long_gaps();
log::info!(
"present {pa}/s, admitted {ca}/s, encoded {raw_fps}/s, \
starved {starved}, dropped {dropped}, capture {cap_ms:.1}ms, \
encode {enc_ms:.1}ms"
);
log::info!(
" gap {gap_avg:.1}/{gap_max:.1}ms, layer {layer_avg:.2}/{layer_max:.2}ms, \
down {down_avg:.2}/{down_max:.2}ms, acquire {acq_avg:.1}/{acq_max:.1}ms, \
hold {hold_avg:.2}/{hold_max:.2}ms, blit-gpu {blit_avg:.3}/{blit_max:.3}ms \
(avg/max), hitches {long_gaps}"
);
match ["cpu", "io", "memory"]
.iter()
.map(|k| pressure_total_us(k))
.collect::<Option<Vec<_>>>()
.and_then(|v| <[(u64, u64); 3]>::try_from(v.as_slice()).ok())
{
Some(now) => {
if let Some(before) = prev_pressure {
// Printed as milliseconds stalled in the second just ended,
// which is the same unit as everything else on these lines
// and directly comparable with `gap`.
let ms = |i: usize, full: bool| {
let (s_now, f_now) = now[i];
let (s_before, f_before) = before[i];
let (a, b) = if full {
(f_now, f_before)
} else {
(s_now, s_before)
};
a.saturating_sub(b) as f64 / 1000.0
};
log::info!(
" pressure: cpu {:.1}ms, io {:.1}/{:.1}ms, memory {:.1}/{:.1}ms \
(some/full, stalled in the last second)",
ms(0, false),
ms(1, false),
ms(1, true),
ms(2, false),
ms(2, true),
);
}
prev_pressure = Some(now);
}
None if !pressure_said_missing => {
log::info!(
" pressure: /proc/pressure is unreadable, so this guest cannot say \
whether a stall was cpu, io or memory (CONFIG_PSI off, or psi=0)"
);
pressure_said_missing = true;
}
None => {}
}
if let Some(ref socket) = socket {
let mut buf = Vec::with_capacity(22);
nesprotocol::stats::encode_hudless_stats(&mut buf, fps, enc_ms, dropped, pa, ca, cap_ms);
let _ = socket.send(&buf);
}
}
log::info!("stats sender exited");

17
apps/nescapture/src/instance.rs
View File

@@ -60,6 +60,23 @@ pub unsafe extern "system" fn vkCreateInstance(
b"vkGetPhysicalDeviceMemoryProperties\0",
)
},
get_physical_device_format_properties2: unsafe {
crate::try_load_instance_fn(
next_gipa,
instance,
b"vkGetPhysicalDeviceFormatProperties2\0",
)
},
get_physical_device_properties: unsafe {
crate::try_load_instance_fn(next_gipa, instance, b"vkGetPhysicalDeviceProperties\0")
},
get_physical_device_queue_family_properties: unsafe {
crate::try_load_instance_fn(
next_gipa,
instance,
b"vkGetPhysicalDeviceQueueFamilyProperties\0",
)
},
create_device: unsafe { load_instance_fn(next_gipa, instance, b"vkCreateDevice\0") },
});

32
apps/nescapture/src/lib.rs
View File

@@ -31,12 +31,15 @@ mod dmabuf_import;
mod encode;
mod framebuffer;
mod instance;
mod memory;
mod modifiers;
mod pacing;
mod pipeline;
mod present;
mod shader;
mod slots;
mod state;
mod timing;
mod swapchain;
use commands::{
@@ -54,7 +57,10 @@ use instance::{vkCreateInstance, vkDestroyInstance};
use pipeline::{vkCreateGraphicsPipelines, vkDestroyPipeline};
use present::vkQueuePresentKHR;
use shader::{vkCreateShaderModule, vkDestroyShaderModule};
use swapchain::{vkCreateSwapchainKHR, vkDestroySwapchainKHR, vkGetSwapchainImagesKHR};
use swapchain::{
vkAcquireNextImage2KHR, vkAcquireNextImageKHR, vkCreateSwapchainKHR, vkDestroySwapchainKHR,
vkGetSwapchainImagesKHR,
};
use dispatch::{PFN_vkGetDeviceProcAddr, PFN_vkGetInstanceProcAddr, RawFn};
use state::{DEVICE_STATE, INSTANCE_STATE};
@@ -154,6 +160,17 @@ pub(crate) unsafe fn try_load_device_fn<T>(
Some(unsafe { std::mem::transmute_copy(&raw) })
}
/// Like [`load_instance_fn`], but `None` rather than a panic when the entry
/// point is absent. For instance functions the layer can do without.
pub(crate) unsafe fn try_load_instance_fn<T>(
get: PFN_vkGetInstanceProcAddr,
instance: vk::Instance,
name: &[u8],
) -> Option<T> {
let raw = unsafe { get(instance, name.as_ptr() as *const c_char) }?;
Some(unsafe { std::mem::transmute_copy(&raw) })
}
pub(crate) unsafe fn load_instance_fn<T>(
get: PFN_vkGetInstanceProcAddr,
instance: vk::Instance,
@@ -313,6 +330,19 @@ unsafe fn match_device_fn(name: &[u8]) -> Option<RawFn> {
}
Some(to_raw(vkGetSwapchainImagesKHR as *const () as usize))
}
// Hooked only to time the wait. See `swapchain::record_acquire`.
b"vkAcquireNextImageKHR" => {
if log_this {
log::debug!("gdpa: → our vkAcquireNextImageKHR");
}
Some(to_raw(vkAcquireNextImageKHR as *const () as usize))
}
b"vkAcquireNextImage2KHR" => {
if log_this {
log::debug!("gdpa: → our vkAcquireNextImage2KHR");
}
Some(to_raw(vkAcquireNextImage2KHR as *const () as usize))
}
_ => {
if log_this {

128
apps/nescapture/src/memory.rs Normal file
View File

@@ -0,0 +1,128 @@
// ─────────────────────────────────────────────────────────────────────────────
// memory.rs — choosing a memory type for a capture slot
//
// Split out and made pure so the choice can be tested. It used to be a single
// "first type with HOST_VISIBLE | HOST_COHERENT" inside capture.rs, applied to
// every allocation including the exported ones — which on a discrete GPU put
// the capture ring in system RAM. The blit then wrote a full frame across the
// bus and the encoder read it back across the bus, sixty times a second, for
// a buffer neither side ever maps.
//
// Measured on the target (RX 9060 XT, RADV): memoryTypes[2] is the first
// HOST_VISIBLE|HOST_COHERENT type and sits on heapIndex 0, the 31 GiB heap
// with no DEVICE_LOCAL bit. memoryTypes[0] is DEVICE_LOCAL on heapIndex 1,
// the 16 GiB one. The old rule picked [2] every time.
// ─────────────────────────────────────────────────────────────────────────────
use ash::vk;
/// One entry of `VkPhysicalDeviceMemoryProperties::memoryTypes`.
#[derive(Clone, Copy, Debug)]
pub struct MemoryType {
pub flags: vk::MemoryPropertyFlags,
}
/// What the caller intends to do with the allocation.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Want {
/// The GPU writes it and another device imports it. Nothing maps it.
DeviceLocal,
/// The CPU reads it back. `read_frame_pixels` needs this.
HostCoherent,
}
/// Pick a memory type index from `types`, restricted to those set in `bits`.
///
/// `DeviceLocal` is a preference: a device with no device-local type the image
/// can use must still get an allocation, so it falls back to anything allowed.
/// `HostCoherent` is a requirement: memory that cannot be mapped cannot serve
/// the readback path at all, and handing it over would fault on the first
/// `vkMapMemory` rather than degrade.
pub fn pick_memory_type(types: &[MemoryType], bits: u32, want: Want) -> Option<u32> {
let allowed = |i: usize| bits & (1u32 << i) != 0;
match want {
Want::DeviceLocal => (0..types.len())
.find(|&i| allowed(i) && types[i].flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL))
.or_else(|| (0..types.len()).find(|&i| allowed(i)))
.map(|i| i as u32),
Want::HostCoherent => (0..types.len())
.find(|&i| {
allowed(i)
&& types[i].flags.contains(
vk::MemoryPropertyFlags::HOST_VISIBLE
| vk::MemoryPropertyFlags::HOST_COHERENT,
)
})
.map(|i| i as u32),
}
}
#[cfg(test)]
mod tests {
use super::*;
use ash::vk::MemoryPropertyFlags as F;
fn t(flags: F) -> MemoryType {
MemoryType { flags }
}
/// The target's actual layout, trimmed to the types that matter: a
/// device-local type first, a host-coherent one on the system heap after
/// it. The old rule scanned for host-coherent and took the second.
#[test]
fn device_local_is_preferred_over_a_host_type() {
let types = [t(F::DEVICE_LOCAL), t(F::HOST_VISIBLE | F::HOST_COHERENT)];
assert_eq!(pick_memory_type(&types, 0b11, Want::DeviceLocal), Some(0));
}
/// Order must not decide it. Same two types, host-coherent first.
#[test]
fn device_local_is_preferred_even_when_it_comes_second() {
let types = [t(F::HOST_VISIBLE | F::HOST_COHERENT), t(F::DEVICE_LOCAL)];
assert_eq!(pick_memory_type(&types, 0b11, Want::DeviceLocal), Some(1));
}
/// A type the image's `memoryTypeBits` excludes may not be chosen, however
/// well it matches. Binding an image to a type it forbids is invalid.
#[test]
fn a_type_the_image_forbids_is_never_chosen() {
let types = [t(F::HOST_VISIBLE | F::HOST_COHERENT), t(F::DEVICE_LOCAL)];
assert_eq!(pick_memory_type(&types, 0b01, Want::DeviceLocal), Some(0));
}
/// No device-local type the image can use is not a failure: the allocation
/// still has to happen, just without the preference.
#[test]
fn device_local_falls_back_to_whatever_is_allowed() {
let types = [t(F::HOST_VISIBLE | F::HOST_COHERENT)];
assert_eq!(pick_memory_type(&types, 0b1, Want::DeviceLocal), Some(0));
}
/// Readback has no fallback.
#[test]
fn host_coherent_has_no_fallback() {
let types = [t(F::DEVICE_LOCAL)];
assert_eq!(pick_memory_type(&types, 0b1, Want::HostCoherent), None);
}
/// A device-local *and* host-visible type still satisfies readback. RADV
/// offers one (the ReBAR window) and refusing it would be wrong.
#[test]
fn host_coherent_accepts_a_device_local_type_that_is_also_mappable() {
let types = [
t(F::DEVICE_LOCAL),
t(F::DEVICE_LOCAL | F::HOST_VISIBLE | F::HOST_COHERENT),
];
assert_eq!(pick_memory_type(&types, 0b11, Want::HostCoherent), Some(1));
}
/// Nothing allowed at all is None rather than index zero. Falling back to
/// zero was the old behaviour and it binds to a type the image forbids.
#[test]
fn no_allowed_type_is_none() {
let types = [t(F::DEVICE_LOCAL), t(F::HOST_VISIBLE | F::HOST_COHERENT)];
assert_eq!(pick_memory_type(&types, 0, Want::DeviceLocal), None);
assert_eq!(pick_memory_type(&types, 0, Want::HostCoherent), None);
}
}

112
apps/nescapture/src/modifiers.rs Normal file
View File

@@ -0,0 +1,112 @@
// ─────────────────────────────────────────────────────────────────────────────
// modifiers.rs — choosing a DRM format modifier for the capture ring
//
// The importer has always been able to take a tiled buffer: `dmabuf_import.rs`
// builds `VkImageDrmFormatModifierExplicitCreateInfoEXT` with per-plane
// layouts and creates the image with `DRM_FORMAT_MODIFIER_EXT` tiling. Only
// the producer was linear — hard-coded `ImageTiling::LINEAR` and a `modifier`
// of zero passed down with every frame — so every capture detiled a full frame
// on the write and the encoder sampled a linear image on the read.
//
// Picking the modifier is the whole of the decision and it is pure, so it is
// here and tested rather than buried in an unsafe block.
// ─────────────────────────────────────────────────────────────────────────────
/// One entry of `VkDrmFormatModifierPropertiesListEXT`, already filtered to
/// modifiers whose tiling features cover both our write and the encoder's read.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ModifierProps {
pub modifier: u64,
pub plane_count: u32,
}
/// `DRM_FORMAT_MOD_LINEAR`.
pub const LINEAR: u64 = 0;
/// `DRM_FORMAT_MOD_INVALID`, which a driver may list and which means "let the
/// driver choose" — not something to ask for explicitly.
pub const INVALID: u64 = 0x00ff_ffff_ffff_ffff;
/// Pick the modifier to allocate the capture ring with.
///
/// Single-plane only, and deliberately so. A multi-plane modifier needs an
/// offset and a stride per plane on the import side, and the export path hands
/// out one fd with one stride — so accepting one would produce an image the far
/// side reads at the wrong offsets, which arrives at the right size and frame
/// rate carrying nonsense. Prefer any real tiled modifier; fall back to linear,
/// which is what the ring used before this existed and always works.
pub fn pick_modifier(candidates: &[ModifierProps]) -> Option<ModifierProps> {
let usable = |m: &&ModifierProps| m.plane_count == 1 && m.modifier != INVALID;
candidates
.iter()
.filter(usable)
.find(|m| m.modifier != LINEAR)
.or_else(|| candidates.iter().filter(usable).find(|m| m.modifier == LINEAR))
.copied()
}
#[cfg(test)]
mod tests {
use super::*;
const TILED: u64 = 0x0200_0000_0000_0001;
#[test]
fn a_tiled_modifier_beats_linear() {
let c = [
ModifierProps { modifier: LINEAR, plane_count: 1 },
ModifierProps { modifier: TILED, plane_count: 1 },
];
assert_eq!(pick_modifier(&c).unwrap().modifier, TILED);
}
/// Order must not decide it — the driver lists them in its own order.
#[test]
fn a_tiled_modifier_wins_from_either_position() {
let c = [
ModifierProps { modifier: TILED, plane_count: 1 },
ModifierProps { modifier: LINEAR, plane_count: 1 },
];
assert_eq!(pick_modifier(&c).unwrap().modifier, TILED);
}
/// A multi-plane modifier with one exported fd would be imported at the
/// wrong plane offsets and produce a corrupt frame rather than an error.
#[test]
fn multi_plane_modifiers_are_refused() {
let c = [
ModifierProps { modifier: 0x0200_0000_0000_0002, plane_count: 2 },
ModifierProps { modifier: LINEAR, plane_count: 1 },
];
assert_eq!(pick_modifier(&c).unwrap().modifier, LINEAR);
}
/// A multi-plane tiled modifier must not beat a single-plane linear one
/// just for being tiled.
#[test]
fn tiling_does_not_excuse_a_plane_count_we_cannot_export() {
let c = [
ModifierProps { modifier: TILED, plane_count: 4 },
ModifierProps { modifier: LINEAR, plane_count: 1 },
];
assert_eq!(pick_modifier(&c).unwrap().modifier, LINEAR);
}
/// `DRM_FORMAT_MOD_INVALID` is not a modifier to ask for.
#[test]
fn the_invalid_modifier_is_never_chosen() {
let c = [ModifierProps { modifier: INVALID, plane_count: 1 }];
assert_eq!(pick_modifier(&c), None);
}
#[test]
fn nothing_usable_is_none() {
let c = [ModifierProps { modifier: TILED, plane_count: 4 }];
assert_eq!(pick_modifier(&c), None);
}
#[test]
fn an_empty_list_is_none() {
assert_eq!(pick_modifier(&[]), None);
}
}

512
apps/nescapture/src/pacing.rs
View File

@@ -2,56 +2,130 @@
// pacing.rs — capture frame-rate gate
//
// The layer is the only place that sees every frame the game produces, so it
// is the only place that can decide which ones are worth capturing. Nothing
// here consults the compositor: no vblank, no surface, no present feedback,
// just the monotonic clock. nescope runs uncapped by design and must stay out
// of this decision.
// is the only place that can pace one. Nothing here consults the compositor:
// no vblank, no surface, no present feedback, just the monotonic clock.
//
// The gate only ever *drops*. It never waits, never blocks the game's present,
// and never admits more frames than the game offered. A game running below the
// target is passed through untouched.
// Two jobs, deliberately apart:
//
// FrameGate — which frames are captured. Only ever *drops*: it never
// waits, never blocks the game's present, and never admits
// more frames than the game offered.
// FramePacer — how fast the game may run. This one does block, at the end
// of the present hook, which is the only thing in the process
// that can hold a game whose V-Sync is off.
//
// Both are needed. The pacer holds the game to the target, and the gate is the
// backstop for what a sleep cannot promise: `thread::sleep` lands within a
// fraction of a millisecond, not exactly, so frames still arrive early.
//
// A game running below the target is untouched by either — including, and this
// is the whole difficulty, a game whose average is below the target but which
// delivers its frames in fast runs separated by stalls. That is what a
// CPU-starved game looks like, and it is the case the first gate got wrong.
//
// One correction worth recording: this file used to say nescope "runs uncapped
// by design". It does not. Its `--fps` defaults to 60 and drives both the
// advertised output refresh and the `wl_surface.frame` cadence, and nothing
// passes a tier's rate down to it.
// ─────────────────────────────────────────────────────────────────────────────
use std::time::{Duration, Instant};
/// Admits at most `target_fps` frames per second, dropping the rest.
///
/// The deadline advances by a fixed interval rather than being recomputed from
/// the admitted frame's arrival time, so a game presenting slightly off-cadence
/// does not accumulate drift.
/// A token bucket, not a deadline. The first version compared each present
/// against a fixed deadline and, after a stall, dropped whatever debt had built
/// up so that a resuming game could not replay it as a burst. That is right for
/// a game coming back from a load screen and wrong for one that micro-stalls
/// constantly: the fast runs between stalls were thinned to the target while
/// the stalls themselves were never made up, so a game averaging fifty frames a
/// second lost a fifth of them to a gate set to sixty. Measured on the target
/// as `present 50/s, admitted 43/s`.
///
/// Credit accrues with real time and is spent one unit per admitted frame, so
/// over any window the admitted rate is the smaller of the game's rate and the
/// target — which is the property that was wanted all along. The cap on
/// accumulated credit is what keeps a long stall from banking a replay.
pub struct FrameGate {
/// Zero means uncapped — every frame is admitted.
interval: Duration,
/// Frames arriving within this much of the deadline are admitted early.
/// Credit available, in frames. One is spent per admitted frame.
credit: f64,
/// The most credit that may accumulate, in frames.
///
/// Without it, a game running at exactly the target rate beats against the
/// gate: presents land a hair before each deadline, get rejected, and the
/// stream loses a frame every time the two rates drift past each other.
slack: Duration,
/// `None` until the first frame establishes the cadence.
next_deadline: Option<Instant>,
/// This is the whole of the stall policy, and [`FrameGate::BURST_WINDOW`]
/// sets it: the gate enforces the target over any window longer than that
/// and leaves shorter ones alone.
burst: f64,
/// When the previous present arrived. `None` until the first.
last: Option<Instant>,
}
impl FrameGate {
/// How far behind the target a game may fall and still catch up.
///
/// Equivalently: the gate enforces the target rate averaged over any window
/// longer than this and does not constrain shorter ones. It has to exceed
/// the longest stall the game takes mid-scene, because credit stops
/// accruing once the bucket is full, and every millisecond of silence after
/// that is a frame the game went on to present and this gate refused.
///
/// It was `CAPTURE_SLOTS` — four frames, 67ms — on the reasoning that
/// banking more bought frames the ring could not hold. That reasoning was
/// wrong. A burst after a stall arrives serially, as fast as the game
/// presents it, not all at once, and the ring drains a frame every three
/// milliseconds; the ring was never the constraint. Measured on the target,
/// stalls reach 101ms, and against a 67ms bucket a game offering 59 frames
/// in a second had 52 of them taken.
const BURST_WINDOW: Duration = Duration::from_millis(250);
pub fn new(target_fps: u32) -> Self {
let interval = if target_fps == 0 {
Duration::ZERO
} else {
Duration::from_nanos(1_000_000_000 / u64::from(target_fps))
};
let burst = if interval.is_zero() {
0.0
} else {
// Never below one whole frame. The window is a duration, so below
// four frames a second it works out at less than a single token —
// and since credit is capped at it, the bucket could then never
// hold enough to admit anything at all. At `NESCAPTURE_FPS=1` that
// is a layer that captures nothing, builds no encoder, opens no
// sockets, and says none of it.
(Self::BURST_WINDOW.as_secs_f64() / interval.as_secs_f64()).max(1.0)
};
Self {
interval,
slack: interval / 8,
next_deadline: None,
// Nearly empty, not full. One unit admits the first frame, and the
// second absorbs a frame of jitter before any history exists.
// Starting full would hand a fast game a quarter second of free
// frames and make its first reported second read seventy-odd
// against a gate set to sixty — a burst in the one measurement
// this is read by.
credit: 2.0_f64.min(burst),
burst,
last: None,
}
}
/// Read the target from the environment. `0` disables the gate.
pub fn from_env() -> Self {
let fps = std::env::var("NESCAPTURE_FPS")
let raw = std::env::var("NESCAPTURE_FPS");
let fps = raw
.as_deref()
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(60);
// Said out loud, because every other symptom of getting this wrong is
// silence. A layer capturing at the wrong rate, or at a rate somebody
// thought they had overridden and had not, looks exactly like a layer
// working.
match raw.as_deref() {
Ok(value) => log::info!("capture rate: {fps} fps (NESCAPTURE_FPS={value})"),
Err(_) => log::info!("capture rate: {fps} fps (NESCAPTURE_FPS unset)"),
}
Self::new(fps)
}
@@ -69,30 +143,119 @@ impl FrameGate {
if self.interval.is_zero() {
return true;
}
let Some(deadline) = self.next_deadline else {
self.next_deadline = Some(now + self.interval);
return true;
// `saturating_duration_since` because `now` comes from the caller and a
// present arriving out of order would otherwise panic. Zero elapsed
// simply earns no credit.
let elapsed = match self.last {
Some(last) => now.saturating_duration_since(last),
None => Duration::ZERO,
};
if now + self.slack < deadline {
return false;
self.last = Some(now);
let earned = elapsed.as_secs_f64() / self.interval.as_secs_f64();
self.credit = (self.credit + earned).min(self.burst);
if self.credit >= 1.0 {
self.credit -= 1.0;
true
} else {
false
}
}
}
// Advance one interval from the deadline, not from `now`, so a game
// presenting slightly early or late keeps an exact cadence.
let advanced = deadline + self.interval;
/// Holds the game to the target rate, by delaying its present's *return*.
///
/// [`FrameGate`] decides which frames are captured and never touches the game;
/// this decides how fast the game is allowed to run. They are separate because
/// they answer separate questions, and because a game paced to the target still
/// needs the gate as a backstop: `thread::sleep` lands within a fraction of a
/// millisecond, not exactly, so a frame occasionally arrives early.
///
/// # Why the layer and not the compositor
///
/// A compositor cannot pace a game that has V-Sync off. `IMMEDIATE` and
/// `MAILBOX` swapchains do not wait on `wl_surface.frame` — that callback is
/// FIFO's throttle and nothing else's — so a player turning V-Sync off leaves
/// the compositor with no lever at all. The only one it has left is withholding
/// `wl_buffer.release` to starve the swapchain, which stalls the game inside
/// `vkAcquireNextImageKHR` at a depth the *application* chose when it picked an
/// image count. This process sees every present and owns a monotonic clock,
/// which is the whole of what pacing needs.
///
/// # Why the hold goes after the present, not before it
///
/// Sleeping before calling down delays the frame reaching the screen, which is
/// latency added to a frame that was ready. Sleeping after it means the frame
/// went out the moment it was ready and the application is merely held back
/// from *starting* the next one. Same cadence, no added latency — which is
/// where every other frame limiter puts it.
pub struct FramePacer {
/// Zero means no limiting — the game runs as fast as it can.
interval: Duration,
/// When the next present may return. `None` until the first one does.
due: Option<Instant>,
}
// Slip clamp. If the game stalled — a load screen, a shader compile, a
// hitch — the deadline can end up many intervals in the past. Advancing
// by one interval at a time would leave the gate "owing" frames and it
// would admit a burst of them back to back the moment the game resumes,
// which is exactly when the GPU can least afford it. Drop the debt and
// restart the cadence from now.
self.next_deadline = Some(if advanced <= now {
impl FramePacer {
pub fn new(target_fps: u32) -> Self {
Self {
interval: if target_fps == 0 {
Duration::ZERO
} else {
Duration::from_nanos(1_000_000_000 / u64::from(target_fps))
},
due: None,
}
}
/// Read the target from the environment.
///
/// The same `NESCAPTURE_FPS` the gate reads, because a capture rate and a
/// game rate that disagree is a stream sending frames nobody asked for or
/// dropping frames somebody paid to render. `NESCAPTURE_LIMIT=0` leaves the
/// game alone and captures at the rate anyway, which is how the capture
/// path's cost is measured without the limiter hiding it.
pub fn from_env() -> Self {
let limit = std::env::var("NESCAPTURE_LIMIT")
.map(|v| v != "0")
.unwrap_or(true);
if !limit {
log::info!("game frame limiting off (NESCAPTURE_LIMIT=0)");
return Self::new(0);
}
let fps = std::env::var("NESCAPTURE_FPS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(60);
Self::new(fps)
}
/// How long to hold the application before its present may return.
///
/// Pure, and returns the duration rather than sleeping, so the cadence can
/// be tested without a clock that really waits. Zero whenever the game is
/// already at or below the target: a slow game is never made slower.
pub fn hold(&mut self, now: Instant) -> Duration {
if self.interval.is_zero() {
return Duration::ZERO;
}
let slot = self.due.unwrap_or(now);
let wait = slot.saturating_duration_since(now);
// Advance from the slot, not from `now`, so a game presenting a hair
// early or late keeps an exact cadence rather than drifting. The clamp
// is for a real stall: a slot far enough in the past would otherwise
// let the game run a burst of frames back to back to "catch up", and
// the frames it would be catching up on were never rendered.
let next = slot + self.interval;
self.due = Some(if next <= now {
now + self.interval
} else {
advanced
next
});
true
wait
}
}
@@ -114,11 +277,27 @@ mod tests {
#[test]
fn fast_game_is_thinned_to_the_target() {
// 300 fps offered, 60 wanted: one frame in five, over a full second.
// The bucket starts full, so the first few come through back to back —
// hence the allowance above 60 rather than a tight band.
let mut g = FrameGate::new(60);
let t0 = Instant::now();
let step = Duration::from_nanos(3_333_333);
let admitted = (0..300).filter(|i| g.admit(t0 + step * *i)).count();
assert!((59..=61).contains(&admitted), "admitted {admitted}");
assert!((59..=65).contains(&admitted), "admitted {admitted}");
}
/// Over a long enough run the starting credit stops mattering and the rate
/// is the target, which is the property the gate exists for.
#[test]
fn a_fast_game_settles_at_the_target_over_ten_seconds() {
let mut g = FrameGate::new(60);
let t0 = Instant::now();
let step = Duration::from_nanos(3_333_333);
let admitted = (0..3000).filter(|i| g.admit(t0 + step * *i)).count();
assert!(
(595..=610).contains(&admitted),
"admitted {admitted} in 10s"
);
}
#[test]
@@ -150,23 +329,262 @@ mod tests {
}
}
/// The case the deadline version got wrong, and the reason this is a
/// bucket. A CPU-starved game presents in fast runs broken by micro-stalls:
/// four frames at 12ms, then a 52ms gap — five frames per 100ms, fifty a
/// second, well under the sixty the gate allows. Every one must survive.
/// The old gate dropped a fifth of them, which is what
/// `present 50/s, admitted 43/s` was on the target.
#[test]
fn a_stall_does_not_bank_a_burst() {
fn a_stuttering_game_below_the_target_keeps_every_frame() {
let mut g = FrameGate::new(60);
let mut at = Instant::now();
let steps = [
Duration::from_millis(12),
Duration::from_millis(12),
Duration::from_millis(12),
Duration::from_millis(12),
Duration::from_millis(52),
];
let mut admitted = 0;
for i in 0..200 {
at += steps[i % steps.len()];
if g.admit(at) {
admitted += 1;
}
}
assert_eq!(
admitted,
200,
"dropped {} frames of a 50 fps game",
200 - admitted
);
}
/// The stall the target actually exhibits: gaps up to 101ms, in a second
/// where the game still offers fewer frames than the gate allows. Every one
/// must survive. At the old 67ms bucket this lost seven frames in sixty —
/// `present 59/s, admitted 52/s` on the target.
#[test]
fn a_hundred_millisecond_stall_costs_no_frames() {
let mut g = FrameGate::new(60);
let mut at = Instant::now();
let mut admitted = 0;
let mut offered = 0;
// Ten frames at 11ms, then 100ms of silence: 10 frames per 210ms, about
// 48 a second, well inside a gate set to 60.
let step = |i: usize| {
if i % 11 == 10 {
Duration::from_millis(100)
} else {
Duration::from_millis(11)
}
};
// The bucket starts nearly empty, so the first second of a game faster
// than the target loses a handful of frames while it fills. That is the
// deliberate trade for not bursting at startup, and it is not what this
// is testing — the claim is about steady state, so warm up first.
for i in 0..100 {
at += step(i);
g.admit(at);
}
for i in 100..400 {
at += step(i);
offered += 1;
if g.admit(at) {
admitted += 1;
}
}
assert_eq!(
admitted,
offered,
"dropped {} frames of a stalling sub-target game",
offered - admitted
);
}
/// The bucket bridges a stall; it does not raise the ceiling. A game that
/// is genuinely faster than the target over a long run is still thinned to
/// it, however it distributes its frames.
#[test]
fn a_sustained_fast_game_is_still_held_to_the_target() {
let mut g = FrameGate::new(60);
let mut at = Instant::now();
let mut admitted = 0;
// 120 fps in bursts of eight with a 20ms pause: ~8 frames per 76ms,
// about 105 a second, sustained for ten seconds.
for i in 0..1000 {
at += if i % 9 == 8 {
Duration::from_millis(20)
} else {
Duration::from_millis(8)
};
if g.admit(at) {
admitted += 1;
}
}
let seconds = 9.5;
let rate = admitted as f64 / seconds;
assert!(
rate < 66.0,
"admitted {admitted} frames, about {rate:.0}/s, from a gate set to 60"
);
}
/// A target low enough that a quarter-second window is less than one
/// frame. The burst is a duration, so at 1 fps it works out at 0.25 of a
/// token — and with credit capped at the burst, the bucket could never hold
/// the whole token an admission costs. The gate admitted nothing, ever.
///
/// What that looked like was not a slow stream. Nothing is captured, so the
/// encode pipeline is never built, no IPC socket is opened and no stats are
/// logged: the layer loads, reports a swapchain, and goes quiet.
#[test]
fn a_target_below_the_burst_window_still_admits_frames() {
for fps in [1, 2, 3, 4, 5] {
let mut g = FrameGate::new(fps);
let t0 = Instant::now();
let interval = Duration::from_secs(1) / fps;
let admitted = (0..20).filter(|i| g.admit(t0 + interval * *i)).count();
assert!(
admitted >= 19,
"at {fps} fps a game presenting at exactly that rate had \
{admitted} of 20 frames taken"
);
}
}
#[test]
fn a_stall_does_not_bank_an_unbounded_burst() {
let mut g = FrameGate::new(60);
let t0 = Instant::now();
assert!(g.admit(t0));
// Two seconds of nothing — a load screen.
// Two seconds of nothing — a load screen, not a hitch.
let resume = t0 + Duration::from_secs(2);
// The first frame back is admitted immediately...
assert!(g.admit(resume));
// ...but the game resuming at 300 fps must not replay the ~120 frames
// the gate "missed" during the stall. Across the next 100ms it may
// admit only what 60 fps allows: about six, not all thirty offered.
// The game resuming at 300 fps must not replay the ~120 frames the gate
// "missed". The bucket is a quarter second deep, so what comes through
// is that plus what the elapsed time earns — well short of everything
// on offer.
let step = Duration::from_nanos(3_333_333);
let burst = (1..30).filter(|i| g.admit(resume + step * *i)).count();
assert!((4..=7).contains(&burst), "admitted {burst} frames in 100ms");
let burst = (1..120).filter(|i| g.admit(resume + step * *i)).count();
assert!(
burst < 45,
"admitted {burst} frames in 400ms after a two-second stall"
);
}
/// Credit is bounded however long the stall, so the burst after one does
/// not grow with it. This is what stops a load screen becoming a replay.
#[test]
fn a_longer_stall_does_not_bank_a_deeper_burst() {
let after = |stall: Duration| {
let mut g = FrameGate::new(60);
let t0 = Instant::now();
g.admit(t0);
let resume = t0 + stall;
let step = Duration::from_nanos(3_333_333);
(0..120).filter(|i| g.admit(resume + step * *i)).count()
};
assert_eq!(
after(Duration::from_secs(2)),
after(Duration::from_secs(600)),
"a longer stall banked a deeper burst"
);
}
}
#[cfg(test)]
mod pacer_tests {
use super::*;
/// Run a game that renders each frame in `render`, for `frames` frames,
/// obeying whatever hold the pacer asks for. Returns the wall time taken.
fn run(pacer: &mut FramePacer, frames: u32, render: Duration) -> Duration {
let start = Instant::now();
let mut now = start;
for _ in 0..frames {
now += render;
now += pacer.hold(now);
}
now.saturating_duration_since(start)
}
#[test]
fn no_target_never_holds() {
let mut p = FramePacer::new(0);
let now = Instant::now();
for i in 0..100 {
assert_eq!(p.hold(now + Duration::from_micros(i)), Duration::ZERO);
}
}
/// A game that could render at 500 fps is held to 60: sixty frames take
/// about a second, not a tenth of one.
#[test]
fn a_fast_game_is_held_to_the_target() {
let mut p = FramePacer::new(60);
let took = run(&mut p, 60, Duration::from_millis(2));
assert!(
took >= Duration::from_millis(970) && took <= Duration::from_millis(1030),
"sixty frames at a 60 fps limit took {took:?}"
);
}
/// A game slower than the target is never delayed. Holding one would be
/// this layer making a struggling game slower still.
#[test]
fn a_slow_game_is_never_held() {
let mut p = FramePacer::new(120);
let mut now = Instant::now();
for i in 0..60 {
// 40 fps against a 120 fps limit.
now += Duration::from_millis(25);
assert_eq!(
p.hold(now),
Duration::ZERO,
"frame {i} of a 40 fps game was held against a 120 fps limit"
);
}
}
/// After a stall, the frames that were never rendered are not owed back.
/// Advancing one interval at a time would let the game run flat out until
/// it had "caught up" on frames that do not exist.
#[test]
fn a_stall_is_not_repaid_with_a_burst() {
let mut p = FramePacer::new(60);
let t0 = Instant::now();
p.hold(t0);
// Two seconds gone — a load screen.
let resume = t0 + Duration::from_secs(2);
assert_eq!(
p.hold(resume),
Duration::ZERO,
"the first frame back waited"
);
// And the next frame is paced normally rather than let through free.
let next = resume + Duration::from_millis(2);
let held = p.hold(next);
assert!(
held >= Duration::from_millis(13),
"the frame after a stall was held only {held:?}"
);
}
/// The rate holds over a long run, which is the property that matters —
/// per-frame exactness is not something a sleep can promise.
#[test]
fn the_rate_holds_over_ten_seconds() {
let mut p = FramePacer::new(120);
let took = run(&mut p, 1200, Duration::from_micros(500));
assert!(
took >= Duration::from_millis(9_900) && took <= Duration::from_millis(10_100),
"1200 frames at a 120 fps limit took {took:?}"
);
}
}

338
apps/nescapture/src/present.rs
View File

@@ -5,22 +5,6 @@ use crate::state::{DEVICE_STATE, QUEUE_TO_DEVICE_KEY};
use ash::vk::{self, Handle};
use std::os::raw::c_void;
use std::sync::atomic::Ordering;
use std::sync::mpsc;
/// A frame already blitted into a ring slot, waiting to be handed to the
/// encoder. The GPU work is submitted before this is queued, so the worker's
/// only job is to wait for it and export the buffer.
pub struct CaptureJob {
pub ds_key: usize,
/// Holds the ring slot until the encoder is finished with it.
pub slot: SlotGuard,
pub width: u32,
pub height: u32,
pub sc_fmt: vk::Format,
/// When the game handed this frame to `vkQueuePresentKHR`. The only honest
/// capture time — everything downstream is queued behind something.
pub present_time: std::time::Instant,
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkQueuePresentKHR(
@@ -44,15 +28,25 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
None => return vk::Result::ERROR_DEVICE_LOST,
};
// Entry. `gap` is measured from where the previous present *returned*, so
// it is the game's own frame time with this layer's cost excluded — the
// three spans then partition the wall clock between presents exactly.
let entered = std::time::Instant::now();
ds.frame_counter.fetch_add(1, Ordering::Relaxed);
ds.hud_detected_frame.store(false, Ordering::Relaxed);
ds.pending_capture_frame.store(false, Ordering::Relaxed);
ds.capture_injected_frame.store(false, Ordering::Relaxed);
ds.skipped_draws_frame.store(0, Ordering::Relaxed);
if let Ok(enc) = ds.encoder.lock() {
if let Some(ref h) = *enc {
h.present_attempts.fetch_add(1, Ordering::Relaxed);
if let Ok(enc) = ds.encoder.lock()
&& let Some(ref h) = *enc
{
h.present_attempts.fetch_add(1, Ordering::Relaxed);
if let Ok(prev) = ds.last_present_return.lock()
&& let Some(prev) = *prev
{
h.timing.record_gap(entered.saturating_duration_since(prev));
}
}
@@ -71,15 +65,24 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
None
};
let down_us = std::cell::Cell::new(std::time::Duration::ZERO);
let call_down = |info: *const vk::PresentInfoKHR| match ds.fp.queue_present_khr {
Some(f) => unsafe { f(queue, info) },
Some(f) => {
let t = std::time::Instant::now();
let r = unsafe { f(queue, info) };
down_us.set(t.elapsed());
r
}
None => vk::Result::ERROR_EXTENSION_NOT_PRESENT,
};
let Some(submission) = submission else {
return call_down(p_present_info);
let r = call_down(p_present_info);
finish(&ds, entered, down_us.get());
return r;
};
// The blit consumed the application's wait semaphores, so the present waits
// on ours instead. Presenting on the originals as well would be a second
// wait on an already-consumed signal.
@@ -109,9 +112,63 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
capture::retire_present_semaphore(&ds, image_index);
}
finish(&ds, entered, down_us.get());
result
}
/// Close out a present: record what this layer cost and stamp the return.
///
/// `layer` is everything in this hook that is not the down-call, both sides of
/// it added together, so `gap + layer + down` accounts for the wall clock
/// between one present and the next with nothing unattributed.
fn finish(
ds: &crate::state::DeviceState,
entered: std::time::Instant,
down: std::time::Duration,
) {
// Everything this hook cost, before any deliberate waiting.
let worked = std::time::Instant::now();
// Hold the game to the target rate. *After* the down-call, so the frame
// went out the moment it was ready and this only keeps the application
// from starting the next one — the cadence without the latency.
let held = match ds.frame_pacer.lock() {
Ok(mut pacer) => pacer.hold(worked),
// A poisoned pacer must not wedge the game.
Err(_) => std::time::Duration::ZERO,
};
if !held.is_zero() {
std::thread::sleep(held);
}
let now = std::time::Instant::now();
// What the sleep actually cost, not what it asked for. `thread::sleep`
// guarantees a floor and nothing else: on a box whose CPUs are all busy,
// waking is a scheduling decision and the overshoot can be many times the
// request. Recording the request would hide exactly the case worth seeing,
// and the overshoot lands nowhere else — the present-return is stamped
// after it, so `gap` cannot show it either.
let held = now.saturating_duration_since(worked);
if let Ok(enc) = ds.encoder.lock()
&& let Some(ref h) = *enc
{
// `layer` is this layer's cost, so the hold comes out of it: it is
// time spent on purpose, not overhead.
h.timing.layer.record(
worked
.saturating_duration_since(entered)
.saturating_sub(down),
);
h.timing.down.record(down);
h.timing.hold.record(held);
}
// Stamped after the hold, so the next frame's `gap` is the game's own
// work and not the waiting this layer asked it to do.
if let Ok(mut last) = ds.last_present_return.lock() {
*last = Some(now);
}
}
/// Everything the present hook needs to carry from the blit to the worker.
struct Submission {
slot: SlotGuard,
@@ -128,14 +185,14 @@ unsafe fn try_capture(
pi: &vk::PresentInfoKHR,
) -> Option<Submission> {
let image_index = unsafe { *pi.p_image_indices } as usize;
let (sc_image, sc_fmt, sc_ext) = {
let (sc_image, sc_fmt, sc_ext, image_count) = {
let images = ds.swapchain_images.lock().ok()?;
let fmt = *ds.swapchain_format.lock().ok()?;
let ext = *ds.swapchain_extent.lock().ok()?;
if image_index >= images.len() || ext.width == 0 || ext.height == 0 {
return None;
}
(images[image_index], fmt, ext)
(images[image_index], fmt, ext, images.len())
};
// Gate before any GPU work is queued. A game presenting faster than the
@@ -151,6 +208,13 @@ unsafe fn try_capture(
return None;
}
// Before the blit, not after it. A frame captured with nowhere to send it
// costs a slot, a copy and an export for nothing.
let ds_key = unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) };
if !encoder_ready(ds, ds_key, sc_ext.width, sc_ext.height) {
return None;
}
if let Ok(enc) = ds.encoder.lock() {
if let Some(ref h) = *enc {
h.capture_attempts.fetch_add(1, Ordering::Relaxed);
@@ -167,7 +231,16 @@ unsafe fn try_capture(
};
let submission = unsafe {
capture::capture_present_frame(ds, queue, sc_image, sc_fmt, sc_ext, image_index, app_waits)
capture::capture_present_frame(
ds,
queue,
sc_image,
sc_fmt,
sc_ext,
image_index,
image_count,
app_waits,
)
}?;
Some(Submission {
@@ -183,122 +256,121 @@ unsafe fn try_capture(
fn queue_for_encode(ds: &crate::state::DeviceState, submission: Submission) {
let ds_key = unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) };
{
let mut ctx = match ds.capture_tx.lock() {
Ok(c) => c,
Err(_) => return,
};
if ctx.is_none() {
let (tx, rx) = mpsc::channel();
start_capture_worker(ds_key, rx);
*ctx = Some(tx);
}
if let Some(tx) = ctx.as_ref() {
// Unbounded, but bounded in practice: the ring hands out a fixed
// number of slots and a job holds one for its whole life, so the
// queue can never exceed the slot count.
let _ = tx.send(CaptureJob {
ds_key,
slot: submission.slot,
width: submission.width,
height: submission.height,
sc_fmt: submission.sc_fmt,
present_time: submission.present_time,
});
let Ok(enc) = ds.encoder.lock() else {
return;
};
let Some(handle) = enc.as_ref() else {
return;
};
// `push_frame` is a `try_send`, so this never blocks the game's thread. A
// refused frame is dropped and its slot goes back when the `CapturedFrame`
// does, which is the backpressure the ring was always providing.
handle.push_frame(CapturedFrame {
ds_key,
width: submission.width,
height: submission.height,
vk_format: submission.sc_fmt.as_raw() as u32,
vk_colorspace: ds.swapchain_colorspace.load(Ordering::Relaxed),
present_time: submission.present_time,
slot: Some(submission.slot),
});
}
/// Make sure the encode pipeline exists, without building it here.
///
/// Returns false until it is ready, and the frame is simply not captured. The
/// build opens a pixelforge `VideoContext` and starts four threads; doing that
/// on the game's present thread would be a visible hitch on the first frame,
/// and doing it after a blit would waste one. It runs once, on a thread of its
/// own, and frames presented in the meantime are skipped before any GPU work is
/// queued for them.
fn encoder_ready(ds: &crate::state::DeviceState, ds_key: usize, width: u32, height: u32) -> bool {
if let Ok(enc) = ds.encoder.lock() {
if enc.is_some() {
return true;
}
}
}
pub fn start_capture_worker(ds_key: usize, capture_rx: mpsc::Receiver<CaptureJob>) {
if ds.encoder_starting.swap(true, Ordering::SeqCst) {
return false;
}
std::thread::Builder::new()
.name("nescapture-capture".into())
.name("nescapture-encoder-init".into())
.spawn(move || {
while let Ok(job) = capture_rx.recv() {
let ds = match DEVICE_STATE.get(&job.ds_key) {
Some(s) => s.clone(),
None => {
log::error!("capture worker: device state gone");
break;
}
};
// Copy the slot's handles out and release the ring lock before
// waiting: the present hook needs that lock every frame and
// must not queue behind a GPU wait.
let Some((fence, dmabuf_fd, stride, image, memory)) = ({
let ring = ds.capture_ring.lock().unwrap();
ring.as_ref()
.and_then(|r| r.slots.get(job.slot.index()))
.map(|s| (s.fence, s.dmabuf_fd, s.stride, s.image, s.memory))
}) else {
continue;
};
// The encoder reads this buffer from pixelforge's own VkDevice,
// which shares no timeline with ours, so the handover has to be
// on the CPU. Waiting here rather than in the present hook is
// the whole point of the worker thread.
let waited = unsafe {
(ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 1_000_000_000)
};
if waited != vk::Result::SUCCESS {
log::warn!("capture blit did not complete — frame dropped");
continue;
}
let source = if dmabuf_fd >= 0 {
let duped = unsafe { libc::dup(dmabuf_fd) };
if duped < 0 {
log::warn!("dup of capture DMA-BUF failed — frame dropped");
continue;
}
FrameSource::DmaBuf {
fd: duped,
stride,
modifier: 0,
}
} else {
match unsafe {
capture::read_frame_pixels(&ds, image, memory, job.width, job.height)
} {
Some(p) if !p.is_empty() => FrameSource::Pixels(p),
_ => continue,
}
};
// Lazy-init encoder
{
let mut enc = ds.encoder.lock().unwrap();
if enc.is_none() {
if let Some(cfg) = PipelineConfig::from_env(job.width, job.height) {
match PipelineHandle::new(cfg) {
Ok(h) => *enc = Some(h),
Err(e) => panic!("{e}"),
}
}
let Some(ds) = DEVICE_STATE.get(&ds_key).map(|s| s.clone()) else {
return;
};
let Some(cfg) = PipelineConfig::from_env(width, height) else {
log::error!("no encode pipeline configuration; capture disabled");
return;
};
match PipelineHandle::new(cfg) {
Ok(h) => {
if let Ok(mut enc) = ds.encoder.lock() {
*enc = Some(h);
}
}
let enc_guard = ds.encoder.lock().unwrap();
if let Some(ref encoder) = *enc_guard {
// Measured from the game's present, not from the start of
// this iteration: the wait above is part of what capture
// costs, and timing only the parts after it hid that.
let capture_elapsed = job.present_time.elapsed().as_secs_f32() * 1000.0;
encoder
.capture_ms
.store(capture_elapsed.to_bits(), Ordering::Relaxed);
encoder.push_frame(CapturedFrame {
source,
width: job.width,
height: job.height,
vk_format: job.sc_fmt.as_raw() as u32,
vk_colorspace: ds.swapchain_colorspace.load(Ordering::Relaxed),
present_time: job.present_time,
slot: Some(job.slot),
});
}
Err(e) => log::error!("encode pipeline: {e}"),
}
log::info!("capture worker exiting");
})
.ok();
false
}
/// Wait for a frame's blit and turn its slot into something the encoder reads.
///
/// This is the CPU handover the two devices need: pixelforge's `VkDevice`
/// shares no timeline with the game's, so no semaphore can bridge them and
/// somebody has to block. It used to be a thread of its own between the present
/// hook and the encoder; it is now the first thing the encoder thread does with
/// a frame, which costs that thread nothing — the blit it waits for was
/// submitted a whole frame earlier and has long since completed — and takes a
/// channel and a wakeup out of every frame's path.
pub fn resolve_source(
ds: &crate::state::DeviceState,
frame: &CapturedFrame,
) -> Option<FrameSource> {
let slot_index = frame.slot.as_ref()?.index();
// Copy the handles out and drop the ring lock before waiting: the present
// hook needs that lock every frame and must not queue behind a GPU wait.
let (fence, dmabuf_fd, stride, modifier, image, memory) = {
let ring = ds.capture_ring.lock().ok()?;
ring.as_ref()
.and_then(|r| r.slots.get(slot_index))
.map(|s| (s.fence, s.dmabuf_fd, s.stride, s.modifier, s.image, s.memory))?
};
let waited = unsafe { (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 1_000_000_000) };
if waited != vk::Result::SUCCESS {
log::warn!("capture blit did not complete — frame dropped");
return None;
}
// After the fence, so the queries have landed and `WAIT` returns at once.
if let Some(ns) = unsafe { capture::blit_gpu_time_ns(ds, slot_index) }
&& let Ok(enc) = ds.encoder.lock()
&& let Some(ref h) = *enc
{
h.timing.blit.record(std::time::Duration::from_nanos(ns));
}
if dmabuf_fd >= 0 {
let duped = unsafe { libc::dup(dmabuf_fd) };
if duped < 0 {
log::warn!("dup of capture DMA-BUF failed — frame dropped");
return None;
}
return Some(FrameSource::DmaBuf {
fd: duped,
stride,
// The slot's own modifier. This was hard-coded to zero, which was
// true only because the producer could not ask for anything else.
modifier,
});
}
match unsafe { capture::read_frame_pixels(ds, image, memory, frame.width, frame.height) } {
Some(p) if !p.is_empty() => Some(FrameSource::Pixels(p)),
_ => None,
}
}

18
apps/nescapture/src/slots.rs
View File

@@ -9,8 +9,18 @@
// happens to be — including on the error paths that abandon a frame.
// ─────────────────────────────────────────────────────────────────────────────
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
/// Presents that were admitted by the frame gate and then found no free slot.
///
/// Diagnostic only, and global rather than per-device because the stats thread
/// has no route to a `DeviceState`. It answers the one question the existing
/// counters cannot: a frame missing between `capture_attempts` and the encoder
/// was either never captured because the encoder still held every slot, or was
/// captured and lost further down. Those have opposite fixes.
pub static SLOT_STARVED: AtomicU32 = AtomicU32::new(0);
pub struct SlotPool {
free: Mutex<Vec<usize>>,
count: usize,
@@ -30,7 +40,13 @@ impl SlotPool {
/// thread, where waiting for the encoder to catch up would be a stutter the
/// player can feel. A frame with no slot is simply not captured.
pub fn try_acquire(self: &Arc<Self>) -> Option<SlotGuard> {
let index = self.free.lock().ok()?.pop()?;
let index = match self.free.lock().ok().and_then(|mut f| f.pop()) {
Some(i) => i,
None => {
SLOT_STARVED.fetch_add(1, Ordering::Relaxed);
return None;
}
};
Some(SlotGuard {
pool: Arc::clone(self),
index,

111
apps/nescapture/src/state.rs
View File

@@ -29,7 +29,13 @@ pub struct CaptureSlot {
/// failed, which sends that frame down the CPU readback path instead.
pub dmabuf_fd: std::os::raw::c_int,
pub stride: u32,
pub command_buffer: vk::CommandBuffer,
/// The DRM format modifier the driver gave this slot's image.
///
/// Carried per slot rather than assumed, and passed to the importer, which
/// creates its side with this exact value. It used to be hard-coded to
/// `DRM_FORMAT_MOD_LINEAR` on both sides — true at the time, because the
/// producer only ever asked for linear.
pub modifier: u64,
/// Signalled when this slot's blit has finished reading the swapchain and
/// writing the slot. The capture worker waits on it before handing the
/// DMA-BUF to the encoder, which reads it from a different VkDevice and so
@@ -40,6 +46,40 @@ pub struct CaptureSlot {
pub struct CaptureRing {
pub command_pool: vk::CommandPool,
pub slots: Vec<CaptureSlot>,
/// One command buffer per (swapchain image, slot) pair, recorded on first
/// use and re-submitted from then on.
///
/// The blit's contents depend on the source image, the destination image
/// and the extent, and on nothing else — so recording it again every frame
/// was work done on the game's own present thread for a result that never
/// changed. Indexed by [`blit_index`].
pub blits: Vec<vk::CommandBuffer>,
/// Which entries of `blits` hold a valid recording.
///
/// Cleared wholesale when the swapchain is recreated and when the extent
/// changes: a recorded buffer names specific `VkImage` handles and bakes in
/// the copy region, and a recreated swapchain's images are different
/// objects at a possibly different size. Submitting a stale one reads freed
/// memory.
pub blits_recorded: Vec<bool>,
/// How many swapchain images the ring allocated command buffers for.
pub image_count: usize,
/// Two timestamps per slot, bracketing that slot's blit, or null where the
/// presenting queue family cannot timestamp.
///
/// Per slot rather than per (image, slot) pair because only one blit per
/// slot is ever in flight — the `SlotGuard` guarantees it — and the query
/// index has to be baked into a command buffer recorded once.
pub timestamp_pool: vk::QueryPool,
/// Nanoseconds per device tick, for turning the pair into a duration.
pub timestamp_period: f32,
/// The extent `blits` were recorded for.
///
/// The ring is kept when the swapchain shrinks — `ensure_capture_ring`
/// accepts a ring at least as large as the request — so the extent can
/// change under a ring that is not rebuilt, and the recordings have to
/// follow it even though the images do not.
pub blit_extent: vk::Extent2D,
pub size: (u32, u32, vk::Format),
/// Queue family the command pool was created for. Command buffers may only
/// be submitted to a queue of the family their pool belongs to, so a
@@ -64,6 +104,58 @@ pub struct CaptureRing {
pub present_wait: Vec<vk::Semaphore>,
}
/// Index into [`CaptureRing::blits`] for one (swapchain image, slot) pair.
///
/// Flat rather than nested so the ring holds one `Vec` and takes one allocation
/// from the command pool. `None` when either index is out of range, which means
/// a swapchain that gained images under a ring built for fewer — a frame
/// skipped rather than a blit from an image the ring never saw.
pub fn blit_index(image_index: usize, slot: usize, image_count: usize) -> Option<usize> {
if image_index >= image_count || slot >= CAPTURE_SLOTS {
return None;
}
Some(image_index * CAPTURE_SLOTS + slot)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_image_and_slot_pair_has_its_own_index() {
let mut seen = std::collections::HashSet::new();
for image in 0..3 {
for slot in 0..CAPTURE_SLOTS {
let i = blit_index(image, slot, 3).expect("in range");
assert!(seen.insert(i), "image {image} slot {slot} collided at {i}");
}
}
assert_eq!(seen.len(), 3 * CAPTURE_SLOTS);
}
/// Every index must land inside a `Vec` of `image_count * CAPTURE_SLOTS`,
/// which is what the ring allocates.
#[test]
fn indices_stay_inside_the_allocation() {
let count = 4;
for image in 0..count {
for slot in 0..CAPTURE_SLOTS {
let i = blit_index(image, slot, count).expect("in range");
assert!(i < count * CAPTURE_SLOTS, "{i} is outside the allocation");
}
}
}
/// A swapchain recreated with more images than the ring was built for.
/// Blitting from an image the ring never allocated a buffer for would
/// submit whatever that slot happened to hold.
#[test]
fn an_out_of_range_image_or_slot_has_no_index() {
assert_eq!(blit_index(3, 0, 3), None);
assert_eq!(blit_index(0, CAPTURE_SLOTS, 3), None);
}
}
// ── Per-pipeline records ──────────────────────────────────────────────────────
#[derive(Clone, Debug, Default)]
@@ -143,8 +235,21 @@ pub struct DeviceState {
/// present hook, before any GPU work is queued, so a dropped frame costs
/// nothing beyond the comparison.
pub frame_gate: std::sync::Mutex<crate::pacing::FrameGate>,
/// Channel for threaded capture worker (present → worker).
pub capture_tx: std::sync::Mutex<Option<std::sync::mpsc::Sender<crate::present::CaptureJob>>>,
/// Holds the game to the target rate. The gate decides what is captured;
/// this decides how fast the game is allowed to produce frames, which no
/// compositor can do once a player turns V-Sync off.
pub frame_pacer: std::sync::Mutex<crate::pacing::FramePacer>,
/// When the previous `vkQueuePresentKHR` returned to the game.
///
/// The base for the `gap` span: time from here to the next present's
/// arrival is the game's own, with nothing this layer does inside it.
pub last_present_return: std::sync::Mutex<Option<std::time::Instant>>,
/// Whether a thread has already been started to build the encode pipeline.
///
/// The build is slow and happens once; without this latch every present
/// arriving before it finishes would start another one.
pub encoder_starting: std::sync::atomic::AtomicBool,
}
// ── Per-command-buffer state ──────────────────────────────────────────────────

101
apps/nescapture/src/swapchain.rs
View File

@@ -61,6 +61,14 @@ pub unsafe extern "system" fn vkCreateSwapchainKHR(
// outgoing swapchain, so they are set aside rather than reused.
crate::capture::retire_all_present_semaphores(&ds);
// Those fresh images also make every recorded blit invalid: a recording
// names its source image by handle, and the handles behind these indices
// now belong to a destroyed swapchain. The extent check in
// `capture_present_frame` catches a resize on its own, but a swapchain
// recreated at the same size — which is the common case, on a format or
// present-mode change — looks identical to it.
crate::capture::invalidate_recorded_blits(&ds);
*ds.swapchain.lock().unwrap() = Some(unsafe { *p_swapchain });
*ds.swapchain_format.lock().unwrap() = ci.image_format;
*ds.swapchain_extent.lock().unwrap() = ci.image_extent;
@@ -92,12 +100,26 @@ pub unsafe extern "system" fn vkCreateSwapchainKHR(
ds.swapchain_colorspace
.store(ci.image_color_space.as_raw() as u32, Ordering::Relaxed);
log::debug!(
"swapchain created — format={:?} colorspace={:?} extent={}x{}",
// `info`, not `debug`, and the present mode is why. It decides whether the
// compositor paces this game at all: a FIFO swapchain waits on
// `wl_surface.frame`, so the game's rate is the compositor's callback
// cadence no matter what the gate here is set to, while MAILBOX and
// IMMEDIATE ignore those callbacks entirely and leave the pacing to this
// layer. The two cases need opposite fixes and nothing else in a log
// distinguishes them — "the game runs at 60" reads identically either way.
//
// An application's in-game V-Sync setting is not the answer either: DXVK
// and VKD3D choose the Vulkan present mode themselves, and what they pick
// from a given setting is theirs to decide.
log::info!(
"swapchain created — format={:?} colorspace={:?} extent={}x{} present_mode={:?} \
min_images={}",
ci.image_format,
ci.image_color_space,
ci.image_extent.width,
ci.image_extent.height,
ci.present_mode,
ci.min_image_count,
);
vk::Result::SUCCESS
@@ -158,3 +180,78 @@ pub unsafe extern "system" fn vkGetSwapchainImagesKHR(
vk::Result::SUCCESS
}
/// Time the game's wait for a swapchain image.
///
/// Hooked for the measurement alone — the image index, the semaphore and the
/// fence are the application's business and nothing here touches them.
///
/// It is the one part of a frame the present hook cannot see. `gap` runs from
/// one present returning to the next arriving and the acquire sits inside it,
/// so a game blocked waiting for the compositor to release a buffer and a game
/// busy rendering are the same number. Under a FIFO swapchain that wait is the
/// compositor's frame pacing, which is a different problem in a different
/// process from anything this layer can fix.
/// The device state for an acquire, with the present hook's fallback.
///
/// These hooks exist only to time the call, but hooking replaces the
/// application's function pointer — so there is no passing through if the
/// lookup misses, and returning an error would break a game for the sake of a
/// measurement. The last resort is the same one `vkQueuePresentKHR` uses: with
/// one device in the process, the only entry is the right one.
fn device_for(device: vk::Device) -> Option<std::sync::Arc<crate::state::DeviceState>> {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
DEVICE_STATE
.get(&key)
.map(|s| s.clone())
.or_else(|| DEVICE_STATE.iter().next().map(|e| e.value().clone()))
}
fn record_acquire(ds: &crate::state::DeviceState, waited: std::time::Duration) {
if let Ok(enc) = ds.encoder.lock()
&& let Some(ref h) = *enc
{
h.timing.acquire.record(waited);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkAcquireNextImageKHR(
device: vk::Device,
swapchain: vk::SwapchainKHR,
timeout: u64,
semaphore: vk::Semaphore,
fence: vk::Fence,
p_image_index: *mut u32,
) -> vk::Result {
let Some(ds) = device_for(device) else {
return vk::Result::ERROR_DEVICE_LOST;
};
let Some(acquire) = ds.fp.acquire_next_image_khr else {
return vk::Result::ERROR_EXTENSION_NOT_PRESENT;
};
let started = std::time::Instant::now();
let result = unsafe { acquire(device, swapchain, timeout, semaphore, fence, p_image_index) };
record_acquire(&ds, started.elapsed());
result
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkAcquireNextImage2KHR(
device: vk::Device,
p_acquire_info: *const vk::AcquireNextImageInfoKHR,
p_image_index: *mut u32,
) -> vk::Result {
let Some(ds) = device_for(device) else {
return vk::Result::ERROR_DEVICE_LOST;
};
let Some(acquire) = ds.fp.acquire_next_image2_khr else {
return vk::Result::ERROR_EXTENSION_NOT_PRESENT;
};
let started = std::time::Instant::now();
let result = unsafe { acquire(device, p_acquire_info, p_image_index) };
record_acquire(&ds, started.elapsed());
result
}

142
apps/nescapture/src/timing.rs Normal file
View File

@@ -0,0 +1,142 @@
// ─────────────────────────────────────────────────────────────────────────────
// timing.rs — where a present's time actually goes
//
// The rate line says how many frames survive each stage. It cannot say why a
// game that offers 68 presents a second yields 56 captures against a gate set
// to 60, because that shortfall is made of gaps: stretches where the game
// presented nothing and the gate had no frames to admit. Whether those gaps
// belong to the game, to the driver, or to this layer is the whole question,
// and nothing measured so far distinguishes them.
//
// Three spans per present, all on the game's own thread, which together
// partition the wall clock between one present and the next:
//
// gap — previous present returning to this one arriving. The game's own
// frame time, everything this layer does excluded.
// layer — this layer's code, both sides of the down-call added together.
// down — the down-call: driver, WSI, compositor.
//
// A hitch shows up in exactly one of them and that names the culprit.
//
// A fourth, blit, is not part of that partition. It is GPU execution time for
// the capture copy, which runs alongside the game rather than in front of it,
// and answers a different question: what the capture takes from the device the
// game is rendering on. CPU time in the hook says nothing about it — the hook
// submits the blit and returns without waiting.
// ─────────────────────────────────────────────────────────────────────────────
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::time::Duration;
/// One span's running total for the current reporting second.
#[derive(Default)]
pub struct Span {
total_us: AtomicU64,
max_us: AtomicU32,
count: AtomicU32,
}
impl Span {
pub fn record(&self, d: Duration) {
let us = d.as_micros().min(u32::MAX as u128) as u32;
self.total_us.fetch_add(u64::from(us), Ordering::Relaxed);
self.max_us.fetch_max(us, Ordering::Relaxed);
self.count.fetch_add(1, Ordering::Relaxed);
}
/// Mean and maximum in milliseconds, resetting for the next second.
pub fn take(&self) -> (f32, f32) {
let total = self.total_us.swap(0, Ordering::Relaxed);
let max = self.max_us.swap(0, Ordering::Relaxed);
let count = self.count.swap(0, Ordering::Relaxed);
let mean = if count == 0 {
0.0
} else {
total as f32 / count as f32 / 1000.0
};
(mean, max as f32 / 1000.0)
}
}
/// Every span of the present path, plus a count of the gaps big enough to be
/// the hitch being hunted.
#[derive(Default)]
pub struct PresentTiming {
pub gap: Span,
pub layer: Span,
pub down: Span,
/// GPU execution time of the capture blit, from timestamps in its own
/// command buffer.
///
/// Not part of the `gap`/`layer`/`down` partition — those three account for
/// the game's thread, and this is the GPU, which runs alongside it. It is
/// the share of the device the capture takes from whatever is rendering.
pub blit: Span,
/// Time the game spent blocked inside `vkAcquireNextImageKHR`.
///
/// A subdivision of `gap`, not a fourth term beside it: the acquire happens
/// while the game is between presents, so this is the part of its frame
/// time spent waiting for the compositor to hand back an image rather than
/// doing work of its own. Under a FIFO swapchain that wait *is* the
/// compositor's pacing, and it is invisible in `gap` alone — a game waiting
/// on a buffer and a game busy rendering produce the same number.
pub acquire: Span,
/// Time this layer deliberately held the game back, to keep it at the
/// target rate.
///
/// Reported so it is never mistaken for cost. It is excluded from `layer`,
/// and the present-return is stamped after it, so `gap` stays the game's
/// own frame time — otherwise every number here would shift the moment
/// limiting was switched on and none of them would say why.
pub hold: Span,
/// Gaps longer than [`LONG_GAP`]. A steady handful per second is a
/// periodic stall; zero means the frame time is merely uneven.
long_gaps: AtomicU32,
}
/// What counts as a hitch rather than jitter: two frames' worth at 60.
pub const LONG_GAP: Duration = Duration::from_millis(33);
impl PresentTiming {
pub fn record_gap(&self, d: Duration) {
self.gap.record(d);
if d >= LONG_GAP {
self.long_gaps.fetch_add(1, Ordering::Relaxed);
}
}
pub fn take_long_gaps(&self) -> u32 {
self.long_gaps.swap(0, Ordering::Relaxed)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_span_reports_mean_and_max_then_resets() {
let s = Span::default();
s.record(Duration::from_micros(1000));
s.record(Duration::from_micros(3000));
let (mean, max) = s.take();
assert!((mean - 2.0).abs() < 0.001, "mean was {mean}");
assert!((max - 3.0).abs() < 0.001, "max was {max}");
// Taking resets, so a quiet second reads zero rather than the last
// second's numbers over again.
let (mean, max) = s.take();
assert_eq!((mean, max), (0.0, 0.0));
}
#[test]
fn only_gaps_past_the_threshold_are_counted_as_hitches() {
let t = PresentTiming::default();
t.record_gap(Duration::from_millis(16));
t.record_gap(Duration::from_millis(32));
t.record_gap(Duration::from_millis(33));
t.record_gap(Duration::from_millis(120));
assert_eq!(t.take_long_gaps(), 2);
assert_eq!(t.take_long_gaps(), 0, "the count did not reset");
}
}