mirror of
https://github.com/nestriness/nestri.git
synced 2026-09-27 04:52:25 +03:00
feat: media bitrate control, HDR (#346)
Fixes: #335 Still a work-in-progress. --------- Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Wanjohi <elviswanjohi47@gmail.com>
This commit is contained in:
Kristian Ollikainen
co-authored by
DatCaptainHorse
Claude Opus 5
Wanjohi
DatCaptainHorse
Claude Opus 5
Wanjohi
parent
1c721962f4
commit
0811f57f1a
@@ -32,9 +32,9 @@ toml = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
# Vulkan Video hardware encoding.
|
||||
pixelforge = { git = "https://github.com/DatCaptainHorse/pixelforge.git", rev = "681fa4dd8bce5dabf008d00983e991b0eb8b3696", features = ["dmabuf"] }
|
||||
pixelforge = { git = "https://github.com/DatCaptainHorse/pixelforge.git", rev = "b4b7b36af6f9cbd0fedae220915edd75de933743" }
|
||||
|
||||
# libc for DMA-BUF OS primitives
|
||||
# libc for socket options
|
||||
libc = "0.2"
|
||||
|
||||
# Shared IPC protocol
|
||||
|
||||
+84
-13
@@ -24,15 +24,12 @@ Game process
|
||||
│ vkCmdBindPipeline → detect HUD │
|
||||
│ vkQueuePresentKHR → capture+encode │
|
||||
└──────────────────────────────────────────────┘
|
||||
│ GPU blit, same device
|
||||
│ GPU blit on the game's queue, signals a timeline semaphore
|
||||
▼
|
||||
final_image (DMA-BUF exportable)
|
||||
│ get_dmabuf_fd(final_memory)
|
||||
ring slot (an image on the game's own device)
|
||||
│ read in place, once the blit's point is reached
|
||||
▼
|
||||
DmaBufImporter (pixelforge VkDevice)
|
||||
│ import_or_reuse() → vk::Image
|
||||
▼
|
||||
ColorConverter (GPU compute shader)
|
||||
ColorConverter (GPU compute shader, the encoder's own queue)
|
||||
│ BGRA/RGB10/FP16 → NV12/P010/YUV444
|
||||
▼
|
||||
Encoder (Vulkan Video: H.264 / H.265 / AV1)
|
||||
@@ -41,8 +38,24 @@ Encoder (Vulkan Video: H.264 / H.265 / AV1)
|
||||
Unix datagram → neshub → the client
|
||||
```
|
||||
|
||||
CPU fallback exists only for driver configurations without DMA-BUF external
|
||||
memory export.
|
||||
The encoder runs on the game's own `VkDevice`. The layer creates that device
|
||||
with what the encoder needs: the extensions and feature bits it asks for, and
|
||||
queues of its own wherever a queue family has one to spare, since a `VkQueue`
|
||||
may not be submitted to from two threads at once. Where no family has room,
|
||||
the game's queue is created internally synchronized and shared. Every step of
|
||||
a frame is ordered on the GPU; nothing waits on the CPU.
|
||||
|
||||
Where the conversion is only the YUV matrix and the device has
|
||||
`VK_VALVE_video_encode_rgb_conversion`, the encoder takes the RGB frame and
|
||||
converts it itself, and the converter is not built. That path runs at limited
|
||||
range, since the one driver offering it writes limited range whatever it is
|
||||
asked; everything else is full range.
|
||||
|
||||
When the game's device cannot host the encoder -- an instance the loader
|
||||
cannot raise to Vulkan 1.1, no queue the encoder could safely use, a driver
|
||||
refusing the additions, or `NESCAPTURE_SHARED_DEVICE=0` -- the encoder gets a
|
||||
device of its own, and each frame is read back on the CPU and uploaded there.
|
||||
That works everywhere and costs a copy each way.
|
||||
|
||||
---
|
||||
|
||||
@@ -93,11 +106,18 @@ implicit layer is loaded into *every* Vulkan process on the system.
|
||||
| `NESCAPTURE_CODEC` | best available | `h264`, `h265` or `av1`; probes if unset |
|
||||
| `NESCAPTURE_FORMAT` | `yuv420` | `yuv420` or `yuv444` |
|
||||
| `NESCAPTURE_DEPTH` | auto | `8` or `10`; inferred from the swapchain `VkFormat` if unset |
|
||||
| `NESCAPTURE_BITRATE` | `10000` | CBR target in kbps. Ignored when `NESCAPTURE_QP` is set |
|
||||
| `NESCAPTURE_QP` | _(unset)_ | Constant QP instead of CBR |
|
||||
| `NESCAPTURE_RC` | _(inferred)_ | `cqp`, `cbr` or `vbr`. Unset infers `cqp` when `NESCAPTURE_QP` is set, `cbr` otherwise |
|
||||
| `NESCAPTURE_BITRATE` | `10000` | Target bitrate in kbps, under `cbr` and `vbr` |
|
||||
| `NESCAPTURE_BITRATE_MAX` | 1.5x the target | VBR ceiling in kbps. Ignored outside `vbr` |
|
||||
| `NESCAPTURE_QP` | _(unset)_ | Constant QP, under `cqp` |
|
||||
| `NESCAPTURE_FPS` | `60` | Target frame rate |
|
||||
| `NESCAPTURE_IDR_INTERVAL` | `4` | Force an IDR every N **seconds** |
|
||||
| `NESCAPTURE_INTRA_REFRESH` | _(off)_ | Set to `1` to replace periodic key frames with an intra refresh cycle |
|
||||
| `NESCAPTURE_INTRA_REFRESH_QP_DELTA` | `-4` | QP shift inside the refresh band; negative spends bits on it |
|
||||
| `NESCAPTURE_INTRA_REFRESH_SHAPE` | auto | `rows`, `columns` or `partitions`; the driver chooses if unset |
|
||||
| `NESCAPTURE_TUNE` | _(unset)_ | `highquality`, `lowlatency`, `ultralowlatency`, `lossless` |
|
||||
| `NESCAPTURE_SHARED_DEVICE` | _(on)_ | Set to `0` to leave the game's device as the game asked for it, and encode on a device of the encoder's own with CPU readback |
|
||||
| `NESCAPTURE_RGB_ENCODE` | _(on)_ | Set to `0` to always convert with the shader, even where the encoder could convert RGB itself |
|
||||
| `NESCAPTURE_CONFIG` | _(unset)_ | Path to the per-app shader-hash TOML |
|
||||
| `NESCAPTURE_GAME_NAME` | exe basename | Override app identification for that config |
|
||||
| `NESCAPTURE_DISCOVER` | _(unset)_ | Set to `1` to log every draw, for finding HUD shaders |
|
||||
@@ -107,6 +127,57 @@ Everything else is decided at runtime: the client asks `neshub` for a codec or
|
||||
bitrate change and it arrives on the command socket, so the encoder is
|
||||
reconfigured without a restart.
|
||||
|
||||
### Intra refresh
|
||||
|
||||
Instead of a key frame every few seconds, each picture codes one slice of the
|
||||
image as intra, so after a full cycle every part has been refreshed. The same
|
||||
cost, paid evenly, with no picture much larger than any other — which is what
|
||||
a link with a latency budget wants, since a key frame is the largest frame
|
||||
there is.
|
||||
|
||||
**The cycle length is not configurable, deliberately.** It is bounded by how
|
||||
many refresh regions the picture actually has, and that depends on the codec's
|
||||
block size: at 1080p an H.265 picture is 17 CTB rows tall where an H.264 one is
|
||||
68 macroblock rows, so the same duration is comfortable for one codec and
|
||||
impossible for the other. The encoder knows the codec, the resolution and what
|
||||
the device allows, and derives it from the key frame interval it replaces.
|
||||
|
||||
The refresh here spreads cost only; it does not make the cycle a recovery
|
||||
point. Doing that would restrict prediction on every picture — expensive, and
|
||||
what turns the refreshed band into a visible discontinuity — to buy a
|
||||
guarantee this stream gets more cheaply from the client asking for an IDR.
|
||||
|
||||
The band is coded intra every cycle, so it carries none of the refinement its
|
||||
neighbours have accumulated and reads as a strip of lower quality sweeping
|
||||
across the picture. `NESCAPTURE_INTRA_REFRESH_QP_DELTA` spends bits back into
|
||||
it, out of the rest of the frame. Measured at 1080p with ColorVideoVDP, `-4`
|
||||
recovers a fifth of what intra refresh costs and the encoded size does not
|
||||
grow — but the best value depends on the content, and too large a shift
|
||||
starves the rest of the frame faster than too small a one helps. Devices that
|
||||
cannot express a negative delta, or whose refresh regions follow the slice
|
||||
layout rather than a block sweep, decline it and say so.
|
||||
|
||||
`NESCAPTURE_INTRA_REFRESH_SHAPE` stays configurable because the device cannot
|
||||
answer it: whether a horizontal or vertical sweep looks better depends on how
|
||||
the content moves. It also changes how many regions there are — a 1080p
|
||||
picture in 64×64 blocks is 17 rows but 30 columns, so `columns` allows a
|
||||
longer cycle and thus less intra per picture.
|
||||
|
||||
### Rate control
|
||||
|
||||
`cbr` holds every frame to the same size, which is what a link with a fixed
|
||||
budget wants. `vbr` holds the same *average* while letting a frame that needs
|
||||
it spend up to the ceiling — a scene change is coded rather than smeared, at
|
||||
the cost of a burst the path has to absorb. `cqp` holds quality constant and
|
||||
lets the bitrate go wherever the content takes it, which is a recording
|
||||
setting rather than a streaming one.
|
||||
|
||||
The command socket carries a target bitrate but has no way to name a mode
|
||||
beyond CBR and constant QP. A target arriving while the encode is `vbr` is
|
||||
therefore applied as a target, leaving the mode and the ceiling alone — so the
|
||||
ceiling asked for at launch survives a session, and a congestion controller
|
||||
adjusts underneath it. Retargeting costs no rebuild and no key frame.
|
||||
|
||||
---
|
||||
|
||||
## Per-app shader-hash config
|
||||
@@ -145,10 +216,10 @@ src/
|
||||
├── framebuffer.rs image view and framebuffer tracking
|
||||
├── commands.rs vkCmdBind*, vkCmdDraw*, vkCmdBeginRenderPass
|
||||
├── swapchain.rs vkCreateSwapchainKHR, image enumeration
|
||||
├── capture.rs GPU blit to the capture image, DMA-BUF export
|
||||
├── capture.rs GPU blit into the capture ring, CPU readback fallback
|
||||
├── present.rs vkQueuePresentKHR, encode dispatch
|
||||
├── encode.rs pixelforge pipeline, codec probing, IPC send
|
||||
├── dmabuf_import.rs cross-device zero-copy import
|
||||
├── shared.rs creating the game's device for the encoder to share
|
||||
├── config.rs per-app TOML shader-hash config
|
||||
└── discovery.rs draw-call logging for shader discovery
|
||||
```
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Runs a Vulkan workload under the compositor with the layer active, then checks
|
||||
# the encoded result against the compositor's own readback of the same frames.
|
||||
# Two independent paths see the same content: the compositor reads the surface
|
||||
# back to the CPU, the layer exports it as a DMA-BUF and encodes it on the GPU.
|
||||
# back to the CPU, the layer encodes it on the GPU without it leaving the device.
|
||||
# Agreement between them is the evidence; a single path cannot tell a correct
|
||||
# frame from a plausible-looking wrong one.
|
||||
#
|
||||
@@ -96,7 +96,8 @@ wait $RECV || true
|
||||
FRAMES="$(cat "$WORK/frames.txt")"
|
||||
echo
|
||||
echo "frames encoded: $FRAMES"
|
||||
grep -m1 "First import" "$WORK/run.log" || echo " (no DMA-BUF import logged)"
|
||||
grep -m1 "encoding on the game's own device" "$WORK/run.log" \
|
||||
|| echo " (encoding on a device of its own, with CPU readback)"
|
||||
|
||||
python3 - "$WORK" "$STREAM" "$FRAMES" <<'PY'
|
||||
import glob, subprocess, sys
|
||||
|
||||
+818
-287
File diff suppressed because it is too large
Load Diff
+163
-86
@@ -14,6 +14,7 @@ use std::os::raw::c_void;
|
||||
use std::sync::Arc;
|
||||
|
||||
const VK_LAYER_LINK_INFO: u32 = 0;
|
||||
const VK_LOADER_DATA_CALLBACK: u32 = 1;
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "system" fn vkCreateDevice(
|
||||
@@ -43,6 +44,14 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
};
|
||||
unsafe { (*layer_info).u.pDeviceLayerInfo = (*dev_link).pNext };
|
||||
|
||||
let set_loader_data = unsafe {
|
||||
find_layer_link::<VkLayerDeviceCreateInfo>(
|
||||
(*p_create_info).p_next as *const c_void,
|
||||
VK_LOADER_DATA_CALLBACK,
|
||||
)
|
||||
}
|
||||
.and_then(|info| unsafe { (*info).u.pfnSetDeviceLoaderData });
|
||||
|
||||
let inst_key = unsafe { dispatch_key(physical_device.as_raw() as *const c_void) };
|
||||
let istate = match INSTANCE_STATE.get(&inst_key) {
|
||||
Some(s) => s.clone(),
|
||||
@@ -54,11 +63,8 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
}
|
||||
};
|
||||
|
||||
// ── Inject DMA-BUF extensions for zero-copy capture ──────────────────
|
||||
let ci = unsafe { &*p_create_info };
|
||||
|
||||
// Collect the game's original extensions.
|
||||
let original_extensions: Vec<*const libc::c_char> =
|
||||
let game_extensions: Vec<*const libc::c_char> =
|
||||
if ci.enabled_extension_count > 0 && !ci.pp_enabled_extension_names.is_null() {
|
||||
unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
@@ -71,69 +77,34 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Extensions we need — static byte strings so pointers stay valid.
|
||||
const EXT_EXTERNAL_MEMORY: &[u8] = b"VK_KHR_external_memory\0";
|
||||
const EXT_EXTERNAL_MEMORY_FD: &[u8] = b"VK_KHR_external_memory_fd\0";
|
||||
const EXT_EXTERNAL_MEMORY_DMABUF: &[u8] = b"VK_EXT_external_memory_dma_buf\0";
|
||||
// Lets the capture ring be allocated tiled. The importer has always created
|
||||
// its side with DRM_FORMAT_MODIFIER_EXT tiling; without this the producer
|
||||
// can only offer it a linear buffer.
|
||||
const EXT_IMAGE_DRM_FORMAT_MODIFIER: &[u8] = b"VK_EXT_image_drm_format_modifier\0";
|
||||
|
||||
let needed: &[&[u8]] = &[
|
||||
EXT_EXTERNAL_MEMORY,
|
||||
EXT_EXTERNAL_MEMORY_FD,
|
||||
EXT_EXTERNAL_MEMORY_DMABUF,
|
||||
EXT_IMAGE_DRM_FORMAT_MODIFIER,
|
||||
];
|
||||
|
||||
// Build extended list: original + any of ours not already present.
|
||||
let mut extended = original_extensions.clone();
|
||||
for &ext in needed {
|
||||
let name_cstr = unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(ext) };
|
||||
let already = extended
|
||||
.iter()
|
||||
.any(|&ptr| unsafe { std::ffi::CStr::from_ptr(ptr) == name_cstr });
|
||||
if !already {
|
||||
extended.push(ext.as_ptr() as *const libc::c_char);
|
||||
// Where it can, the encoder runs on this very device, so the device is
|
||||
// first created with what that needs. Anything refused falls back to the
|
||||
// device exactly as the game asked for it, and the encoder to a device of
|
||||
// its own.
|
||||
let mut shared = None;
|
||||
if let Some(prepared) =
|
||||
unsafe { crate::shared::prepare(&istate, next_gdpa, physical_device, ci, &game_extensions) }
|
||||
{
|
||||
let shared_ci = prepared.create_info(ci);
|
||||
let result =
|
||||
unsafe { (istate.create_device)(physical_device, &shared_ci, p_allocator, p_device) };
|
||||
let (additions, entry, instance) = unsafe { prepared.finish() };
|
||||
if result == vk::Result::SUCCESS {
|
||||
shared = Some((additions, entry, instance));
|
||||
} else {
|
||||
log::warn!(
|
||||
"vkCreateDevice with the encoder's additions failed ({result:?}), \
|
||||
retrying as the game asked"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Try with injected extensions first.
|
||||
//
|
||||
// The device's queue create info is passed through unchanged. An earlier
|
||||
// version bumped the first family's queue count by one to get a dedicated
|
||||
// capture queue, which was then never used — and could not be: the capture
|
||||
// blit has to be submitted to the queue the game presents on, or it gains
|
||||
// no ordering against the present. All the bump did was risk exceeding the
|
||||
// family's available queue count on the way in.
|
||||
let mut modified_ci = *ci;
|
||||
modified_ci.enabled_extension_count = extended.len() as u32;
|
||||
modified_ci.pp_enabled_extension_names = extended.as_ptr();
|
||||
|
||||
let mut dmabuf_available = true;
|
||||
let result =
|
||||
unsafe { (istate.create_device)(physical_device, &modified_ci, p_allocator, p_device) };
|
||||
|
||||
let result = if result != vk::Result::SUCCESS {
|
||||
// Driver rejected our extensions — retry with original create info.
|
||||
log::warn!(
|
||||
"vkCreateDevice with DMA-BUF extensions failed ({:?}), \
|
||||
retrying without — CPU readback fallback will be used",
|
||||
result
|
||||
);
|
||||
dmabuf_available = false;
|
||||
unsafe { (istate.create_device)(physical_device, p_create_info, p_allocator, p_device) }
|
||||
} else {
|
||||
log::info!("DMA-BUF extensions injected successfully");
|
||||
result
|
||||
};
|
||||
if result != vk::Result::SUCCESS {
|
||||
return result;
|
||||
}
|
||||
|
||||
if !dmabuf_available {
|
||||
log::warn!("DMA-BUF extensions missing — will use CPU readback fallback (expensive!)");
|
||||
if shared.is_none() {
|
||||
let result = unsafe {
|
||||
(istate.create_device)(physical_device, p_create_info, p_allocator, p_device)
|
||||
};
|
||||
if result != vk::Result::SUCCESS {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let device = unsafe { *p_device };
|
||||
@@ -156,6 +127,7 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
get_device_proc_addr: next_gdpa,
|
||||
destroy_device: load!(b"vkDestroyDevice\0"),
|
||||
get_device_queue: load!(b"vkGetDeviceQueue\0"),
|
||||
get_device_queue2: try_load!(b"vkGetDeviceQueue2\0"),
|
||||
queue_present_khr: try_load!(b"vkQueuePresentKHR\0"),
|
||||
|
||||
// Phase 1
|
||||
@@ -189,10 +161,6 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
cmd_pipeline_barrier: load!(b"vkCmdPipelineBarrier\0"),
|
||||
cmd_copy_image: load!(b"vkCmdCopyImage\0"),
|
||||
get_image_subresource_layout: load!(b"vkGetImageSubresourceLayout\0"),
|
||||
get_memory_fd_khr: try_load!(b"vkGetMemoryFdKHR\0"),
|
||||
get_image_drm_format_modifier_properties_ext: try_load!(
|
||||
b"vkGetImageDrmFormatModifierPropertiesEXT\0"
|
||||
),
|
||||
create_query_pool: try_load!(b"vkCreateQueryPool\0"),
|
||||
destroy_query_pool: try_load!(b"vkDestroyQueryPool\0"),
|
||||
cmd_reset_query_pool: try_load!(b"vkCmdResetQueryPool\0"),
|
||||
@@ -232,6 +200,17 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
|
||||
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
|
||||
|
||||
let shared = shared.map(|(additions, entry, instance)| unsafe {
|
||||
crate::shared::SharedDevice::adopt(
|
||||
additions,
|
||||
entry,
|
||||
instance,
|
||||
physical_device,
|
||||
device,
|
||||
next_gdpa,
|
||||
)
|
||||
});
|
||||
|
||||
// Phase 3: load shader hash config
|
||||
let shader_hashes = config::resolve_config_path()
|
||||
.as_ref()
|
||||
@@ -252,6 +231,9 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
raw: device,
|
||||
physical_device,
|
||||
fp,
|
||||
shared,
|
||||
shared_active: std::sync::atomic::AtomicBool::new(false),
|
||||
set_loader_data,
|
||||
|
||||
shader_registry: DashMap::new(),
|
||||
pipeline_registry: DashMap::new(),
|
||||
@@ -279,6 +261,7 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
}),
|
||||
swapchain_colorspace: std::sync::atomic::AtomicU32::new(0),
|
||||
frame_counter: std::sync::atomic::AtomicU64::new(0),
|
||||
ring_generation: std::sync::atomic::AtomicU64::new(0),
|
||||
|
||||
hud_detected_frame: std::sync::atomic::AtomicBool::new(false),
|
||||
pending_capture_frame: std::sync::atomic::AtomicBool::new(false),
|
||||
@@ -287,7 +270,6 @@ pub unsafe extern "system" fn vkCreateDevice(
|
||||
|
||||
encoder: std::sync::Mutex::new(None),
|
||||
|
||||
|
||||
frame_gate: std::sync::Mutex::new(crate::pacing::FrameGate::from_env()),
|
||||
frame_pacer: std::sync::Mutex::new(crate::pacing::FramePacer::from_env()),
|
||||
last_present_return: std::sync::Mutex::new(None),
|
||||
@@ -317,23 +299,20 @@ pub unsafe extern "system" fn vkDestroyDevice(
|
||||
ds.pipeline_registry.len(),
|
||||
);
|
||||
|
||||
// ── 1. Shut down encoder pipeline (unblocks encoder + RTP threads) ───
|
||||
{
|
||||
let mut enc_guard = ds.encoder.lock().unwrap();
|
||||
if let Some(handle) = enc_guard.take() {
|
||||
handle.shutdown();
|
||||
// `handle` is dropped here → drops `frame_tx` → encoder thread's
|
||||
// recv_timeout returns Disconnected → encoder thread drops
|
||||
// `encoded_tx` → RTP thread exits too.
|
||||
//
|
||||
// Give threads a moment to drain. In production you'd join the
|
||||
// JoinHandles, but since we don't store them, a short sleep +
|
||||
// the AtomicBool shutdown flag is sufficient.
|
||||
log::info!("encoder pipeline shutdown signaled");
|
||||
// ── 1. Shut down the encode pipeline ──────────────────────────────────
|
||||
//
|
||||
// Waited for, not just signalled. On a shared device the encoder, its
|
||||
// converter and their images are objects on this very device, and
|
||||
// destroying the device under them is a use-after-free. The receive
|
||||
// timeout bounds how long the thread takes to notice.
|
||||
let handle = ds.encoder.lock().ok().and_then(|mut g| g.take());
|
||||
if let Some(handle) = handle {
|
||||
if handle.finish(std::time::Duration::from_secs(2)) {
|
||||
log::info!("encoder pipeline stopped");
|
||||
} else {
|
||||
log::error!("encoder thread did not stop in time; destroying the device anyway");
|
||||
}
|
||||
}
|
||||
// Brief yield to let threads notice the disconnect.
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
// ── 2. Tear down the capture ring ─────────────────────────────────────
|
||||
//
|
||||
@@ -392,13 +371,111 @@ pub unsafe extern "system" fn vkGetDeviceQueue(
|
||||
) {
|
||||
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
|
||||
if let Some(ds) = DEVICE_STATE.get(&key) {
|
||||
unsafe { (ds.fp.get_device_queue)(device, queue_family_index, queue_index, p_queue) };
|
||||
// A queue created internally synchronized, so the encoder can share
|
||||
// it, is invisible to vkGetDeviceQueue: only vkGetDeviceQueue2 with the
|
||||
// matching flags returns it. The game asked for a plain queue and gets
|
||||
// this one.
|
||||
match (
|
||||
shares_family(&ds, queue_family_index),
|
||||
ds.fp.get_device_queue2,
|
||||
) {
|
||||
(true, Some(get2)) => {
|
||||
let info = vk::DeviceQueueInfo2::default()
|
||||
.flags(vk::DeviceQueueCreateFlags::INTERNALLY_SYNCHRONIZED_KHR)
|
||||
.queue_family_index(queue_family_index)
|
||||
.queue_index(queue_index);
|
||||
unsafe { get2(device, &info, p_queue) };
|
||||
}
|
||||
_ => unsafe {
|
||||
(ds.fp.get_device_queue)(device, queue_family_index, queue_index, p_queue)
|
||||
},
|
||||
}
|
||||
let queue = unsafe { *p_queue };
|
||||
// The encoder's queues reach this hook without the loader in between.
|
||||
// For the game's, the loader stamps them again on the way out.
|
||||
unsafe { stamp(&ds, queue.as_raw() as *mut c_void) };
|
||||
QUEUE_TO_DEVICE_KEY.insert(queue.as_raw(), key);
|
||||
crate::state::QUEUE_TO_FAMILY.insert(queue.as_raw(), queue_family_index);
|
||||
}
|
||||
}
|
||||
|
||||
/// Give a dispatchable object this layer obtained itself the loader's
|
||||
/// dispatch data.
|
||||
///
|
||||
/// The loader writes its dispatch pointer into every queue and command buffer
|
||||
/// that passes through its own entry points. One a layer allocates by calling
|
||||
/// the next layer directly never does, and a layer below this one that finds
|
||||
/// its per-object state by that pointer then finds nothing: the validation
|
||||
/// layer aborts, in the first vkCmd* recorded into such a command buffer. The
|
||||
/// loader hands every layer this callback at vkCreateDevice for exactly this.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// `object` must be a queue or command buffer of `ds`'s device.
|
||||
pub unsafe fn stamp(ds: &DeviceState, object: *mut c_void) {
|
||||
if let Some(set) = ds.set_loader_data
|
||||
&& unsafe { set(ds.raw, object) } != vk::Result::SUCCESS
|
||||
{
|
||||
log::warn!("vkSetDeviceLoaderData refused an object of the layer's own");
|
||||
}
|
||||
}
|
||||
|
||||
/// vkAllocateCommandBuffers for the encoder, whose command buffers are the
|
||||
/// layer's own: allocated below the loader, so stamped here. See [`stamp`].
|
||||
pub unsafe extern "system" fn encoder_allocate_command_buffers(
|
||||
device: vk::Device,
|
||||
p_allocate_info: *const vk::CommandBufferAllocateInfo<'_>,
|
||||
p_command_buffers: *mut vk::CommandBuffer,
|
||||
) -> vk::Result {
|
||||
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
|
||||
let Some(ds) = DEVICE_STATE.get(&key).map(|s| s.clone()) else {
|
||||
return vk::Result::ERROR_INITIALIZATION_FAILED;
|
||||
};
|
||||
let result =
|
||||
unsafe { (ds.fp.allocate_command_buffers)(device, p_allocate_info, p_command_buffers) };
|
||||
if result == vk::Result::SUCCESS {
|
||||
let count = unsafe { (*p_allocate_info).command_buffer_count } as usize;
|
||||
for cb in unsafe { std::slice::from_raw_parts(p_command_buffers, count) } {
|
||||
unsafe { stamp(&ds, cb.as_raw() as *mut c_void) };
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Whether the game's queues in `family` were created internally synchronized
|
||||
/// for the encoder to share.
|
||||
fn shares_family(ds: &DeviceState, family: u32) -> bool {
|
||||
ds.shared
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.queues.internally_synchronized.contains(&family))
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "system" fn vkGetDeviceQueue2(
|
||||
device: vk::Device,
|
||||
p_queue_info: *const vk::DeviceQueueInfo2,
|
||||
p_queue: *mut vk::Queue,
|
||||
) {
|
||||
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
|
||||
let Some(ds) = DEVICE_STATE.get(&key) else {
|
||||
return;
|
||||
};
|
||||
let Some(get2) = ds.fp.get_device_queue2 else {
|
||||
return;
|
||||
};
|
||||
// The flags have to match the ones the queue was created with, and for a
|
||||
// shared family the layer added one the game does not know about.
|
||||
let mut info = unsafe { *p_queue_info };
|
||||
if shares_family(&ds, info.queue_family_index) {
|
||||
info.flags |= vk::DeviceQueueCreateFlags::INTERNALLY_SYNCHRONIZED_KHR;
|
||||
}
|
||||
unsafe { get2(device, &info, p_queue) };
|
||||
let queue = unsafe { *p_queue };
|
||||
unsafe { stamp(&ds, queue.as_raw() as *mut c_void) };
|
||||
QUEUE_TO_DEVICE_KEY.insert(queue.as_raw(), key);
|
||||
crate::state::QUEUE_TO_FAMILY.insert(queue.as_raw(), info.queue_family_index);
|
||||
}
|
||||
|
||||
/// Enumerate device extensions supported by the physical device.
|
||||
unsafe fn enumerate_device_extensions(
|
||||
istate: &crate::dispatch::NextInstanceFn,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// discovery.rs — Phase 6: per-draw logging for shader hash discovery
|
||||
//
|
||||
// When HUDLESS_DISCOVER=1 is set, every vkCmdDraw* call is logged to:
|
||||
// /tmp/hudless_discover_$EXE.log
|
||||
// /tmp/nescapture_discover_$EXE.log
|
||||
//
|
||||
// Log format:
|
||||
// frame=0001 draw=00042 vert=0x1a2b3c4d5e6f7890 frag=0xaabbccddeeff0011 verts=6 blend=true depth=false
|
||||
@@ -40,7 +40,7 @@ fn ensure_log_file() -> bool {
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
});
|
||||
|
||||
let path = format!("/tmp/hudless_discover_{}.log", exe_name);
|
||||
let path = format!("/tmp/nescapture_discover_{}.log", exe_name);
|
||||
match OpenOptions::new().create(true).append(true).open(&path) {
|
||||
Ok(f) => {
|
||||
log::info!("discovery logging to {}", path);
|
||||
|
||||
@@ -38,6 +38,8 @@ pub type PFN_vkDestroyDevice =
|
||||
unsafe extern "system" fn(vk::Device, *const vk::AllocationCallbacks);
|
||||
|
||||
pub type PFN_vkGetDeviceQueue = unsafe extern "system" fn(vk::Device, u32, u32, *mut vk::Queue);
|
||||
pub type PFN_vkGetDeviceQueue2 =
|
||||
unsafe extern "system" fn(vk::Device, *const vk::DeviceQueueInfo2, *mut vk::Queue);
|
||||
|
||||
pub type PFN_vkQueuePresentKHR =
|
||||
unsafe extern "system" fn(vk::Queue, *const vk::PresentInfoKHR) -> vk::Result;
|
||||
@@ -190,20 +192,6 @@ pub type PFN_vkGetImageSubresourceLayout = unsafe extern "system" fn(
|
||||
*mut vk::SubresourceLayout,
|
||||
);
|
||||
|
||||
// DRM format modifiers. Both optional: without them the capture ring stays
|
||||
// linear, which is what it was before it could be anything else.
|
||||
pub type PFN_vkGetPhysicalDeviceFormatProperties2 = unsafe extern "system" fn(
|
||||
vk::PhysicalDevice,
|
||||
vk::Format,
|
||||
*mut vk::FormatProperties2<'_>,
|
||||
);
|
||||
|
||||
pub type PFN_vkGetImageDrmFormatModifierPropertiesEXT = unsafe extern "system" fn(
|
||||
vk::Device,
|
||||
vk::Image,
|
||||
*mut vk::ImageDrmFormatModifierPropertiesEXT<'_>,
|
||||
) -> vk::Result;
|
||||
|
||||
// Timestamp queries around the capture blit. All optional: without them the
|
||||
// blit's GPU cost is simply not reported.
|
||||
pub type PFN_vkGetPhysicalDeviceProperties =
|
||||
@@ -219,21 +207,14 @@ pub type PFN_vkCreateQueryPool = unsafe extern "system" fn(
|
||||
*mut vk::QueryPool,
|
||||
) -> vk::Result;
|
||||
|
||||
pub type PFN_vkDestroyQueryPool = unsafe extern "system" fn(
|
||||
vk::Device,
|
||||
vk::QueryPool,
|
||||
*const vk::AllocationCallbacks,
|
||||
);
|
||||
pub type PFN_vkDestroyQueryPool =
|
||||
unsafe extern "system" fn(vk::Device, vk::QueryPool, *const vk::AllocationCallbacks);
|
||||
|
||||
pub type PFN_vkCmdResetQueryPool =
|
||||
unsafe extern "system" fn(vk::CommandBuffer, vk::QueryPool, u32, u32);
|
||||
|
||||
pub type PFN_vkCmdWriteTimestamp = unsafe extern "system" fn(
|
||||
vk::CommandBuffer,
|
||||
vk::PipelineStageFlags,
|
||||
vk::QueryPool,
|
||||
u32,
|
||||
);
|
||||
pub type PFN_vkCmdWriteTimestamp =
|
||||
unsafe extern "system" fn(vk::CommandBuffer, vk::PipelineStageFlags, vk::QueryPool, u32);
|
||||
|
||||
pub type PFN_vkGetQueryPoolResults = unsafe extern "system" fn(
|
||||
vk::Device,
|
||||
@@ -246,13 +227,6 @@ pub type PFN_vkGetQueryPoolResults = unsafe extern "system" fn(
|
||||
vk::QueryResultFlags,
|
||||
) -> vk::Result;
|
||||
|
||||
// DMA-BUF fd export (used to share final_image with pixelforge zero-copy)
|
||||
pub type PFN_vkGetMemoryFdKHR = unsafe extern "system" fn(
|
||||
vk::Device,
|
||||
*const vk::MemoryGetFdInfoKHR,
|
||||
*mut std::os::raw::c_int,
|
||||
) -> vk::Result;
|
||||
|
||||
// ── Phase 4: Synchronisation ─────────────────────────────────────────────────
|
||||
|
||||
pub type PFN_vkCreateFence = unsafe extern "system" fn(
|
||||
@@ -366,13 +340,14 @@ pub type PFN_vkResetCommandBuffer =
|
||||
// ── Dispatch table structs ────────────────────────────────────────────────────
|
||||
|
||||
pub struct NextInstanceFn {
|
||||
/// The instance itself, for building the encoder's view of it.
|
||||
pub instance: vk::Instance,
|
||||
/// The Vulkan version the application asked for, which bounds what core
|
||||
/// functionality exists on its devices.
|
||||
pub api_version: u32,
|
||||
pub get_instance_proc_addr: PFN_vkGetInstanceProcAddr,
|
||||
pub destroy_instance: PFN_vkDestroyInstance,
|
||||
pub get_physical_device_memory_properties: PFN_vkGetPhysicalDeviceMemoryProperties,
|
||||
/// `None` on an instance below Vulkan 1.1 without
|
||||
/// `VK_KHR_get_physical_device_properties2`. Without it the modifier list
|
||||
/// cannot be queried and the capture ring stays linear.
|
||||
pub get_physical_device_format_properties2: Option<PFN_vkGetPhysicalDeviceFormatProperties2>,
|
||||
/// Needed for `timestampPeriod`, which turns device ticks into nanoseconds.
|
||||
pub get_physical_device_properties: Option<PFN_vkGetPhysicalDeviceProperties>,
|
||||
/// Needed for a queue family's `timestampValidBits`. A family reporting
|
||||
@@ -388,6 +363,9 @@ pub struct NextDeviceFn {
|
||||
pub get_device_proc_addr: PFN_vkGetDeviceProcAddr,
|
||||
pub destroy_device: PFN_vkDestroyDevice,
|
||||
pub get_device_queue: PFN_vkGetDeviceQueue,
|
||||
/// Core in 1.1. Needed to fetch queues created with flags, which is how a
|
||||
/// queue shared with the encoder is created.
|
||||
pub get_device_queue2: Option<PFN_vkGetDeviceQueue2>,
|
||||
pub queue_present_khr: Option<PFN_vkQueuePresentKHR>,
|
||||
|
||||
// Phase 1
|
||||
@@ -421,15 +399,6 @@ pub struct NextDeviceFn {
|
||||
pub cmd_pipeline_barrier: PFN_vkCmdPipelineBarrier,
|
||||
pub cmd_copy_image: PFN_vkCmdCopyImage,
|
||||
pub get_image_subresource_layout: PFN_vkGetImageSubresourceLayout,
|
||||
/// `None` when `VK_KHR_external_memory_fd` is unavailable.
|
||||
/// Required for DMA-BUF export to pixelforge's VkDevice.
|
||||
pub get_memory_fd_khr: Option<PFN_vkGetMemoryFdKHR>,
|
||||
/// `None` when `VK_EXT_image_drm_format_modifier` was not enabled. The
|
||||
/// driver picks the modifier from the list it is offered, so this is how
|
||||
/// the layer learns which one it actually got — and the importer needs the
|
||||
/// exact value, not the list.
|
||||
pub get_image_drm_format_modifier_properties_ext:
|
||||
Option<PFN_vkGetImageDrmFormatModifierPropertiesEXT>,
|
||||
|
||||
// Phase 4 — blit timing. All-or-nothing: the ring only times the blit when
|
||||
// every one of these loaded and the presenting queue family can timestamp.
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
//! DMA-BUF import support for zero-copy video encoding.
|
||||
//!
|
||||
//! This module provides the ability to import Linux DMA-BUF file descriptors as
|
||||
//! Vulkan images for direct video encoding without CPU-side copies.
|
||||
//!
|
||||
//! `DmaBufImporter` caches imported Vulkan resources per compositor buffer index
|
||||
//! so that pre-allocated GBM buffers are imported only once. Subsequent frames
|
||||
//! from the same buffer reuse the cached `VkImage` and `VkDeviceMemory`,
|
||||
//! eliminating per-frame Vulkan object creation and layout transitions.
|
||||
|
||||
use anyhow::Result;
|
||||
use ash::vk;
|
||||
use log::debug;
|
||||
use pixelforge::VideoContext;
|
||||
use std::os::fd::RawFd;
|
||||
use std::os::unix::io::{BorrowedFd, IntoRawFd};
|
||||
|
||||
/// Information about a single DMA-BUF plane.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct DmaBufPlane {
|
||||
/// File descriptor for the DMA-BUF.
|
||||
pub fd: RawFd,
|
||||
/// Offset within the DMA-BUF to the start of this plane.
|
||||
pub offset: u32,
|
||||
/// Row stride in bytes.
|
||||
pub stride: u32,
|
||||
/// DRM format modifier.
|
||||
pub modifier: u64,
|
||||
}
|
||||
|
||||
/// Cached Vulkan resources for a single compositor buffer slot.
|
||||
struct CachedImport {
|
||||
image: vk::Image,
|
||||
memory: vk::DeviceMemory,
|
||||
}
|
||||
|
||||
/// Importer for DMA-BUF file descriptors into Vulkan images.
|
||||
///
|
||||
/// Owns a per-buffer-index cache of `VkImage` + `VkDeviceMemory`.
|
||||
/// Layout transitions are deferred to the consumer (e.g. `ColorConverter`)
|
||||
/// to avoid a separate GPU submission per first-time import.
|
||||
pub struct DmaBufImporter {
|
||||
context: VideoContext,
|
||||
external_memory_fd: ash::khr::external_memory_fd::Device,
|
||||
/// Per-buffer-index cache. Index corresponds to `ExportedFrame::buffer_index`.
|
||||
cached_imports: Vec<Option<CachedImport>>,
|
||||
}
|
||||
|
||||
impl DmaBufImporter {
|
||||
/// Create a new DMA-BUF importer.
|
||||
pub fn new(context: VideoContext) -> Result<Self> {
|
||||
let external_memory_fd =
|
||||
ash::khr::external_memory_fd::Device::load(context.instance(), context.device());
|
||||
|
||||
Ok(Self {
|
||||
context,
|
||||
external_memory_fd,
|
||||
cached_imports: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Import a DMA-BUF as a Vulkan image, reusing a cached import when
|
||||
/// the same `buffer_index` has been seen before.
|
||||
///
|
||||
/// The `format` parameter specifies the Vulkan format matching the DMA-BUF
|
||||
/// pixel format (e.g. `B8G8R8A8_UNORM` for SDR, `A2B10G10R10_UNORM_PACK32`
|
||||
/// for 10-bit HDR, `R16G16B16A16_SFLOAT` for FP16 HDR).
|
||||
///
|
||||
/// Returns `(image, needs_transition)` where `needs_transition` is `true`
|
||||
/// for first-time imports whose image is still in `UNDEFINED` layout.
|
||||
/// The caller is responsible for transitioning the image (e.g. by passing
|
||||
/// the appropriate `src_layout` to `ColorConverter::convert`).
|
||||
pub fn import_or_reuse(
|
||||
&mut self,
|
||||
buffer_index: usize,
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: vk::Format,
|
||||
planes: &[DmaBufPlane],
|
||||
) -> Result<(vk::Image, bool)> {
|
||||
// Grow the cache vector if needed.
|
||||
if self.cached_imports.len() <= buffer_index {
|
||||
self.cached_imports.resize_with(buffer_index + 1, || None);
|
||||
}
|
||||
|
||||
if let Some(cached) = &self.cached_imports[buffer_index] {
|
||||
return Ok((cached.image, false));
|
||||
}
|
||||
|
||||
// First time seeing this buffer — full import.
|
||||
debug!(
|
||||
"First import for buffer {buffer_index}: {}x{}, format={:?}, fd={}, stride={}, modifier={:#x}",
|
||||
width, height, format, planes[0].fd, planes[0].stride, planes[0].modifier
|
||||
);
|
||||
|
||||
let (image, memory) = self.import_internal(width, height, format, planes)?;
|
||||
|
||||
self.cached_imports[buffer_index] = Some(CachedImport { image, memory });
|
||||
Ok((image, true))
|
||||
}
|
||||
|
||||
/// Perform the raw Vulkan import of a DMA-BUF with the specified format.
|
||||
///
|
||||
/// Returns the `(VkImage, VkDeviceMemory)` pair. The image is in
|
||||
/// `UNDEFINED` layout; the caller must transition it.
|
||||
fn import_internal(
|
||||
&self,
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: vk::Format,
|
||||
planes: &[DmaBufPlane],
|
||||
) -> Result<(vk::Image, vk::DeviceMemory)> {
|
||||
if planes.is_empty() {
|
||||
return Err(anyhow::anyhow!("At least one DMA-BUF plane is required"));
|
||||
}
|
||||
|
||||
let device = self.context.device();
|
||||
|
||||
// Build DRM format modifier plane layouts for all planes.
|
||||
// AMD modifiers (e.g. tiled/DCC) may require multiple planes;
|
||||
// the layout count must match the modifier's expected plane count.
|
||||
let plane_layouts: Vec<vk::SubresourceLayout> = planes
|
||||
.iter()
|
||||
.map(|p| {
|
||||
vk::SubresourceLayout::default()
|
||||
.offset(p.offset as u64)
|
||||
.row_pitch(p.stride as u64)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let modifier = planes[0].modifier;
|
||||
let mut drm_format_modifier_info =
|
||||
vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
|
||||
.drm_format_modifier(modifier)
|
||||
.plane_layouts(&plane_layouts);
|
||||
|
||||
let mut external_memory_info = vk::ExternalMemoryImageCreateInfo::default()
|
||||
.handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
|
||||
external_memory_info.p_next = &mut drm_format_modifier_info
|
||||
as *mut vk::ImageDrmFormatModifierExplicitCreateInfoEXT
|
||||
as *mut _;
|
||||
|
||||
let mut image_create_info = vk::ImageCreateInfo::default()
|
||||
.image_type(vk::ImageType::TYPE_2D)
|
||||
.format(format)
|
||||
.extent(vk::Extent3D {
|
||||
width,
|
||||
height,
|
||||
depth: 1,
|
||||
})
|
||||
.mip_levels(1)
|
||||
.array_layers(1)
|
||||
.samples(vk::SampleCountFlags::TYPE_1)
|
||||
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
|
||||
.usage(vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::SAMPLED)
|
||||
.sharing_mode(vk::SharingMode::EXCLUSIVE)
|
||||
.initial_layout(vk::ImageLayout::UNDEFINED);
|
||||
image_create_info.p_next =
|
||||
&mut external_memory_info as *mut vk::ExternalMemoryImageCreateInfo as *mut _;
|
||||
|
||||
let image = unsafe { device.create_image(&image_create_info, None) }
|
||||
.map_err(|e| anyhow::anyhow!("DMA-BUF image creation: {e}"))?;
|
||||
|
||||
// Memory requirements.
|
||||
let mem_requirements = unsafe { device.get_image_memory_requirements(image) };
|
||||
|
||||
// FD memory properties.
|
||||
let mut memory_fd_properties = vk::MemoryFdPropertiesKHR::default();
|
||||
unsafe {
|
||||
self.external_memory_fd.get_memory_fd_properties(
|
||||
vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
|
||||
planes[0].fd,
|
||||
&mut memory_fd_properties,
|
||||
)
|
||||
}
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get memory FD properties: {e}"))?;
|
||||
|
||||
// Duplicate the FD — vkAllocateMemory consumes it.
|
||||
let fd = unsafe { BorrowedFd::borrow_raw(planes[0].fd) }
|
||||
.try_clone_to_owned()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to duplicate DMA-BUF FD: {e}"))?
|
||||
.into_raw_fd();
|
||||
|
||||
let mut import_memory_fd_info = vk::ImportMemoryFdInfoKHR::default()
|
||||
.handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
|
||||
.fd(fd);
|
||||
|
||||
let memory_type_bits =
|
||||
mem_requirements.memory_type_bits & memory_fd_properties.memory_type_bits;
|
||||
|
||||
debug!(
|
||||
"Memory allocation: size={}, image_type_bits={:#x}, fd_type_bits={:#x}, combined={:#x}",
|
||||
mem_requirements.size,
|
||||
mem_requirements.memory_type_bits,
|
||||
memory_fd_properties.memory_type_bits,
|
||||
memory_type_bits
|
||||
);
|
||||
|
||||
let memory_type_index = self
|
||||
.context
|
||||
.find_memory_type(memory_type_bits, vk::MemoryPropertyFlags::empty())
|
||||
.ok_or_else(|| {
|
||||
// The numbers, in the error rather than in the debug! above
|
||||
// it: fd_type_bits=0 means the driver could not resolve the
|
||||
// descriptor at all, which is a different fault from a
|
||||
// mismatch, and the difference is the whole diagnosis.
|
||||
anyhow::anyhow!(
|
||||
"No suitable memory type for DMA-BUF import: \
|
||||
image_type_bits={:#x} & fd_type_bits={:#x} = {:#x}, \
|
||||
size={}, format={:?}, modifier={:#x}, stride={}",
|
||||
mem_requirements.memory_type_bits,
|
||||
memory_fd_properties.memory_type_bits,
|
||||
memory_type_bits,
|
||||
mem_requirements.size,
|
||||
format,
|
||||
planes[0].modifier,
|
||||
planes[0].stride
|
||||
)
|
||||
})?;
|
||||
|
||||
// Dedicated allocation (required by many drivers for external memory).
|
||||
let mut dedicated_alloc_info = vk::MemoryDedicatedAllocateInfo::default().image(image);
|
||||
import_memory_fd_info.p_next =
|
||||
&mut dedicated_alloc_info as *mut vk::MemoryDedicatedAllocateInfo as *mut _;
|
||||
|
||||
let mut alloc_info = vk::MemoryAllocateInfo::default()
|
||||
.allocation_size(mem_requirements.size)
|
||||
.memory_type_index(memory_type_index);
|
||||
alloc_info.p_next = &mut import_memory_fd_info as *mut vk::ImportMemoryFdInfoKHR as *mut _;
|
||||
|
||||
let memory = unsafe { device.allocate_memory(&alloc_info, None) }.map_err(|e| {
|
||||
unsafe { device.destroy_image(image, None) };
|
||||
anyhow::anyhow!("DMA-BUF memory import: {e}")
|
||||
})?;
|
||||
|
||||
if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } {
|
||||
unsafe {
|
||||
device.free_memory(memory, None);
|
||||
device.destroy_image(image, None);
|
||||
}
|
||||
return Err(anyhow::anyhow!("DMA-BUF memory bind: {e}"));
|
||||
}
|
||||
|
||||
Ok((image, memory))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DmaBufImporter {
|
||||
fn drop(&mut self) {
|
||||
let device = self.context.device();
|
||||
unsafe {
|
||||
// Clean up cached imports.
|
||||
for cached in self.cached_imports.drain(..).flatten() {
|
||||
device.destroy_image(cached.image, None);
|
||||
device.free_memory(cached.memory, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2604
-356
File diff suppressed because it is too large
Load Diff
@@ -41,7 +41,42 @@ pub unsafe extern "system" fn vkCreateInstance(
|
||||
let next_create: PFN_vkCreateInstance =
|
||||
unsafe { load_instance_fn(next_gipa, vk::Instance::null(), b"vkCreateInstance\0") };
|
||||
|
||||
let result = unsafe { next_create(p_create_info, p_allocator, p_instance) };
|
||||
// The encoder needs Vulkan 1.1 on the game's instance to run on the game's
|
||||
// device. An application asking for 1.0 is raised to 1.1 where the loader
|
||||
// has it: 1.1 only adds to 1.0, so nothing the application can do behaves
|
||||
// any differently, and without it the encoder needs a device of its own.
|
||||
let asked = unsafe {
|
||||
let app = (*p_create_info).p_application_info;
|
||||
if app.is_null() {
|
||||
vk::API_VERSION_1_0
|
||||
} else {
|
||||
(*app).api_version
|
||||
}
|
||||
};
|
||||
let mut raised_app;
|
||||
let mut raised_ci;
|
||||
let mut create_info = p_create_info;
|
||||
let mut api_version = asked;
|
||||
if asked < vk::API_VERSION_1_1 && unsafe { loader_version(next_gipa) } >= vk::API_VERSION_1_1 {
|
||||
raised_app = unsafe {
|
||||
let app = (*p_create_info).p_application_info;
|
||||
if app.is_null() {
|
||||
vk::ApplicationInfo::default()
|
||||
} else {
|
||||
*app
|
||||
}
|
||||
};
|
||||
raised_app.api_version = vk::API_VERSION_1_1;
|
||||
raised_ci = unsafe { *p_create_info };
|
||||
raised_ci.p_application_info = &raised_app;
|
||||
create_info = &raised_ci;
|
||||
api_version = vk::API_VERSION_1_1;
|
||||
log::info!(
|
||||
"instance asked for Vulkan 1.0; created as 1.1 so the encoder can share its devices"
|
||||
);
|
||||
}
|
||||
|
||||
let result = unsafe { next_create(create_info, p_allocator, p_instance) };
|
||||
if result != vk::Result::SUCCESS {
|
||||
return result;
|
||||
}
|
||||
@@ -51,6 +86,8 @@ pub unsafe extern "system" fn vkCreateInstance(
|
||||
|
||||
// ── Build and store per-instance dispatch table ───────────────────────────
|
||||
let istate = Arc::new(NextInstanceFn {
|
||||
instance,
|
||||
api_version,
|
||||
get_instance_proc_addr: next_gipa,
|
||||
destroy_instance: unsafe { load_instance_fn(next_gipa, instance, b"vkDestroyInstance\0") },
|
||||
get_physical_device_memory_properties: unsafe {
|
||||
@@ -60,13 +97,6 @@ pub unsafe extern "system" fn vkCreateInstance(
|
||||
b"vkGetPhysicalDeviceMemoryProperties\0",
|
||||
)
|
||||
},
|
||||
get_physical_device_format_properties2: unsafe {
|
||||
crate::try_load_instance_fn(
|
||||
next_gipa,
|
||||
instance,
|
||||
b"vkGetPhysicalDeviceFormatProperties2\0",
|
||||
)
|
||||
},
|
||||
get_physical_device_properties: unsafe {
|
||||
crate::try_load_instance_fn(next_gipa, instance, b"vkGetPhysicalDeviceProperties\0")
|
||||
},
|
||||
@@ -85,6 +115,26 @@ pub unsafe extern "system" fn vkCreateInstance(
|
||||
vk::Result::SUCCESS
|
||||
}
|
||||
|
||||
/// The highest instance version the loader below supports; 1.0 when it cannot
|
||||
/// say, since vkEnumerateInstanceVersion is itself a 1.1 addition.
|
||||
unsafe fn loader_version(next_gipa: crate::dispatch::PFN_vkGetInstanceProcAddr) -> u32 {
|
||||
type Enumerate = unsafe extern "system" fn(*mut u32) -> vk::Result;
|
||||
let Some(f) = (unsafe {
|
||||
crate::try_load_instance_fn::<Enumerate>(
|
||||
next_gipa,
|
||||
vk::Instance::null(),
|
||||
b"vkEnumerateInstanceVersion\0",
|
||||
)
|
||||
}) else {
|
||||
return vk::API_VERSION_1_0;
|
||||
};
|
||||
let mut version = vk::API_VERSION_1_0;
|
||||
if unsafe { f(&mut version) } != vk::Result::SUCCESS {
|
||||
return vk::API_VERSION_1_0;
|
||||
}
|
||||
version
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "system" fn vkDestroyInstance(
|
||||
instance: vk::Instance,
|
||||
|
||||
@@ -27,20 +27,20 @@ mod config;
|
||||
mod device;
|
||||
mod discovery;
|
||||
mod dispatch;
|
||||
mod dmabuf_import;
|
||||
mod encode;
|
||||
mod framebuffer;
|
||||
mod instance;
|
||||
mod memory;
|
||||
mod modifiers;
|
||||
mod pacing;
|
||||
mod pipeline;
|
||||
mod present;
|
||||
mod rate_probe;
|
||||
mod shader;
|
||||
mod shared;
|
||||
mod slots;
|
||||
mod state;
|
||||
mod timing;
|
||||
mod swapchain;
|
||||
mod timing;
|
||||
|
||||
use commands::{
|
||||
vkCmdBeginRenderPass, vkCmdBeginRenderingKHR, vkCmdBindPipeline, vkCmdDraw, vkCmdDrawIndexed,
|
||||
@@ -48,7 +48,7 @@ use commands::{
|
||||
vkCmdDrawIndirect, vkCmdDrawIndirectCount, vkCmdDrawIndirectCountKHR, vkCmdEndRenderPass,
|
||||
vkCmdEndRenderingKHR,
|
||||
};
|
||||
use device::{vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue};
|
||||
use device::{vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue, vkGetDeviceQueue2};
|
||||
use framebuffer::{
|
||||
vkAllocateCommandBuffers, vkCreateFramebuffer, vkCreateImageView, vkDestroyFramebuffer,
|
||||
vkDestroyImageView, vkFreeCommandBuffers,
|
||||
@@ -100,10 +100,16 @@ pub(crate) struct VkLayerDeviceLink {
|
||||
pfnNextGetDeviceProcAddr: Option<PFN_vkGetDeviceProcAddr>,
|
||||
}
|
||||
|
||||
/// Stamps the loader's dispatch data into a dispatchable object a layer
|
||||
/// created itself. See [`crate::device::stamp`].
|
||||
pub(crate) type PFN_vkSetDeviceLoaderData =
|
||||
unsafe extern "system" fn(vk::Device, *mut c_void) -> vk::Result;
|
||||
|
||||
#[repr(C)]
|
||||
pub(crate) union VkLayerCreateInfoU {
|
||||
pub pLayerInfo: *mut VkLayerInstanceLink,
|
||||
pub pDeviceLayerInfo: *mut VkLayerDeviceLink,
|
||||
pub pfnSetDeviceLoaderData: Option<PFN_vkSetDeviceLoaderData>,
|
||||
}
|
||||
|
||||
const VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO: i32 = 47;
|
||||
@@ -271,6 +277,7 @@ unsafe fn match_device_fn(name: &[u8]) -> Option<RawFn> {
|
||||
b"vkCreateDevice" => Some(to_raw(vkCreateDevice as *const () as usize)),
|
||||
b"vkDestroyDevice" => Some(to_raw(vkDestroyDevice as *const () as usize)),
|
||||
b"vkGetDeviceQueue" => Some(to_raw(vkGetDeviceQueue as *const () as usize)),
|
||||
b"vkGetDeviceQueue2" => Some(to_raw(vkGetDeviceQueue2 as *const () as usize)),
|
||||
b"vkQueuePresentKHR" => Some(to_raw(vkQueuePresentKHR as *const () as usize)),
|
||||
|
||||
b"vkCreateShaderModule" => Some(to_raw(vkCreateShaderModule as *const () as usize)),
|
||||
|
||||
@@ -43,7 +43,12 @@ pub fn pick_memory_type(types: &[MemoryType], bits: u32, want: Want) -> Option<u
|
||||
|
||||
match want {
|
||||
Want::DeviceLocal => (0..types.len())
|
||||
.find(|&i| allowed(i) && types[i].flags.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL))
|
||||
.find(|&i| {
|
||||
allowed(i)
|
||||
&& types[i]
|
||||
.flags
|
||||
.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL)
|
||||
})
|
||||
.or_else(|| (0..types.len()).find(|&i| allowed(i)))
|
||||
.map(|i| i as u32),
|
||||
Want::HostCoherent => (0..types.len())
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// modifiers.rs — choosing a DRM format modifier for the capture ring
|
||||
//
|
||||
// The importer has always been able to take a tiled buffer: `dmabuf_import.rs`
|
||||
// builds `VkImageDrmFormatModifierExplicitCreateInfoEXT` with per-plane
|
||||
// layouts and creates the image with `DRM_FORMAT_MODIFIER_EXT` tiling. Only
|
||||
// the producer was linear — hard-coded `ImageTiling::LINEAR` and a `modifier`
|
||||
// of zero passed down with every frame — so every capture detiled a full frame
|
||||
// on the write and the encoder sampled a linear image on the read.
|
||||
//
|
||||
// Picking the modifier is the whole of the decision and it is pure, so it is
|
||||
// here and tested rather than buried in an unsafe block.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// One entry of `VkDrmFormatModifierPropertiesListEXT`, already filtered to
|
||||
/// modifiers whose tiling features cover both our write and the encoder's read.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ModifierProps {
|
||||
pub modifier: u64,
|
||||
pub plane_count: u32,
|
||||
}
|
||||
|
||||
/// `DRM_FORMAT_MOD_LINEAR`.
|
||||
pub const LINEAR: u64 = 0;
|
||||
|
||||
/// `DRM_FORMAT_MOD_INVALID`, which a driver may list and which means "let the
|
||||
/// driver choose" — not something to ask for explicitly.
|
||||
pub const INVALID: u64 = 0x00ff_ffff_ffff_ffff;
|
||||
|
||||
/// Pick the modifier to allocate the capture ring with.
|
||||
///
|
||||
/// Single-plane only, and deliberately so. A multi-plane modifier needs an
|
||||
/// offset and a stride per plane on the import side, and the export path hands
|
||||
/// out one fd with one stride — so accepting one would produce an image the far
|
||||
/// side reads at the wrong offsets, which arrives at the right size and frame
|
||||
/// rate carrying nonsense. Prefer any real tiled modifier; fall back to linear,
|
||||
/// which is what the ring used before this existed and always works.
|
||||
pub fn pick_modifier(candidates: &[ModifierProps]) -> Option<ModifierProps> {
|
||||
let usable = |m: &&ModifierProps| m.plane_count == 1 && m.modifier != INVALID;
|
||||
candidates
|
||||
.iter()
|
||||
.filter(usable)
|
||||
.find(|m| m.modifier != LINEAR)
|
||||
.or_else(|| candidates.iter().filter(usable).find(|m| m.modifier == LINEAR))
|
||||
.copied()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const TILED: u64 = 0x0200_0000_0000_0001;
|
||||
|
||||
#[test]
|
||||
fn a_tiled_modifier_beats_linear() {
|
||||
let c = [
|
||||
ModifierProps { modifier: LINEAR, plane_count: 1 },
|
||||
ModifierProps { modifier: TILED, plane_count: 1 },
|
||||
];
|
||||
assert_eq!(pick_modifier(&c).unwrap().modifier, TILED);
|
||||
}
|
||||
|
||||
/// Order must not decide it — the driver lists them in its own order.
|
||||
#[test]
|
||||
fn a_tiled_modifier_wins_from_either_position() {
|
||||
let c = [
|
||||
ModifierProps { modifier: TILED, plane_count: 1 },
|
||||
ModifierProps { modifier: LINEAR, plane_count: 1 },
|
||||
];
|
||||
assert_eq!(pick_modifier(&c).unwrap().modifier, TILED);
|
||||
}
|
||||
|
||||
/// A multi-plane modifier with one exported fd would be imported at the
|
||||
/// wrong plane offsets and produce a corrupt frame rather than an error.
|
||||
#[test]
|
||||
fn multi_plane_modifiers_are_refused() {
|
||||
let c = [
|
||||
ModifierProps { modifier: 0x0200_0000_0000_0002, plane_count: 2 },
|
||||
ModifierProps { modifier: LINEAR, plane_count: 1 },
|
||||
];
|
||||
assert_eq!(pick_modifier(&c).unwrap().modifier, LINEAR);
|
||||
}
|
||||
|
||||
/// A multi-plane tiled modifier must not beat a single-plane linear one
|
||||
/// just for being tiled.
|
||||
#[test]
|
||||
fn tiling_does_not_excuse_a_plane_count_we_cannot_export() {
|
||||
let c = [
|
||||
ModifierProps { modifier: TILED, plane_count: 4 },
|
||||
ModifierProps { modifier: LINEAR, plane_count: 1 },
|
||||
];
|
||||
assert_eq!(pick_modifier(&c).unwrap().modifier, LINEAR);
|
||||
}
|
||||
|
||||
/// `DRM_FORMAT_MOD_INVALID` is not a modifier to ask for.
|
||||
#[test]
|
||||
fn the_invalid_modifier_is_never_chosen() {
|
||||
let c = [ModifierProps { modifier: INVALID, plane_count: 1 }];
|
||||
assert_eq!(pick_modifier(&c), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_usable_is_none() {
|
||||
let c = [ModifierProps { modifier: TILED, plane_count: 4 }];
|
||||
assert_eq!(pick_modifier(&c), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_list_is_none() {
|
||||
assert_eq!(pick_modifier(&[]), None);
|
||||
}
|
||||
}
|
||||
+118
-40
@@ -1,5 +1,5 @@
|
||||
use crate::capture;
|
||||
use crate::encode::{CapturedFrame, FrameSource, PipelineConfig, PipelineHandle};
|
||||
use crate::encode::{CapturedFrame, FrameSource, PipelineConfig, PipelineHandle, PresentStep};
|
||||
use crate::slots::SlotGuard;
|
||||
use crate::state::{DEVICE_STATE, QUEUE_TO_DEVICE_KEY};
|
||||
use ash::vk::{self, Handle};
|
||||
@@ -32,6 +32,7 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
|
||||
// it is the game's own frame time with this layer's cost excluded — the
|
||||
// three spans then partition the wall clock between presents exactly.
|
||||
let entered = std::time::Instant::now();
|
||||
note_present(&ds, PresentStep::InHook);
|
||||
|
||||
ds.frame_counter.fetch_add(1, Ordering::Relaxed);
|
||||
ds.hud_detected_frame.store(false, Ordering::Relaxed);
|
||||
@@ -55,9 +56,8 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
|
||||
// Rewriting the wait semaphores is only well defined for a single
|
||||
// swapchain. A multi-swapchain present is rare enough that passing it
|
||||
// through untouched beats getting the interposition subtly wrong.
|
||||
let single_swapchain = pi.swapchain_count == 1
|
||||
&& !pi.p_swapchains.is_null()
|
||||
&& !pi.p_image_indices.is_null();
|
||||
let single_swapchain =
|
||||
pi.swapchain_count == 1 && !pi.p_swapchains.is_null() && !pi.p_image_indices.is_null();
|
||||
|
||||
let submission = if single_swapchain {
|
||||
unsafe { try_capture(&ds, queue, pi) }
|
||||
@@ -68,9 +68,11 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
|
||||
let down_us = std::cell::Cell::new(std::time::Duration::ZERO);
|
||||
let call_down = |info: *const vk::PresentInfoKHR| match ds.fp.queue_present_khr {
|
||||
Some(f) => {
|
||||
note_present(&ds, PresentStep::Presenting);
|
||||
let t = std::time::Instant::now();
|
||||
let r = unsafe { f(queue, info) };
|
||||
down_us.set(t.elapsed());
|
||||
note_present(&ds, PresentStep::InHook);
|
||||
r
|
||||
}
|
||||
None => vk::Result::ERROR_EXTENSION_NOT_PRESENT,
|
||||
@@ -82,7 +84,6 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
|
||||
return r;
|
||||
};
|
||||
|
||||
|
||||
// The blit consumed the application's wait semaphores, so the present waits
|
||||
// on ours instead. Presenting on the originals as well would be a second
|
||||
// wait on an already-consumed signal.
|
||||
@@ -121,11 +122,7 @@ pub unsafe extern "system" fn vkQueuePresentKHR(
|
||||
/// `layer` is everything in this hook that is not the down-call, both sides of
|
||||
/// it added together, so `gap + layer + down` accounts for the wall clock
|
||||
/// between one present and the next with nothing unattributed.
|
||||
fn finish(
|
||||
ds: &crate::state::DeviceState,
|
||||
entered: std::time::Instant,
|
||||
down: std::time::Duration,
|
||||
) {
|
||||
fn finish(ds: &crate::state::DeviceState, entered: std::time::Instant, down: std::time::Duration) {
|
||||
// Everything this hook cost, before any deliberate waiting.
|
||||
let worked = std::time::Instant::now();
|
||||
|
||||
@@ -138,6 +135,7 @@ fn finish(
|
||||
Err(_) => std::time::Duration::ZERO,
|
||||
};
|
||||
if !held.is_zero() {
|
||||
note_present(ds, PresentStep::Holding);
|
||||
std::thread::sleep(held);
|
||||
}
|
||||
|
||||
@@ -167,12 +165,24 @@ fn finish(
|
||||
if let Ok(mut last) = ds.last_present_return.lock() {
|
||||
*last = Some(now);
|
||||
}
|
||||
note_present(ds, PresentStep::InGame);
|
||||
}
|
||||
|
||||
/// Tell the stall watchdog where the game's present thread is. A no-op until
|
||||
/// the pipeline exists.
|
||||
pub fn note_present(ds: &crate::state::DeviceState, step: PresentStep) {
|
||||
if let Ok(enc) = ds.encoder.lock()
|
||||
&& let Some(ref h) = *enc
|
||||
{
|
||||
h.note_present(step);
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything the present hook needs to carry from the blit to the worker.
|
||||
struct Submission {
|
||||
slot: SlotGuard,
|
||||
present_wait: vk::Semaphore,
|
||||
blit: Option<pixelforge::TimelinePoint>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
sc_fmt: vk::Format,
|
||||
@@ -185,6 +195,13 @@ unsafe fn try_capture(
|
||||
pi: &vk::PresentInfoKHR,
|
||||
) -> Option<Submission> {
|
||||
let image_index = unsafe { *pi.p_image_indices } as usize;
|
||||
// An image acquired from a retired swapchain may still be presented, and
|
||||
// the tracked images belong to its replacement: the same index there is a
|
||||
// different image, possibly of a different size.
|
||||
let presented = unsafe { *pi.p_swapchains };
|
||||
if !crate::swapchain::is_current(*ds.swapchain.lock().ok()?, presented) {
|
||||
return None;
|
||||
}
|
||||
let (sc_image, sc_fmt, sc_ext, image_count) = {
|
||||
let images = ds.swapchain_images.lock().ok()?;
|
||||
let fmt = *ds.swapchain_format.lock().ok()?;
|
||||
@@ -196,7 +213,7 @@ unsafe fn try_capture(
|
||||
};
|
||||
|
||||
// Gate before any GPU work is queued. A game presenting faster than the
|
||||
// target would otherwise pay a full blit and DMA-BUF export for frames the
|
||||
// target would otherwise pay a full blit for frames the
|
||||
// encoder throws away moments later.
|
||||
let present_time = std::time::Instant::now();
|
||||
let admitted = match ds.frame_gate.lock() {
|
||||
@@ -221,14 +238,14 @@ unsafe fn try_capture(
|
||||
}
|
||||
}
|
||||
|
||||
let app_waits: &[vk::Semaphore] = if pi.wait_semaphore_count == 0 || pi.p_wait_semaphores.is_null()
|
||||
{
|
||||
&[]
|
||||
} else {
|
||||
unsafe {
|
||||
std::slice::from_raw_parts(pi.p_wait_semaphores, pi.wait_semaphore_count as usize)
|
||||
}
|
||||
};
|
||||
let app_waits: &[vk::Semaphore] =
|
||||
if pi.wait_semaphore_count == 0 || pi.p_wait_semaphores.is_null() {
|
||||
&[]
|
||||
} else {
|
||||
unsafe {
|
||||
std::slice::from_raw_parts(pi.p_wait_semaphores, pi.wait_semaphore_count as usize)
|
||||
}
|
||||
};
|
||||
|
||||
let submission = unsafe {
|
||||
capture::capture_present_frame(
|
||||
@@ -246,6 +263,7 @@ unsafe fn try_capture(
|
||||
Some(Submission {
|
||||
slot: submission.slot,
|
||||
present_wait: submission.present_wait,
|
||||
blit: submission.blit,
|
||||
width: sc_ext.width,
|
||||
height: sc_ext.height,
|
||||
sc_fmt,
|
||||
@@ -273,6 +291,7 @@ fn queue_for_encode(ds: &crate::state::DeviceState, submission: Submission) {
|
||||
vk_colorspace: ds.swapchain_colorspace.load(Ordering::Relaxed),
|
||||
present_time: submission.present_time,
|
||||
slot: Some(submission.slot),
|
||||
blit: submission.blit,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -303,11 +322,31 @@ fn encoder_ready(ds: &crate::state::DeviceState, ds_key: usize, width: u32, heig
|
||||
log::error!("no encode pipeline configuration; capture disabled");
|
||||
return;
|
||||
};
|
||||
match PipelineHandle::new(cfg) {
|
||||
// On the game's own device where it was created for that, and
|
||||
// on a device of the encoder's own where it was not, or where
|
||||
// adopting it fails.
|
||||
let shared = ds.shared.as_ref().and_then(|s| match s.video_context() {
|
||||
Ok(ctx) => Some(ctx),
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"could not encode on the game's device ({e}); using a device of its own"
|
||||
);
|
||||
None
|
||||
}
|
||||
});
|
||||
let on_shared = shared.is_some();
|
||||
match PipelineHandle::new(cfg, shared) {
|
||||
Ok(h) => {
|
||||
h.set_stall_probe(Box::new(move || describe_gpu(ds_key)));
|
||||
// Before the handle is published: the first capture after
|
||||
// it builds the ring, and must build it for this device.
|
||||
ds.shared_active.store(on_shared, Ordering::Release);
|
||||
if let Ok(mut enc) = ds.encoder.lock() {
|
||||
*enc = Some(h);
|
||||
}
|
||||
if on_shared {
|
||||
log::info!("encoding on the game's own device");
|
||||
}
|
||||
}
|
||||
Err(e) => log::error!("encode pipeline: {e}"),
|
||||
}
|
||||
@@ -316,9 +355,13 @@ fn encoder_ready(ds: &crate::state::DeviceState, ds_key: usize, width: u32, heig
|
||||
false
|
||||
}
|
||||
|
||||
/// Wait for a frame's blit and turn its slot into something the encoder reads.
|
||||
/// Turn a frame's slot into something the encoder reads.
|
||||
///
|
||||
/// This is the CPU handover the two devices need: pixelforge's `VkDevice`
|
||||
/// On a device the encoder shares this is immediate: the slot's image is read
|
||||
/// in place, and the encoder's GPU work waits on the blit's timeline point, so
|
||||
/// nothing here waits at all.
|
||||
///
|
||||
/// Otherwise this is the CPU handover two devices need: pixelforge's `VkDevice`
|
||||
/// shares no timeline with the game's, so no semaphore can bridge them and
|
||||
/// somebody has to block. It used to be a thread of its own between the present
|
||||
/// hook and the encoder; it is now the first thing the encoder thread does with
|
||||
@@ -331,13 +374,19 @@ pub fn resolve_source(
|
||||
) -> Option<FrameSource> {
|
||||
let slot_index = frame.slot.as_ref()?.index();
|
||||
|
||||
if let Some(blit) = frame.blit {
|
||||
let ring = ds.capture_ring.lock().ok()?;
|
||||
let image = ring.as_ref()?.slots.get(slot_index)?.image;
|
||||
return Some(FrameSource::Shared { image, blit });
|
||||
}
|
||||
|
||||
// Copy the handles out and drop the ring lock before waiting: the present
|
||||
// hook needs that lock every frame and must not queue behind a GPU wait.
|
||||
let (fence, dmabuf_fd, stride, modifier, image, memory) = {
|
||||
let (fence, image, memory) = {
|
||||
let ring = ds.capture_ring.lock().ok()?;
|
||||
ring.as_ref()
|
||||
.and_then(|r| r.slots.get(slot_index))
|
||||
.map(|s| (s.fence, s.dmabuf_fd, s.stride, s.modifier, s.image, s.memory))?
|
||||
.map(|s| (s.fence, s.image, s.memory))?
|
||||
};
|
||||
|
||||
let waited = unsafe { (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 1_000_000_000) };
|
||||
@@ -347,30 +396,59 @@ pub fn resolve_source(
|
||||
}
|
||||
|
||||
// After the fence, so the queries have landed and `WAIT` returns at once.
|
||||
if let Some(ns) = unsafe { capture::blit_gpu_time_ns(ds, slot_index) }
|
||||
if let Some(ns) = unsafe { capture::blit_gpu_time_ns(ds, slot_index, true) }
|
||||
&& let Ok(enc) = ds.encoder.lock()
|
||||
&& let Some(ref h) = *enc
|
||||
{
|
||||
h.timing.blit.record(std::time::Duration::from_nanos(ns));
|
||||
}
|
||||
|
||||
if dmabuf_fd >= 0 {
|
||||
let duped = unsafe { libc::dup(dmabuf_fd) };
|
||||
if duped < 0 {
|
||||
log::warn!("dup of capture DMA-BUF failed — frame dropped");
|
||||
return None;
|
||||
}
|
||||
return Some(FrameSource::DmaBuf {
|
||||
fd: duped,
|
||||
stride,
|
||||
// The slot's own modifier. This was hard-coded to zero, which was
|
||||
// true only because the producer could not ask for anything else.
|
||||
modifier,
|
||||
});
|
||||
}
|
||||
|
||||
match unsafe { capture::read_frame_pixels(ds, image, memory, frame.width, frame.height) } {
|
||||
Some(p) if !p.is_empty() => Some(FrameSource::Pixels(p)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the GPU has got through the capture work this layer gave it, for
|
||||
/// the stall watchdog.
|
||||
///
|
||||
/// A blit is queued behind the game's frame on the game's own queue, so a blit
|
||||
/// that never completes means that queue stopped: the game is waiting on the
|
||||
/// GPU. One that did complete means the GPU finished everything in front of
|
||||
/// it, and whatever the game is waiting for is not GPU work on that queue.
|
||||
///
|
||||
/// Never blocks. It runs exactly when something may be holding a lock forever.
|
||||
fn describe_gpu(ds_key: usize) -> String {
|
||||
let Some(ds) = DEVICE_STATE.get(&ds_key).map(|s| s.clone()) else {
|
||||
return "device gone".into();
|
||||
};
|
||||
let slots = format!(
|
||||
"{} of {} capture slots free",
|
||||
ds.capture_slots.available(),
|
||||
crate::state::CAPTURE_SLOTS
|
||||
);
|
||||
let Ok(ring) = ds.capture_ring.try_lock() else {
|
||||
return format!("{slots}; capture ring locked");
|
||||
};
|
||||
let Some(ring) = ring.as_ref() else {
|
||||
return format!("{slots}; no capture ring");
|
||||
};
|
||||
if let Some(shared) = ds.shared.as_ref()
|
||||
&& !ring.blit_timeline.is_null()
|
||||
{
|
||||
let reached = shared
|
||||
.counter(ring.blit_timeline)
|
||||
.map_or("unknown".to_string(), |v| v.to_string());
|
||||
return format!(
|
||||
"{slots}; last blit submitted signals {}, the GPU has reached {reached}",
|
||||
ring.blit_value
|
||||
);
|
||||
}
|
||||
let pending = ring
|
||||
.slots
|
||||
.iter()
|
||||
.filter(|s| unsafe { (ds.fp.wait_for_fences)(ds.raw, 1, &s.fence, vk::TRUE, 0) }
|
||||
== vk::Result::TIMEOUT)
|
||||
.count();
|
||||
format!("{slots}; {pending} blits submitted and not yet complete")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
//! How long the encoder takes to actually reach a bitrate it was told to use.
|
||||
//!
|
||||
//! # Why this exists
|
||||
//!
|
||||
//! The bitrate controller decides how often it is worth deciding. Everything
|
||||
//! else about its design follows from one number nobody has measured: the time
|
||||
//! between `set_target_bitrate` and the encoder actually producing that rate.
|
||||
//!
|
||||
//! Simulated against a measured 1000-mile path, the difference is the whole
|
||||
//! design. With the encoder settling in a quarter second, a controller sampling
|
||||
//! the send queue five times a second holds the queue at 50 ms. With the
|
||||
//! encoder taking a second, the same controller at any rate holds it at about
|
||||
//! 1400 ms, which is unplayable, and sampling faster buys nothing at all --
|
||||
//! there is no point reacting quicker than the thing being steered can move.
|
||||
//!
|
||||
//! So this measures it, rather than picking a control rate and hoping.
|
||||
//!
|
||||
//! # Shape
|
||||
//!
|
||||
//! Self-driving on purpose. The ladder and the dwell are fixed here so that two
|
||||
//! people on two networks produce numbers that can be laid beside each other;
|
||||
//! if the steps came from a person moving a slider, they would not be.
|
||||
//!
|
||||
//! While a sweep is running the encoder ignores bitrates from anywhere else.
|
||||
//! A controller adjusting in the background would be a second hand on the same
|
||||
//! dial, and the measurement would describe the argument rather than the
|
||||
//! encoder.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Targets to step through, in kbps.
|
||||
///
|
||||
/// Large steps and small ones, downward and upward. Downward matters most --
|
||||
/// that is the direction taken under congestion, and the direction where being
|
||||
/// slow costs latency rather than picture -- but an encoder can easily be quick
|
||||
/// one way and slow the other, so both are here.
|
||||
const LADDER: [u32; 6] = [6_000, 1_500, 6_000, 3_000, 1_000, 4_000];
|
||||
|
||||
/// How long to sit at each rung.
|
||||
///
|
||||
/// Long enough to settle and then be seen to be steady. If settling turns out
|
||||
/// to take longer than this, that is itself the answer and it is reported as a
|
||||
/// failure to settle rather than as a number.
|
||||
const DWELL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Window over which the produced rate is measured.
|
||||
///
|
||||
/// Short enough to see a change quickly, long enough that one large frame does
|
||||
/// not look like a rate. At sixty frames a second this averages about thirty.
|
||||
const WINDOW: Duration = Duration::from_millis(500);
|
||||
|
||||
/// How close counts as arrived.
|
||||
const TOLERANCE: f32 = 0.10;
|
||||
|
||||
/// How long it must stay inside the tolerance to count as settled, rather than
|
||||
/// having passed through on the way somewhere else.
|
||||
const HOLD: Duration = Duration::from_millis(500);
|
||||
|
||||
/// What one rung of the ladder turned out to cost.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct StepReport {
|
||||
pub from_kbps: u32,
|
||||
pub to_kbps: u32,
|
||||
/// Milliseconds from the command to the produced rate first being inside
|
||||
/// the tolerance and staying there. `None` if it never settled.
|
||||
pub settle_ms: Option<u32>,
|
||||
pub settle_frames: Option<u32>,
|
||||
/// Produced rate over the last second of the dwell, as a fraction of the
|
||||
/// target. This is the overshoot the controller has to divide out: a
|
||||
/// hardware encoder asked for 1000 does not produce 1000.
|
||||
pub steady_ratio: f32,
|
||||
/// Keyframes during the rung.
|
||||
pub keyframes: u32,
|
||||
/// The largest keyframe seen, in bytes.
|
||||
///
|
||||
/// Reported rather than folded in, because it is a different quantity with
|
||||
/// a different consumer. Settling is about the rate control finding its
|
||||
/// operating point; a keyframe is a single burst handed to the transport
|
||||
/// whole. The controller needs both and must not confuse them: measured at
|
||||
/// 4 s GOP, a run where keyframes were counted in the rate said settling
|
||||
/// took 2266 ms where the same encoder with no keyframes said 450 ms, and
|
||||
/// in one case the number went *down* when keyframes were added. That is
|
||||
/// not an encoder being erratic, it is a window catching an IDR.
|
||||
pub keyframe_bytes: u32,
|
||||
/// How long that keyframe alone occupies the link at this rung's target,
|
||||
/// in milliseconds. This is the burst a queue has to absorb, and the
|
||||
/// reason a queue setpoint cannot simply be set below it.
|
||||
pub keyframe_ms: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Phase {
|
||||
/// Letting the encoder reach a steady state before the first step.
|
||||
WarmUp,
|
||||
Stepping,
|
||||
Done,
|
||||
}
|
||||
|
||||
pub struct RateProbe {
|
||||
phase: Phase,
|
||||
rung: usize,
|
||||
/// When the current rung was commanded.
|
||||
began: Instant,
|
||||
current_kbps: u32,
|
||||
/// `(when, bytes)` inside the measurement window, delta frames only.
|
||||
///
|
||||
/// Keyframes are deliberately absent. One IDR is worth many delta frames,
|
||||
/// so a half-second window containing one reports a rate several times the
|
||||
/// truth, leaves the tolerance band, and restarts the settle clock -- which
|
||||
/// measures the GOP rather than the encoder.
|
||||
samples: Vec<(Instant, u32)>,
|
||||
keyframe_bytes: u32,
|
||||
/// Frames since the current rung was commanded.
|
||||
frames: u32,
|
||||
keyframes: u32,
|
||||
/// When the produced rate first entered the tolerance, if it still is.
|
||||
inside_since: Option<Instant>,
|
||||
settled: Option<(u32, u32)>,
|
||||
/// The last second of the dwell, for the steady-state ratio.
|
||||
steady: Vec<(Instant, u32)>,
|
||||
reports: Vec<StepReport>,
|
||||
}
|
||||
|
||||
impl RateProbe {
|
||||
/// A probe that has not started stepping yet.
|
||||
pub fn new(now: Instant, starting_kbps: u32) -> Self {
|
||||
Self {
|
||||
phase: Phase::WarmUp,
|
||||
rung: 0,
|
||||
began: now,
|
||||
current_kbps: starting_kbps,
|
||||
samples: Vec::new(),
|
||||
keyframe_bytes: 0,
|
||||
frames: 0,
|
||||
keyframes: 0,
|
||||
inside_since: None,
|
||||
settled: None,
|
||||
steady: Vec::new(),
|
||||
reports: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finished(&self) -> bool {
|
||||
self.phase == Phase::Done
|
||||
}
|
||||
|
||||
/// Whether a bitrate from elsewhere should be ignored.
|
||||
pub fn owns_the_bitrate(&self) -> bool {
|
||||
self.phase != Phase::Done
|
||||
}
|
||||
|
||||
/// One encoded frame.
|
||||
pub fn observe(&mut self, now: Instant, bytes: u32, keyframe: bool) {
|
||||
if keyframe {
|
||||
self.keyframe_bytes = self.keyframe_bytes.max(bytes);
|
||||
} else {
|
||||
self.samples.push((now, bytes));
|
||||
}
|
||||
self.samples
|
||||
.retain(|(t, _)| now.duration_since(*t) <= WINDOW);
|
||||
// The steady-state ratio *does* include keyframes: it answers what the
|
||||
// link actually carries for a given target, which is the whole output.
|
||||
self.steady.push((now, bytes));
|
||||
self.steady
|
||||
.retain(|(t, _)| now.duration_since(*t) <= Duration::from_secs(1));
|
||||
if self.phase != Phase::Stepping {
|
||||
return;
|
||||
}
|
||||
self.frames += 1;
|
||||
if keyframe {
|
||||
self.keyframes += 1;
|
||||
}
|
||||
if self.settled.is_some() {
|
||||
return;
|
||||
}
|
||||
let Some(rate) = self.measured_kbps(now) else {
|
||||
return;
|
||||
};
|
||||
let drift =
|
||||
(rate as f32 - self.current_kbps as f32).abs() / self.current_kbps.max(1) as f32;
|
||||
if drift > TOLERANCE {
|
||||
// Left the band, so whatever it was doing was not settling.
|
||||
self.inside_since = None;
|
||||
return;
|
||||
}
|
||||
let entered = *self.inside_since.get_or_insert(now);
|
||||
if now.duration_since(entered) >= HOLD {
|
||||
// Credit the moment it arrived, not the moment it had stayed long
|
||||
// enough to prove it: the hold is evidence about the arrival, not
|
||||
// part of the time the encoder took.
|
||||
let ms = entered.duration_since(self.began).as_millis() as u32;
|
||||
self.settled = Some((ms, self.frames));
|
||||
}
|
||||
}
|
||||
|
||||
/// Produced rate over the window, or `None` before there is a window's worth.
|
||||
fn measured_kbps(&self, now: Instant) -> Option<u32> {
|
||||
let oldest = self.samples.first()?.0;
|
||||
let span = now.duration_since(oldest);
|
||||
if span < WINDOW / 2 {
|
||||
return None;
|
||||
}
|
||||
let bits: u64 = self.samples.iter().map(|(_, b)| u64::from(*b) * 8).sum();
|
||||
Some((bits as f64 / span.as_secs_f64() / 1000.0) as u32)
|
||||
}
|
||||
|
||||
/// The next target to apply, when the current rung is done.
|
||||
pub fn due_step(&mut self, now: Instant) -> Option<u32> {
|
||||
if now.duration_since(self.began) < DWELL {
|
||||
return None;
|
||||
}
|
||||
if self.phase == Phase::Stepping {
|
||||
self.close_rung(now);
|
||||
}
|
||||
if self.rung >= LADDER.len() {
|
||||
self.phase = Phase::Done;
|
||||
return None;
|
||||
}
|
||||
let next = LADDER[self.rung];
|
||||
self.rung += 1;
|
||||
self.phase = Phase::Stepping;
|
||||
self.began = now;
|
||||
self.frames = 0;
|
||||
self.keyframes = 0;
|
||||
self.keyframe_bytes = 0;
|
||||
self.inside_since = None;
|
||||
self.settled = None;
|
||||
let from = self.current_kbps;
|
||||
self.current_kbps = next;
|
||||
let _ = from;
|
||||
Some(next)
|
||||
}
|
||||
|
||||
fn close_rung(&mut self, now: Instant) {
|
||||
let steady_bits: u64 = self.steady.iter().map(|(_, b)| u64::from(*b) * 8).sum();
|
||||
let span = self
|
||||
.steady
|
||||
.first()
|
||||
.map(|(t, _)| now.duration_since(*t).as_secs_f64())
|
||||
.unwrap_or(0.0);
|
||||
let steady_kbps = if span > 0.0 {
|
||||
steady_bits as f64 / span / 1000.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let from = self.reports.last().map_or(0, |r| r.to_kbps);
|
||||
self.reports.push(StepReport {
|
||||
from_kbps: from,
|
||||
to_kbps: self.current_kbps,
|
||||
settle_ms: self.settled.map(|(ms, _)| ms),
|
||||
settle_frames: self.settled.map(|(_, f)| f),
|
||||
steady_ratio: (steady_kbps / f64::from(self.current_kbps.max(1))) as f32,
|
||||
keyframes: self.keyframes,
|
||||
keyframe_bytes: self.keyframe_bytes,
|
||||
keyframe_ms: (u64::from(self.keyframe_bytes) * 8 / u64::from(self.current_kbps.max(1)))
|
||||
as u32,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn reports(&self) -> &[StepReport] {
|
||||
&self.reports
|
||||
}
|
||||
|
||||
/// The number the controller design turns on: the slowest settle observed.
|
||||
///
|
||||
/// The slowest rather than the average, for the same reason the bitrate
|
||||
/// controller reads the worst client's report: a loop that keeps up with
|
||||
/// the typical step and not the worst one is a loop that falls behind
|
||||
/// exactly when the path is changing, which is the only time it matters.
|
||||
pub fn worst_settle_ms(&self) -> Option<u32> {
|
||||
self.reports.iter().map(|r| r.settle_ms).max().flatten()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Feed a probe frames at `kbps` for `secs`, sixty a second.
|
||||
fn feed(probe: &mut RateProbe, start: Instant, secs: f32, kbps: u32) -> Instant {
|
||||
let frames = (secs * 60.0) as u32;
|
||||
let bytes = (u64::from(kbps) * 1000 / 8 / 60) as u32;
|
||||
let mut now = start;
|
||||
for i in 1..=frames {
|
||||
// From the start each time: stepping by a rounded frame interval
|
||||
// drifts, and a helper that loses 4% of every second makes a probe
|
||||
// measuring seconds look broken when it is not.
|
||||
now = start + Duration::from_secs_f64(f64::from(i) / 60.0);
|
||||
probe.observe(now, bytes, false);
|
||||
}
|
||||
now
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_encoder_that_changes_at_once_settles_at_once() {
|
||||
let t0 = Instant::now();
|
||||
let mut p = RateProbe::new(t0, 6_000);
|
||||
let mut now = feed(&mut p, t0, 5.1, 6_000);
|
||||
let step = p.due_step(now).expect("first rung");
|
||||
assert_eq!(step, 6_000);
|
||||
now = feed(&mut p, now, 5.1, 6_000);
|
||||
assert_eq!(p.due_step(now), Some(1_500));
|
||||
// Produces the new rate immediately.
|
||||
feed(&mut p, now, 2.0, 1_500);
|
||||
let r = p.reports();
|
||||
assert!(!r.is_empty());
|
||||
assert!(
|
||||
r[0].settle_ms.is_some_and(|ms| ms < 1_200),
|
||||
"an instant encoder reported {:?}",
|
||||
r[0].settle_ms
|
||||
);
|
||||
}
|
||||
|
||||
/// The case that would kill the fast-loop design, and must be visible.
|
||||
#[test]
|
||||
fn an_encoder_that_never_gets_there_reports_no_settle() {
|
||||
let t0 = Instant::now();
|
||||
let mut p = RateProbe::new(t0, 6_000);
|
||||
let mut now = feed(&mut p, t0, 5.1, 6_000);
|
||||
p.due_step(now);
|
||||
now = feed(&mut p, now, 5.1, 6_000);
|
||||
assert_eq!(p.due_step(now), Some(1_500));
|
||||
// Ignores the command completely.
|
||||
now = feed(&mut p, now, 5.1, 6_000);
|
||||
p.due_step(now);
|
||||
let r = p.reports();
|
||||
let step = r
|
||||
.iter()
|
||||
.find(|r| r.to_kbps == 1_500)
|
||||
.expect("the 1500 rung");
|
||||
assert_eq!(
|
||||
step.settle_ms, None,
|
||||
"an encoder that ignored us looked settled"
|
||||
);
|
||||
assert!(
|
||||
step.steady_ratio > 3.0,
|
||||
"steady ratio {} did not show it producing four times the target",
|
||||
step.steady_ratio
|
||||
);
|
||||
}
|
||||
|
||||
/// Overshoot is the other number the controller needs, so it must be real.
|
||||
#[test]
|
||||
fn steady_ratio_reports_the_overshoot() {
|
||||
let t0 = Instant::now();
|
||||
let mut p = RateProbe::new(t0, 6_000);
|
||||
let mut now = feed(&mut p, t0, 5.1, 6_000);
|
||||
p.due_step(now);
|
||||
now = feed(&mut p, now, 5.1, 6_000);
|
||||
p.due_step(now);
|
||||
// Asked for 1500, produces 1875: the 25% overshoot measured on hardware.
|
||||
now = feed(&mut p, now, 5.1, 1_875);
|
||||
p.due_step(now);
|
||||
let step = p.reports().iter().find(|r| r.to_kbps == 1_500).unwrap();
|
||||
assert!(
|
||||
(step.steady_ratio - 1.25).abs() < 0.1,
|
||||
"steady ratio {} should be about 1.25",
|
||||
step.steady_ratio
|
||||
);
|
||||
}
|
||||
|
||||
/// The confound that made a 4 s GOP look like a three-times slower
|
||||
/// encoder, including making one rung appear *faster* when keyframes were
|
||||
/// added -- which no encoder does, and which gave the measurement away.
|
||||
#[test]
|
||||
fn a_keyframe_landing_mid_window_does_not_delay_the_reported_settle() {
|
||||
let t0 = Instant::now();
|
||||
let mut p = RateProbe::new(t0, 6_000);
|
||||
let mut now = feed(&mut p, t0, 5.1, 6_000);
|
||||
p.due_step(now);
|
||||
now = feed(&mut p, now, 5.1, 6_000);
|
||||
assert_eq!(p.due_step(now), Some(1_500));
|
||||
|
||||
// Settles immediately, but an IDR worth a second of bitrate lands in
|
||||
// the middle of the window that is meant to prove it.
|
||||
let bytes = (1_500u64 * 1000 / 8 / 60) as u32;
|
||||
let base = now;
|
||||
for i in 1..=320u32 {
|
||||
now = base + Duration::from_secs_f64(f64::from(i) / 60.0);
|
||||
let key = i == 40;
|
||||
p.observe(now, if key { 1_500 * 1000 / 8 } else { bytes }, key);
|
||||
}
|
||||
p.due_step(now);
|
||||
let step = p.reports().iter().find(|r| r.to_kbps == 1_500).unwrap();
|
||||
assert!(
|
||||
step.settle_ms.is_some_and(|ms| ms < 1_000),
|
||||
"an encoder that settled at once reported {:?} because of one keyframe",
|
||||
step.settle_ms
|
||||
);
|
||||
// And the burst is still reported, because the queue has to absorb it.
|
||||
assert_eq!(step.keyframes, 1);
|
||||
assert!(
|
||||
step.keyframe_ms >= 900,
|
||||
"a keyframe worth a second of bitrate reported {} ms",
|
||||
step.keyframe_ms
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sweep_ends_and_does_not_step_forever() {
|
||||
let t0 = Instant::now();
|
||||
let mut p = RateProbe::new(t0, 6_000);
|
||||
let mut now = t0;
|
||||
for _ in 0..(LADDER.len() + 2) {
|
||||
now = feed(&mut p, now, 5.1, 3_000);
|
||||
p.due_step(now);
|
||||
}
|
||||
assert!(p.finished());
|
||||
assert!(
|
||||
!p.owns_the_bitrate(),
|
||||
"a finished sweep still holds the dial"
|
||||
);
|
||||
assert_eq!(p.reports().len(), LADDER.len());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,7 @@
|
||||
// slots.rs — ownership of the capture ring's destination buffers
|
||||
//
|
||||
// A captured frame travels from the present hook, through the capture worker,
|
||||
// into the encoder thread, and its DMA-BUF must not be written again until the
|
||||
// into the encoder thread, and its image must not be written again until the
|
||||
// encoder has finished reading it. Tracking that by hand across three threads
|
||||
// is how the single-buffer version got it wrong. Instead the slot index is
|
||||
// carried by a guard that returns it to the pool when it drops, wherever that
|
||||
|
||||
@@ -25,21 +25,10 @@ pub const CAPTURE_SLOTS: usize = 4;
|
||||
pub struct CaptureSlot {
|
||||
pub image: vk::Image,
|
||||
pub memory: vk::DeviceMemory,
|
||||
/// Exported once at allocation and duplicated per frame. -1 if the export
|
||||
/// failed, which sends that frame down the CPU readback path instead.
|
||||
pub dmabuf_fd: std::os::raw::c_int,
|
||||
pub stride: u32,
|
||||
/// The DRM format modifier the driver gave this slot's image.
|
||||
///
|
||||
/// Carried per slot rather than assumed, and passed to the importer, which
|
||||
/// creates its side with this exact value. It used to be hard-coded to
|
||||
/// `DRM_FORMAT_MOD_LINEAR` on both sides — true at the time, because the
|
||||
/// producer only ever asked for linear.
|
||||
pub modifier: u64,
|
||||
/// Signalled when this slot's blit has finished reading the swapchain and
|
||||
/// writing the slot. The capture worker waits on it before handing the
|
||||
/// DMA-BUF to the encoder, which reads it from a different VkDevice and so
|
||||
/// cannot be synchronised with a semaphore.
|
||||
/// writing the slot. Where the encoder has a device of its own, the encoder
|
||||
/// thread waits on it before reading the slot back on the CPU; a device of
|
||||
/// its own shares no timeline with the game's.
|
||||
pub fence: vk::Fence,
|
||||
}
|
||||
|
||||
@@ -75,12 +64,14 @@ pub struct CaptureRing {
|
||||
pub timestamp_period: f32,
|
||||
/// The extent `blits` were recorded for.
|
||||
///
|
||||
/// The ring is kept when the swapchain shrinks — `ensure_capture_ring`
|
||||
/// accepts a ring at least as large as the request — so the extent can
|
||||
/// change under a ring that is not rebuilt, and the recordings have to
|
||||
/// follow it even though the images do not.
|
||||
/// Kept separate from `size` because a swapchain can be recreated at the
|
||||
/// same extent — on a format or present-mode change — which invalidates the
|
||||
/// recordings without invalidating the images.
|
||||
pub blit_extent: vk::Extent2D,
|
||||
pub size: (u32, u32, vk::Format),
|
||||
/// Which ring this is, counting from the device's first. For the log, so a
|
||||
/// resize shows up as the ring it caused.
|
||||
pub generation: u64,
|
||||
/// Queue family the command pool was created for. Command buffers may only
|
||||
/// be submitted to a queue of the family their pool belongs to, so a
|
||||
/// present arriving on a different family rebuilds the ring rather than
|
||||
@@ -102,6 +93,15 @@ pub struct CaptureRing {
|
||||
/// cannot reacquire it until the present that waited on this semaphore is
|
||||
/// done.
|
||||
pub present_wait: Vec<vk::Semaphore>,
|
||||
/// Timeline semaphore every blit signals, one value higher each time, or
|
||||
/// null when the encoder does not share this device.
|
||||
///
|
||||
/// On a shared device this is the whole handover: the frame carries the
|
||||
/// value its blit signals, and the encoder's GPU work waits on it. Nothing
|
||||
/// waits on the CPU.
|
||||
pub blit_timeline: vk::Semaphore,
|
||||
/// The value the last blit signalled.
|
||||
pub blit_value: u64,
|
||||
}
|
||||
|
||||
/// Index into [`CaptureRing::blits`] for one (swapchain image, slot) pair.
|
||||
@@ -177,6 +177,17 @@ pub struct DeviceState {
|
||||
pub raw: vk::Device,
|
||||
pub physical_device: vk::PhysicalDevice,
|
||||
pub fp: NextDeviceFn,
|
||||
/// The encoder's view of this device, when it runs on it. `None` when the
|
||||
/// device could not be created with what the encoder needs, in which case
|
||||
/// the encoder uses a device of its own.
|
||||
pub shared: Option<crate::shared::SharedDevice>,
|
||||
/// Whether the encode pipeline really runs on `shared`. Set once the
|
||||
/// pipeline has been built on it; until then, and for good if that fails,
|
||||
/// capture works as it does for a device of the encoder's own.
|
||||
pub shared_active: std::sync::atomic::AtomicBool,
|
||||
/// The loader's callback for stamping dispatchable objects this layer
|
||||
/// creates itself. See [`crate::device::stamp`].
|
||||
pub set_loader_data: Option<crate::PFN_vkSetDeviceLoaderData>,
|
||||
|
||||
// Phase 1: shader / pipeline
|
||||
pub shader_registry: DashMap<u64, u64>,
|
||||
@@ -197,7 +208,7 @@ pub struct DeviceState {
|
||||
pub hudless_memory: std::sync::Mutex<Option<vk::DeviceMemory>>,
|
||||
pub hudless_size: std::sync::Mutex<(u32, u32, vk::Format)>,
|
||||
|
||||
// Phase 4: final-frame capture (DMA-BUF exportable)
|
||||
// Phase 4: final-frame capture
|
||||
pub capture_ring: std::sync::Mutex<Option<CaptureRing>>,
|
||||
/// Which ring slots are free. Held separately from the ring itself so a
|
||||
/// slot can be returned from the encoder thread without taking the lock
|
||||
@@ -219,6 +230,9 @@ pub struct DeviceState {
|
||||
pub swapchain_transfer_src: std::sync::atomic::AtomicBool,
|
||||
|
||||
pub frame_counter: std::sync::atomic::AtomicU64,
|
||||
/// Rings built for this device so far. Names the current one; see
|
||||
/// [`CaptureRing::generation`].
|
||||
pub ring_generation: std::sync::atomic::AtomicU64,
|
||||
|
||||
// Phase 3/4: per-frame HUD detection flags
|
||||
pub hud_detected_frame: std::sync::atomic::AtomicBool,
|
||||
@@ -229,7 +243,6 @@ pub struct DeviceState {
|
||||
// Phase 7: encode + IPC pipeline (lazy-init on first frame)
|
||||
pub encoder: std::sync::Mutex<Option<PipelineHandle>>,
|
||||
|
||||
|
||||
// ── Frame-rate throttle ───────────────────────────────────────────
|
||||
/// Decides which presented frames are worth capturing. Consulted in the
|
||||
/// present hook, before any GPU work is queued, so a dropped frame costs
|
||||
@@ -252,6 +265,16 @@ pub struct DeviceState {
|
||||
pub encoder_starting: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl DeviceState {
|
||||
/// The shared device, when the encoder is actually running on it.
|
||||
pub fn shared_encoder(&self) -> Option<&crate::shared::SharedDevice> {
|
||||
self.shared.as_ref().filter(|_| {
|
||||
self.shared_active
|
||||
.load(std::sync::atomic::Ordering::Acquire)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Per-command-buffer state ──────────────────────────────────────────────────
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
|
||||
@@ -125,6 +125,13 @@ pub unsafe extern "system" fn vkCreateSwapchainKHR(
|
||||
vk::Result::SUCCESS
|
||||
}
|
||||
|
||||
/// Whether `swapchain` is the one this layer tracks, which is the one most
|
||||
/// recently created. A swapchain retired through `oldSwapchain` stays valid
|
||||
/// until destroyed, and calls on it must not touch the tracked state.
|
||||
pub fn is_current(tracked: Option<vk::SwapchainKHR>, swapchain: vk::SwapchainKHR) -> bool {
|
||||
swapchain != vk::SwapchainKHR::null() && tracked == Some(swapchain)
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "system" fn vkDestroySwapchainKHR(
|
||||
device: vk::Device,
|
||||
@@ -133,8 +140,17 @@ pub unsafe extern "system" fn vkDestroySwapchainKHR(
|
||||
) {
|
||||
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
|
||||
if let Some(ds) = DEVICE_STATE.get(&key) {
|
||||
*ds.swapchain.lock().unwrap() = None;
|
||||
*ds.swapchain_images.lock().unwrap() = Vec::new();
|
||||
// Only the swapchain being tracked. A game recreating through
|
||||
// `oldSwapchain` destroys the retired one after its replacement is
|
||||
// created and its images fetched, and clearing then would leave the
|
||||
// live swapchain with no images: every present after would skip
|
||||
// capture, silently, until the next recreation.
|
||||
let mut current = ds.swapchain.lock().unwrap();
|
||||
if is_current(*current, swapchain) {
|
||||
*current = None;
|
||||
*ds.swapchain_images.lock().unwrap() = Vec::new();
|
||||
}
|
||||
drop(current);
|
||||
if let Some(destroy_fn) = ds.fp.destroy_swapchain_khr {
|
||||
unsafe { destroy_fn(device, swapchain, p_allocator) };
|
||||
}
|
||||
@@ -171,7 +187,8 @@ pub unsafe extern "system" fn vkGetSwapchainImagesKHR(
|
||||
return result;
|
||||
}
|
||||
|
||||
if !p_swapchain_images.is_null() {
|
||||
// Images of a retired swapchain are not the ones capture reads.
|
||||
if !p_swapchain_images.is_null() && is_current(*ds.swapchain.lock().unwrap(), swapchain) {
|
||||
let count = unsafe { *p_swapchain_image_count as usize };
|
||||
let images = unsafe { std::slice::from_raw_parts(p_swapchain_images, count) };
|
||||
*ds.swapchain_images.lock().unwrap() = images.to_vec();
|
||||
@@ -231,9 +248,11 @@ pub unsafe extern "system" fn vkAcquireNextImageKHR(
|
||||
return vk::Result::ERROR_EXTENSION_NOT_PRESENT;
|
||||
};
|
||||
|
||||
crate::present::note_present(&ds, crate::encode::PresentStep::Acquiring);
|
||||
let started = std::time::Instant::now();
|
||||
let result = unsafe { acquire(device, swapchain, timeout, semaphore, fence, p_image_index) };
|
||||
record_acquire(&ds, started.elapsed());
|
||||
crate::present::note_present(&ds, crate::encode::PresentStep::InGame);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -250,8 +269,37 @@ pub unsafe extern "system" fn vkAcquireNextImage2KHR(
|
||||
return vk::Result::ERROR_EXTENSION_NOT_PRESENT;
|
||||
};
|
||||
|
||||
crate::present::note_present(&ds, crate::encode::PresentStep::Acquiring);
|
||||
let started = std::time::Instant::now();
|
||||
let result = unsafe { acquire(device, p_acquire_info, p_image_index) };
|
||||
record_acquire(&ds, started.elapsed());
|
||||
crate::present::note_present(&ds, crate::encode::PresentStep::InGame);
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sc(raw: u64) -> vk::SwapchainKHR {
|
||||
vk::SwapchainKHR::from_raw(raw)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tracked_swapchain_is_current() {
|
||||
assert!(is_current(Some(sc(2)), sc(2)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retired_swapchain_is_not() {
|
||||
// Created 2 with oldSwapchain = 1; destroying 1 afterwards must leave
|
||||
// 2's state alone.
|
||||
assert!(!is_current(Some(sc(2)), sc(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_is_current_once_the_tracked_one_is_gone() {
|
||||
assert!(!is_current(None, sc(1)));
|
||||
assert!(!is_current(None, vk::SwapchainKHR::null()));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user