diff --git a/Cargo.lock b/Cargo.lock index 710f8f78..62632dd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,14 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ash" +version = "0.38.0+1.4.329" +source = "git+https://github.com/ash-rs/ash?rev=55dd56906bbb5760e9e9e6c56f45be67f67e0649#55dd56906bbb5760e9e9e6c56f45be67f67e0649" +dependencies = [ + "libloading", +] + [[package]] name = "atomic_float" version = "1.1.0" @@ -111,7 +119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "annotate-snippets", - "bitflags", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools", @@ -123,6 +131,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -164,7 +178,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ - "bitflags", + "bitflags 2.13.1", "nix", "polling", "rustix 1.1.4", @@ -344,6 +358,51 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + [[package]] name = "digest" version = "0.10.7" @@ -366,7 +425,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytemuck", "drm-ffi", "drm-fourcc", @@ -415,6 +474,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -443,13 +525,28 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + [[package]] name = "gbm" version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" dependencies = [ - "bitflags", + "bitflags 2.13.1", "drm", "drm-fourcc", "gbm-sys", @@ -525,6 +622,12 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.17.1" @@ -556,7 +659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -565,7 +668,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbdc09524a91f9cacd26f16734ff63d7dc650daffadd2b6f84d17a285bd875a9" dependencies = [ - "bitflags", + "bitflags 2.13.1", "input-sys", "libc", "log", @@ -604,6 +707,42 @@ dependencies = [ "either", ] +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "khronos_api" version = "3.1.0" @@ -638,7 +777,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "882f7427e7989dcc9d388b7f05c4630390a1d7696f9ffa469cd4a7a48f0b4c40" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cc", "cookie-factory", "libc", @@ -687,6 +826,15 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -723,6 +871,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "nescapture" +version = "0.2.0" +dependencies = [ + "anyhow", + "ash", + "bytemuck", + "dashmap", + "env_logger", + "libc", + "log", + "nesprotocol", + "once_cell", + "pixelforge", + "pollster", + "serde", + "sha2", + "toml 0.8.23", +] + [[package]] name = "nescope" version = "0.1.0" @@ -767,7 +935,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -831,6 +999,19 @@ dependencies = [ "cmake", ] +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" @@ -849,7 +1030,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bde71084c4e25959d68f1ea54daa75e5ecdb338e5caf0b5510143b79baa32d5c" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "libspa", "libspa-sys", @@ -868,6 +1049,17 @@ dependencies = [ "system-deps", ] +[[package]] +name = "pixelforge" +version = "0.7.1" +source = "git+https://github.com/hgaiser/pixelforge.git?rev=85c8654d383e194b984e27b319de96e4335e2b4f#85c8654d383e194b984e27b319de96e4335e2b4f" +dependencies = [ + "ash", + "futures-channel", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "pixman" version = "0.2.1" @@ -906,6 +1098,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1002,6 +1215,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "regex" version = "1.13.1" @@ -1043,7 +1265,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -1056,7 +1278,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -1069,6 +1291,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -1089,6 +1321,15 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -1150,7 +1391,7 @@ checksum = "740cea6927892bc182d5bf70c8f79806c8bc9f68f2fb96e55a30be171b63af98" dependencies = [ "appendlist", "atomic_float", - "bitflags", + "bitflags 2.13.1", "calloop", "cc", "cgmath", @@ -1223,7 +1464,7 @@ dependencies = [ "cfg-expr", "heck", "pkg-config", - "toml", + "toml 1.1.4+spec-1.1.0", "version-compare", ] @@ -1295,6 +1536,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit", +] + [[package]] name = "toml" version = "1.1.4+spec-1.1.0" @@ -1303,11 +1556,20 @@ checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] @@ -1319,15 +1581,35 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -1478,7 +1760,7 @@ version = "0.31.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" dependencies = [ - "bitflags", + "bitflags 2.13.1", "rustix 1.1.4", "wayland-backend", "wayland-scanner", @@ -1490,7 +1772,7 @@ version = "0.32.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" dependencies = [ - "bitflags", + "bitflags 2.13.1", "wayland-backend", "wayland-client", "wayland-scanner", @@ -1503,7 +1785,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" dependencies = [ - "bitflags", + "bitflags 2.13.1", "wayland-backend", "wayland-protocols", "wayland-scanner", @@ -1516,7 +1798,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" dependencies = [ - "bitflags", + "bitflags 2.13.1", "wayland-backend", "wayland-protocols", "wayland-scanner", @@ -1540,7 +1822,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75" dependencies = [ - "bitflags", + "bitflags 2.13.1", "downcast-rs", "rustix 1.1.4", "wayland-backend", @@ -1710,6 +1992,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" diff --git a/Cargo.toml b/Cargo.toml index 3388c06c..5d78d4a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ [workspace] resolver = "3" members = [ + "apps/nescapture", "apps/nescope", "apps/neswire", "crates/nesprotocol", diff --git a/apps/nescapture/.gitignore b/apps/nescapture/.gitignore new file mode 100644 index 00000000..3211ec5f --- /dev/null +++ b/apps/nescapture/.gitignore @@ -0,0 +1,4 @@ +.vscode/ +.idea/ +.zed/ +/target/ diff --git a/apps/nescapture/Cargo.toml b/apps/nescapture/Cargo.toml new file mode 100644 index 00000000..8f09bae4 --- /dev/null +++ b/apps/nescapture/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "nescapture" +version = "0.2.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Vulkan implicit layer: direct frame capture, Vulkan Video encode, and IPC out to the session transport" + +[lib] +name = "nescapture_layer" +crate-type = ["cdylib"] + +[dependencies] +# Vulkan bindings +ash = { git = "https://github.com/ash-rs/ash", rev = "55dd56906bbb5760e9e9e6c56f45be67f67e0649" } +# Shader fingerprinting +sha2 = "0.10" +bytemuck = "1" + +# Concurrent state maps +dashmap = "6" +once_cell = "1" +pollster = "0.4.0" + +# Logging +log = "0.4" +env_logger = "0.11" +anyhow = "1" + +# Per-game shader-hash config +toml = "0.8" +serde = { version = "1", features = ["derive"] } + +# Vulkan Video hardware encoding. +pixelforge = { git = "https://github.com/hgaiser/pixelforge.git", rev = "85c8654d383e194b984e27b319de96e4335e2b4f", features = ["dmabuf"] } + +# libc for DMA-BUF OS primitives +libc = "0.2" + +# Shared IPC protocol +nesprotocol = { path = "../../crates/nesprotocol" } + +[profile.release] +opt-level = 3 +lto = "thin" diff --git a/apps/nescapture/README.md b/apps/nescapture/README.md new file mode 100644 index 00000000..ed3cf96f --- /dev/null +++ b/apps/nescapture/README.md @@ -0,0 +1,209 @@ +# nescapture + +A Vulkan implicit layer that captures frames from a running game, encodes them +with **Vulkan Video** hardware acceleration (H.264 / H.265 / AV1), packetizes with +Reed-Solomon FEC, and streams over RTP/UDP to a Moonlight-compatible client — +all with **zero CPU copies** — fully GPU from game rendering to RTP output. + +--- + +## Architecture + +``` +Game process + │ Vulkan calls + ▼ +┌───────────────────────────────────────────────┐ +│ nescapture Vulkan implicit layer │ +│ │ +│ vkCreateShaderModule → SHA-256 hash │ +│ vkCreateGraphicsPipelines → track hashes │ +│ vkCmdBindPipeline → detect HUD shaders │ +│ vkCmdEndRenderPass/Rendering → inject copy │ +│ vkQueuePresentKHR → capture + encode │ +└───────────────────────────────────────────────┘ + │ GPU blit (same device) + ▼ +final_image (DMA-BUF exportable) + │ get_dmabuf_fd(final_memory) + ▼ +DmaBufImporter (pixelforge VkDevice) + │ import_or_reuse() → vk::Image + ▼ +ColorConverter (GPU compute shader) + │ BGRA/RGB10/FP16 → NV12/P010/YUV444 + ▼ +Encoder (Vulkan Video, hardware H.264/H.265) + │ encode() + ▼ +EncodedPacket (Annex-B) + │ + ▼ +Packetizer (Moonlight wire format, Reed-Solomon FEC) + │ UDP datagrams + ▼ +Moonlight client (or any RTP/UDP receiver) + +CPU fallback: only when DMA-BUF export unavailable (rare driver config) +``` + +--- + +## Quick start + +```bash +# 1. Build +cargo build --release + +# 2. Install the layer manifest +sudo cp manifest/VK_LAYER_nescapture.json /usr/share/vulkan/implicit_layer.d/ +# Edit the manifest's `library_path` to point at target/release/libnescapture.so + +# 3. Configure and launch a game +export NESCAPTURE_ENABLE=1 +export NESCAPTURE_RTP_HOST=192.168.1.50 # Moonlight / receiver IP +export NESCAPTURE_RTP_PORT=47998 # default +export NESCAPTURE_CODEC=h265 # h264 | h265 | av1 (auto-probes if unset) +export NESCAPTURE_BITRATE=10000 # kbps (CBR; ignored if NESCAPTURE_QP is set) +export NESCAPTURE_FPS=60 +export RUST_LOG=info # or NESCAPTURE_LOG=debug + +wine MyGame.exe # or native Vulkan game +``` + +--- + +## Environment variables + +| Variable | Default | Description | +| ------------------------- | -------------- | ------------------------------------------------------ | +| `NESCAPTURE_ENABLE` | _(unset)_ | Set to `1` to activate the layer | +| `NESCAPTURE_RTP_HOST` | _(required)_ | Destination IP / hostname for RTP stream | +| `NESCAPTURE_RTP_PORT` | `47998` | Destination UDP port | +| `NESCAPTURE_CODEC` | auto | `h264`, `h265` or `av1` — falls back to h264 if unsupported | +| `NESCAPTURE_BITRATE` | `10000` | CBR target bitrate in kbps | +| `NESCAPTURE_QP` | _(unset)_ | If set, use CQP with this quality level instead of CBR | +| `NESCAPTURE_FPS` | `60` | Target frame rate | +| `NESCAPTURE_IDR_INTERVAL` | `120` | Force an IDR keyframe every N frames | +| `NESCAPTURE_FEC_PCT` | `20` | Reed-Solomon FEC percentage | +| `NESCAPTURE_MIN_FEC` | `2` | Minimum FEC packets per block | +| `NESCAPTURE_PACKET_SIZE` | `1392` | Max UDP payload size (bytes) | +| `NESCAPTURE_CTRL_PORT` | `47999` | UDP port for the control stream | +| `NESCAPTURE_CAPTURE_HUDLESS` | _(unset)_ | Set to `1` to also capture HUDless frames | +| `NESCAPTURE_CONFIG` | _(unset)_ | Path to per-game shader-hash TOML config | +| `NESCAPTURE_GAME_NAME` | (exe basename) | Override game identification | +| `NESCAPTURE_DISCOVER` | _(unset)_ | Set to `1` to enable discovery mode (logs all draws) | +| `NESCAPTURE_LOG` | `info` | Log level (`error`, `warn`, `info`, `debug`, `trace`) | +| `NESCAPTURE_RTP_FORMAT` | `moonlight` | `standard` for RFC 6184/7798 RTP (GStreamer/FFmpeg compatible), `moonlight` for Moonlight wire format with FEC | + +--- + +## Control stream + +Send single-byte UDP datagrams to `NESCAPTURE_CTRL_PORT` (default 47999): + +| Byte | Command | +| ------ | -------------------- | +| `0x01` | Start streaming | +| `0x02` | Stop streaming | +| `0x03` | Request IDR keyframe | + +Example with netcat: + +```bash +# Request IDR +printf '\x03' | nc -u -q1 localhost 47999 +``` + +The control handle wiring in `present.rs` is currently a commented-out stub. +See `control.rs` for the full wiring instructions. + +--- + +## Per-game shader-hash config + +HUD shader detection requires a per-game config. Run the game once with +`NESCAPTURE_DISCOVER=1` to log all shaders, then identify HUD pipelines by the +`[SUSPECT]` marker (blend=true, depth=false, ≤6 vertices). + +```toml +# ~/.config/nescapture/games.toml +[game."GameName.exe"] +hud_fragment_shaders = ["0xaabbccddeeff0011"] +hud_vertex_shaders = ["0x1a2b3c4d5e6f7890"] +skip_fragment_shaders = ["0x1122334455667788"] +``` + +```bash +export NESCAPTURE_CONFIG=~/.config/nescapture/games.toml +export NESCAPTURE_GAME_NAME=GameName.exe +``` + +--- + +## Dependencies + +| Crate | Purpose | +| ---------------------- | -------------------------------------------- | +| `pixelforge` | Vulkan Video hardware encode (H.264 / H.265) | +| `ash` | Vulkan bindings | +| `reed-solomon-erasure` | FEC for RTP packetizer | +| `sha2` + `bytemuck` | SPIR-V shader fingerprinting | +| `dashmap` | Lock-free concurrent state maps | +| `serde` + `toml` | Per-game shader config | + +--- + +## Zero-copy GPU pipeline + +The full GPU path is implemented — no CPU color conversion: + +1. `final_image` allocated with `DMA_BUF_EXT` external memory (`capture.rs`). +2. `get_dmabuf_fd()` exports the DMA-BUF fd (`capture.rs`). +3. `DmaBufImporter::import_or_reuse()` imports the fd as a `vk::Image` (`dmabuf_import.rs`). +4. `ColorConverter::convert()` runs a GPU compute shader: + BGRA/RGBA/RGB10/FP16 → NV12/P010/YUV444 (`encode.rs`). +5. `Encoder::encode()` encodes from the converter's output directly. +6. RTP packetizer + UDP send. + +CPU fallback exists only for rare driver configurations that don't support +DMA-BUF external memory export. + +--- + +## File structure + +``` +nescapture/ +├── Cargo.toml +├── README.md +├── ARCHITECTURE.md +├── manifest/ +│ └── VK_LAYER_nescapture.json +└── src/ + ├── lib.rs — entry points, dispatch routing + ├── dispatch.rs — Vulkan function-pointer types + tables + ├── state.rs — global DashMaps, DeviceState, CbState + ├── instance.rs — vkCreateInstance / vkDestroyInstance + ├── device.rs — vkCreateDevice / vkDestroyDevice + ├── shader.rs — SPIR-V hashing + ├── pipeline.rs — graphics pipeline tracking + ├── framebuffer.rs — image view / framebuffer tracking + ├── commands.rs — vkCmdBind*, vkCmdDraw*, vkCmdBeginRenderPass + ├── swapchain.rs — vkCreateSwapchainKHR, image enumeration + ├── capture.rs — GPU blit to capture image, DMA-BUF export + ├── present.rs — vkQueuePresentKHR, encode dispatch + ├── encode.rs — pixelforge pipeline, codec probing, RTP send + ├── dmabuf_import.rs — DmaBufImporter (cross-device zero-copy import) + ├── packetizer.rs — Moonlight RTP packetizer (Reed-Solomon FEC) + ├── shard_batch.rs — zero-alloc shard buffer + ├── control.rs — UDP control stream (IDR / start / stop) + ├── config.rs — per-game TOML shader-hash config + └── discovery.rs — draw-call logging for shader discovery +``` + +--- + +## License + +TBD diff --git a/apps/nescapture/manifest/VK_LAYER_nescapture.json b/apps/nescapture/manifest/VK_LAYER_nescapture.json new file mode 100644 index 00000000..b83b7129 --- /dev/null +++ b/apps/nescapture/manifest/VK_LAYER_nescapture.json @@ -0,0 +1,22 @@ +{ + "file_format_version": "1.2.0", + "layer": { + "name": "VK_LAYER_nescapture", + "type": "GLOBAL", + "library_path": "/usr/lib/libnescapture_layer.so", + "api_version": "1.3.290", + "implementation_version": "1", + "description": "direct game frame capture for Nestri", + "functions": { + "vkGetInstanceProcAddr": "vkGetInstanceProcAddr", + "vkGetDeviceProcAddr": "vkGetDeviceProcAddr", + "vkNegotiateLoaderLayerInterfaceVersion": "vkNegotiateLoaderLayerInterfaceVersion" + }, + "enable_environment": { + "NESCAPTURE_ENABLE": "1" + }, + "disable_environment": { + "DISABLE_NESCAPTURE": "1" + } + } +} diff --git a/apps/nescapture/src/capture.rs b/apps/nescapture/src/capture.rs new file mode 100644 index 00000000..53a96d1b --- /dev/null +++ b/apps/nescapture/src/capture.rs @@ -0,0 +1,737 @@ +// ───────────────────────────────────────────────────────────────────────────── +// capture.rs — Frame capture helpers +// +// final_image is allocated with VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT +// so that after the GPU blit we can export an fd and import it into pixelforge's +// separate VkDevice for zero-copy hardware encoding via DmaBufImporter. +// +// After ensure_final_image allocates (or re-allocates) the image, we query +// its SubresourceLayout and cache the row stride in DeviceState::final_stride. +// The stride is needed by the encoder to correctly import the LINEAR image. +// ───────────────────────────────────────────────────────────────────────────── + +use crate::state::{CB_STATE, CaptureResources, DEVICE_STATE}; +use ash::vk::{self, Handle}; +use std::os::raw::c_int; +use std::sync::atomic::Ordering; + +fn make_subresource_range() -> vk::ImageSubresourceRange { + vk::ImageSubresourceRange { + aspect_mask: vk::ImageAspectFlags::COLOR, + base_mip_level: 0, + level_count: 1, + base_array_layer: 0, + layer_count: 1, + } +} +fn make_subresource_layers() -> vk::ImageSubresourceLayers { + vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + } +} + +macro_rules! image_barrier { + ($src:expr, $dst:expr, $old:expr, $new:expr, $img:expr) => { + vk::ImageMemoryBarrier { + s_type: vk::StructureType::IMAGE_MEMORY_BARRIER, + p_next: std::ptr::null(), + src_access_mask: $src, + dst_access_mask: $dst, + old_layout: $old, + new_layout: $new, + src_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED, + image: $img, + subresource_range: make_subresource_range(), + _marker: std::marker::PhantomData, + } + }; +} + +// ── Memory helper ───────────────────────────────────────────────────────────── + +unsafe fn find_host_coherent_mt(ds: &crate::state::DeviceState, bits: u32) -> u32 { + let mut mp = vk::PhysicalDeviceMemoryProperties::default(); + let k = unsafe { crate::dispatch_key(ds.physical_device.as_raw() as *const std::ffi::c_void) }; + if let Some(i) = crate::state::INSTANCE_STATE.get(&k) { + unsafe { (i.get_physical_device_memory_properties)(ds.physical_device, &mut mp) }; + } + (0..mp.memory_type_count) + .find(|&i| { + (bits & (1 << i)) != 0 + && mp.memory_types[i as usize].property_flags.contains( + vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT, + ) + }) + .unwrap_or(0) +} + +// ── Image allocators ────────────────────────────────────────────────────────── + +/// Plain HOST_VISIBLE image (nescapture capture — no cross-device sharing needed). +unsafe fn allocate_host_image( + ds: &crate::state::DeviceState, + w: u32, + h: u32, + fmt: vk::Format, + label: &str, +) -> Option<(vk::Image, vk::DeviceMemory)> { + let ci = vk::ImageCreateInfo { + s_type: vk::StructureType::IMAGE_CREATE_INFO, + p_next: std::ptr::null(), + flags: vk::ImageCreateFlags::empty(), + image_type: vk::ImageType::TYPE_2D, + format: fmt, + extent: vk::Extent3D { + width: w, + height: h, + depth: 1, + }, + mip_levels: 1, + array_layers: 1, + samples: vk::SampleCountFlags::TYPE_1, + tiling: vk::ImageTiling::LINEAR, + usage: vk::ImageUsageFlags::TRANSFER_DST, + sharing_mode: vk::SharingMode::EXCLUSIVE, + queue_family_index_count: 0, + p_queue_family_indices: std::ptr::null(), + initial_layout: vk::ImageLayout::UNDEFINED, + _marker: std::marker::PhantomData, + }; + unsafe { alloc_image(ds, &ci, None, label) } +} + +/// DMA-BUF exportable image (final capture — imported into pixelforge for encoding). +/// +/// Falls back to a plain host image if the driver rejects external memory. +/// In that case `get_dmabuf_fd` will return `None` and the encoder will use +/// the CPU pixel-readback fallback. +unsafe fn allocate_dmabuf_image( + ds: &crate::state::DeviceState, + w: u32, + h: u32, + fmt: vk::Format, + label: &str, +) -> Option<(vk::Image, vk::DeviceMemory)> { + let ext_img = vk::ExternalMemoryImageCreateInfo { + s_type: vk::StructureType::EXTERNAL_MEMORY_IMAGE_CREATE_INFO, + p_next: std::ptr::null_mut(), + handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + _marker: std::marker::PhantomData, + }; + let ci = vk::ImageCreateInfo { + s_type: vk::StructureType::IMAGE_CREATE_INFO, + p_next: &ext_img as *const _ as *const _, + flags: vk::ImageCreateFlags::empty(), + image_type: vk::ImageType::TYPE_2D, + format: fmt, + extent: vk::Extent3D { + width: w, + height: h, + depth: 1, + }, + mip_levels: 1, + array_layers: 1, + samples: vk::SampleCountFlags::TYPE_1, + tiling: vk::ImageTiling::LINEAR, + usage: vk::ImageUsageFlags::TRANSFER_DST, + sharing_mode: vk::SharingMode::EXCLUSIVE, + queue_family_index_count: 0, + p_queue_family_indices: std::ptr::null(), + initial_layout: vk::ImageLayout::UNDEFINED, + _marker: std::marker::PhantomData, + }; + let export_ai = vk::ExportMemoryAllocateInfo { + s_type: vk::StructureType::EXPORT_MEMORY_ALLOCATE_INFO, + p_next: std::ptr::null_mut(), + handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + _marker: std::marker::PhantomData, + }; + if let Some(r) = unsafe { alloc_image(ds, &ci, Some(&export_ai), label) } { + return Some(r); + } + log::warn!( + "DMA-BUF alloc failed for '{}' — using plain host image. \ + Zero-copy GPU path will be unavailable; CPU readback fallback active.", + label + ); + unsafe { allocate_host_image(ds, w, h, fmt, label) } +} + +unsafe fn alloc_image( + ds: &crate::state::DeviceState, + ci: &vk::ImageCreateInfo, + export: Option<&vk::ExportMemoryAllocateInfo>, + label: &str, +) -> Option<(vk::Image, vk::DeviceMemory)> { + let mut image = vk::Image::null(); + if unsafe { (ds.fp.create_image)(ds.raw, ci, std::ptr::null(), &mut image) } + != vk::Result::SUCCESS + { + return None; + } + let mut mr = vk::MemoryRequirements { + size: 0, + alignment: 0, + memory_type_bits: 0, + }; + unsafe { (ds.fp.get_image_memory_requirements)(ds.raw, image, &mut mr) }; + let mt = unsafe { find_host_coherent_mt(ds, mr.memory_type_bits) }; + let p_next: *const _ = match export { + Some(e) => e as *const _ as *const _, + None => std::ptr::null(), + }; + let ai = vk::MemoryAllocateInfo { + s_type: vk::StructureType::MEMORY_ALLOCATE_INFO, + p_next, + allocation_size: mr.size, + memory_type_index: mt, + _marker: std::marker::PhantomData, + }; + let mut mem = vk::DeviceMemory::null(); + if unsafe { (ds.fp.allocate_memory)(ds.raw, &ai, std::ptr::null(), &mut mem) } + != vk::Result::SUCCESS + { + unsafe { (ds.fp.destroy_image)(ds.raw, image, std::ptr::null()) }; + return None; + } + if unsafe { (ds.fp.bind_image_memory)(ds.raw, image, mem, 0) } != vk::Result::SUCCESS { + unsafe { (ds.fp.free_memory)(ds.raw, mem, std::ptr::null()) }; + unsafe { (ds.fp.destroy_image)(ds.raw, image, std::ptr::null()) }; + return None; + } + log::info!( + "alloc {} {}x{} fmt={} ({} bytes)", + label, + ci.extent.width, + ci.extent.height, + ci.format.as_raw(), + mr.size + ); + Some((image, mem)) +} + +// ── Stride query ────────────────────────────────────────────────────────────── + +/// Query and cache the row stride of final_image. +/// Returns stride in bytes; 0 on failure. +pub unsafe fn query_and_cache_final_stride( + ds: &crate::state::DeviceState, + image: vk::Image, +) -> u32 { + let subresource = vk::ImageSubresource { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + array_layer: 0, + }; + let mut layout = vk::SubresourceLayout { + offset: 0, + size: 0, + row_pitch: 0, + array_pitch: 0, + depth_pitch: 0, + }; + unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &subresource, &mut layout) }; + let stride = layout.row_pitch as u32; + ds.final_stride.store(stride, Ordering::Relaxed); + stride +} + +// ── DMA-BUF fd export ───────────────────────────────────────────────────────── + +/// Export `memory` as a DMA-BUF fd via vkGetMemoryFdKHR. +/// Callers own the fd and must close it when done. +/// Returns `None` if VK_KHR_external_memory_fd is unavailable. +pub unsafe fn get_dmabuf_fd( + ds: &crate::state::DeviceState, + memory: vk::DeviceMemory, +) -> Option { + let f = match ds.fp.get_memory_fd_khr { + Some(f) => f, + None => { + log::warn!("get_dmabuf_fd: vkGetMemoryFdKHR not available"); + return None; + } + }; + let fi = vk::MemoryGetFdInfoKHR { + s_type: vk::StructureType::MEMORY_GET_FD_INFO_KHR, + p_next: std::ptr::null(), + memory, + handle_type: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + _marker: std::marker::PhantomData, + }; + let mut fd: c_int = -1; + let result = unsafe { f(ds.raw, &fi, &mut fd) }; + if result == vk::Result::SUCCESS && fd >= 0 { + Some(fd) + } else { + log::warn!("get_dmabuf_fd failed: result={:?} fd={}", result, fd); + None + } +} + +// ── ensure helpers ──────────────────────────────────────────────────────────── + +pub unsafe fn ensure_hudless_image(ds: &crate::state::DeviceState, w: u32, h: u32, f: vk::Format) { + let mut ig = ds.hudless_image.lock().unwrap(); + let mut mg = ds.hudless_memory.lock().unwrap(); + let mut sg = ds.hudless_size.lock().unwrap(); + if let (Some(i), Some(m)) = (*ig, *mg) { + let (ew, eh, ef) = *sg; + if ew >= w && eh >= h && ef == f { + return; + } + unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) }; + unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) }; + *ig = None; + *mg = None; + } + if let Some((i, m)) = unsafe { allocate_host_image(ds, w, h, f, "nescapture") } { + *ig = Some(i); + *mg = Some(m); + *sg = (w, h, f); + } +} + +pub unsafe fn ensure_final_image(ds: &crate::state::DeviceState, w: u32, h: u32, f: vk::Format) { + let mut ig = ds.final_image.lock().unwrap(); + let mut mg = ds.final_memory.lock().unwrap(); + let mut sg = ds.final_size.lock().unwrap(); + if let (Some(i), Some(m)) = (*ig, *mg) { + let (ew, eh, ef) = *sg; + if ew >= w && eh >= h && ef == f { + return; + } + unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) }; + unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) }; + *ig = None; + *mg = None; + ds.final_stride.store(0, Ordering::Relaxed); + } + if let Some((i, m)) = unsafe { allocate_dmabuf_image(ds, w, h, f, "final") } { + // Query stride immediately after allocation so it's available on first frame. + unsafe { query_and_cache_final_stride(ds, i) }; + *ig = Some(i); + *mg = Some(m); + *sg = (w, h, f); + } +} + +// ── HUDless command injection ───────────────────────────────────────────────── + +pub unsafe fn inject_hudless_copy(cb: vk::CommandBuffer, dk: usize) { + let ds = match DEVICE_STATE.get(&dk) { + Some(s) => s.clone(), + None => return, + }; + let cbk = cb.as_raw(); + let cs = match CB_STATE.get(&cbk) { + Some(e) => e.value().clone(), + None => return, + }; + let ci = match cs.current_color_image { + Some(i) => i, + None => return, + }; + let fmt = match cs.current_image_format { + Some(f) => f, + None => return, + }; + let ext = match cs.current_image_extent { + Some(e) => e, + None => return, + }; + let sc = *ds.swapchain_extent.lock().unwrap(); + if sc.width > 0 && sc.height > 0 && (ext.width != sc.width || ext.height != sc.height) { + return; + } + unsafe { ensure_hudless_image(&ds, ext.width, ext.height, fmt) }; + let (hi, _) = { + let a = ds.hudless_image.lock().unwrap(); + let b = ds.hudless_memory.lock().unwrap(); + match (*a, *b) { + (Some(i), Some(m)) => (i, m), + _ => return, + } + }; + // src → TRANSFER_SRC + let b1 = image_barrier!( + vk::AccessFlags::COLOR_ATTACHMENT_WRITE, + vk::AccessFlags::TRANSFER_READ, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + ci + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b1, + ); + } + // dst → TRANSFER_DST + let b2 = image_barrier!( + vk::AccessFlags::empty(), + vk::AccessFlags::TRANSFER_WRITE, + vk::ImageLayout::UNDEFINED, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + hi + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b2, + ); + } + let cr = vk::ImageCopy { + src_subresource: make_subresource_layers(), + src_offset: vk::Offset3D { x: 0, y: 0, z: 0 }, + dst_subresource: make_subresource_layers(), + dst_offset: vk::Offset3D { x: 0, y: 0, z: 0 }, + extent: vk::Extent3D { + width: ext.width, + height: ext.height, + depth: 1, + }, + }; + unsafe { + (ds.fp.cmd_copy_image)( + cb, + ci, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + hi, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + 1, + &cr, + ); + } + // restore src + let b3 = image_barrier!( + vk::AccessFlags::TRANSFER_READ, + vk::AccessFlags::COLOR_ATTACHMENT_WRITE, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL, + ci + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::TRANSFER, + vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b3, + ); + } + if let Some(mut s) = CB_STATE.get_mut(&cb.as_raw()) { + s.pending_capture = false; + s.capture_injected = true; + } +} + +// ── Final frame GPU blit (swapchain → final_image) ──────────────────────────── + +pub unsafe fn capture_final_frame( + ds: &crate::state::DeviceState, + queue: vk::Queue, + si: vk::Image, + fmt: vk::Format, + ext: vk::Extent2D, + _frame: u64, +) { + if ext.width == 0 || ext.height == 0 { + return; + } + unsafe { ensure_final_image(ds, ext.width, ext.height, fmt) }; + let fi = match *ds.final_image.lock().unwrap() { + Some(i) => i, + None => return, + }; + + // ── Lazy-init reusable capture resources ────────────────────── + let mut res_guard = ds.capture_resources.lock().unwrap(); + let res = match res_guard.as_mut() { + Some(r) => r, + None => match unsafe { create_capture_resources(ds) } { + Some(r) => { + *res_guard = Some(r); + res_guard.as_mut().unwrap() + } + None => return, + }, + }; + + let idx = res.current; + let cb = res.command_buffers[idx]; + let fence = res.fences[idx]; + + // Wait for THIS slot's previous use to finish (not the other slot). + // Use a short timeout — if the GPU is busy with game rendering, skip + // this capture instead of stalling the game's render loop. + unsafe { + let result = (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 1_000_000); // 1ms timeout + if result != vk::Result::SUCCESS { + // GPU not ready — skip this capture, try next slot + res.current = (idx + 1) % 4; + return; + } + let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence); + } + + // Reset and re-record + unsafe { + let _ = (ds.fp.reset_command_buffer)(cb, vk::CommandBufferResetFlags::empty()); + } + + let bi = vk::CommandBufferBeginInfo { + s_type: vk::StructureType::COMMAND_BUFFER_BEGIN_INFO, + p_next: std::ptr::null(), + flags: vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT, + p_inheritance_info: std::ptr::null(), + _marker: std::marker::PhantomData, + }; + if unsafe { (ds.fp.begin_command_buffer)(cb, &bi) } != vk::Result::SUCCESS { + return; + } + + let b1 = image_barrier!( + vk::AccessFlags::MEMORY_READ, + vk::AccessFlags::TRANSFER_READ, + vk::ImageLayout::PRESENT_SRC_KHR, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + si + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::BOTTOM_OF_PIPE, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b1, + ); + } + let b2 = image_barrier!( + vk::AccessFlags::empty(), + vk::AccessFlags::TRANSFER_WRITE, + vk::ImageLayout::UNDEFINED, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + fi + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::TRANSFER, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b2, + ); + } + let cr = vk::ImageCopy { + src_subresource: make_subresource_layers(), + src_offset: vk::Offset3D { x: 0, y: 0, z: 0 }, + dst_subresource: make_subresource_layers(), + dst_offset: vk::Offset3D { x: 0, y: 0, z: 0 }, + extent: vk::Extent3D { + width: ext.width, + height: ext.height, + depth: 1, + }, + }; + unsafe { + (ds.fp.cmd_copy_image)( + cb, + si, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + fi, + vk::ImageLayout::TRANSFER_DST_OPTIMAL, + 1, + &cr, + ); + } + let b3 = image_barrier!( + vk::AccessFlags::TRANSFER_READ, + vk::AccessFlags::MEMORY_READ, + vk::ImageLayout::TRANSFER_SRC_OPTIMAL, + vk::ImageLayout::PRESENT_SRC_KHR, + si + ); + unsafe { + (ds.fp.cmd_pipeline_barrier)( + cb, + vk::PipelineStageFlags::TRANSFER, + vk::PipelineStageFlags::TOP_OF_PIPE, + vk::DependencyFlags::empty(), + 0, + std::ptr::null(), + 0, + std::ptr::null(), + 1, + &b3, + ); + } + + if unsafe { (ds.fp.end_command_buffer)(cb) } != vk::Result::SUCCESS { + return; + } + + let subi = vk::SubmitInfo { + s_type: vk::StructureType::SUBMIT_INFO, + p_next: std::ptr::null(), + wait_semaphore_count: 0, + p_wait_semaphores: std::ptr::null(), + p_wait_dst_stage_mask: std::ptr::null(), + command_buffer_count: 1, + p_command_buffers: &cb, + signal_semaphore_count: 0, + p_signal_semaphores: std::ptr::null(), + _marker: std::marker::PhantomData, + }; + + unsafe { + if (ds.fp.queue_submit)(queue, 1, &subi, fence) != vk::Result::SUCCESS { + log::warn!("capture queue_submit failed — frame skipped"); + let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence); + return; + } + } + + // Toggle to the next slot + res.current = (idx + 1) % 4; +} + +unsafe fn create_capture_resources(ds: &crate::state::DeviceState) -> Option { + let pci = vk::CommandPoolCreateInfo { + s_type: vk::StructureType::COMMAND_POOL_CREATE_INFO, + p_next: std::ptr::null(), + flags: vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER, // allow per-cb reset + queue_family_index: 0, + _marker: std::marker::PhantomData, + }; + let mut cp = vk::CommandPool::null(); + if unsafe { (ds.fp.create_command_pool)(ds.raw, &pci, std::ptr::null(), &mut cp) } + != vk::Result::SUCCESS + { + return None; + } + + let ai = vk::CommandBufferAllocateInfo { + s_type: vk::StructureType::COMMAND_BUFFER_ALLOCATE_INFO, + p_next: std::ptr::null(), + command_pool: cp, + level: vk::CommandBufferLevel::PRIMARY, + command_buffer_count: 4, + _marker: std::marker::PhantomData, + }; + let mut cbs = [vk::CommandBuffer::null(); 4]; + if unsafe { (ds.fp.allocate_command_buffers)(ds.raw, &ai, cbs.as_mut_ptr()) } + != vk::Result::SUCCESS + { + unsafe { (ds.fp.destroy_command_pool)(ds.raw, cp, std::ptr::null()) }; + return None; + } + + // Create fences PRE-SIGNALED so the first wait_for_fences returns immediately + let fci = vk::FenceCreateInfo { + s_type: vk::StructureType::FENCE_CREATE_INFO, + p_next: std::ptr::null(), + flags: vk::FenceCreateFlags::SIGNALED, + _marker: std::marker::PhantomData, + }; + let mut fences = [vk::Fence::null(); 4]; + for f in &mut fences { + if unsafe { (ds.fp.create_fence)(ds.raw, &fci, std::ptr::null(), f) } != vk::Result::SUCCESS + { + unsafe { (ds.fp.destroy_command_pool)(ds.raw, cp, std::ptr::null()) }; + return None; + } + } + + Some(CaptureResources { + command_pool: cp, + command_buffers: cbs, + fences, + current: 0, + }) +} + +// ── CPU pixel readback (fallback when DMA-BUF unavailable) ─────────────────── + +pub unsafe fn read_frame_pixels( + ds: &crate::state::DeviceState, + image: vk::Image, + mem: vk::DeviceMemory, + w: u32, + h: u32, +) -> Option> { + if w == 0 || h == 0 { + return None; + } + let subresource = vk::ImageSubresource { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + array_layer: 0, + }; + let mut layout = vk::SubresourceLayout { + offset: 0, + size: 0, + row_pitch: 0, + array_pitch: 0, + depth_pitch: 0, + }; + unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &subresource, &mut layout) }; + let row_pitch = layout.row_pitch as usize; + let bpr = w as usize * 4; + let mut mp: *mut std::os::raw::c_void = std::ptr::null_mut(); + if unsafe { + (ds.fp.map_memory)( + ds.raw, + mem, + 0, + vk::WHOLE_SIZE, + vk::MemoryMapFlags::empty(), + &mut mp, + ) != vk::Result::SUCCESS + } { + return None; + } + let mut pixels = vec![0u8; bpr * h as usize]; + let base = mp as *const u8; + for row in 0..h as usize { + let src = unsafe { std::slice::from_raw_parts(base.add(row * row_pitch), bpr) }; + pixels[row * bpr..row * bpr + bpr].copy_from_slice(src); + } + unsafe { (ds.fp.unmap_memory)(ds.raw, mem) }; + Some(pixels) +} diff --git a/apps/nescapture/src/commands.rs b/apps/nescapture/src/commands.rs new file mode 100644 index 00000000..0f036ac4 --- /dev/null +++ b/apps/nescapture/src/commands.rs @@ -0,0 +1,731 @@ +// ───────────────────────────────────────────────────────────────────────────── +// commands.rs — Phase 2: command buffer hooks +// +// vkCmdBindPipeline → update CbState.active_vert_hash/active_frag_hash +// vkCmdBeginRenderPass → resolve framebuffer → views → image, populate CbState +// vkCmdEndRenderPass → clear CbState attachment fields +// vkCmdBeginRenderingKHR → resolve views → image from VkRenderingInfoKHR +// vkCmdEndRenderingKHR → clear CbState attachment fields +// ───────────────────────────────────────────────────────────────────────────── + +use crate::capture; +use crate::discovery; +use crate::state::{CB_STATE, CMD_BUF_TO_DEVICE_KEY, DEVICE_STATE}; +use ash::vk::{self, Handle}; + +// ───────────────────────────────────────────────────────────────────────────── +// vkCmdBindPipeline — track which pipeline (and thus which shader hashes) +// is currently bound for draw calls. +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdBindPipeline( + command_buffer: vk::CommandBuffer, + pipeline_bind_point: vk::PipelineBindPoint, + pipeline: vk::Pipeline, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + (ds.fp.cmd_bind_pipeline)(command_buffer, pipeline_bind_point, pipeline); + } + + if pipeline_bind_point == vk::PipelineBindPoint::GRAPHICS { + let hashes = ds + .pipeline_registry + .get(&pipeline.as_raw()) + .map(|r| r.clone()); + if let Some(hashes) = hashes { + if let Some(mut state) = CB_STATE.get_mut(&cb_key) { + state.active_vert_hash = hashes.vert_hash; + state.active_frag_hash = hashes.frag_hash; + } else { + CB_STATE.insert( + cb_key, + crate::state::CbState { + device_key, + active_vert_hash: hashes.vert_hash, + active_frag_hash: hashes.frag_hash, + ..Default::default() + }, + ); + } + + // Phase 3: Check against loaded shader hash config + if let Some(ref shader_set) = ds.shader_hashes { + let is_hud = shader_set.is_hud_shader(hashes.vert_hash, hashes.frag_hash); + let is_skip = shader_set.is_skip_shader(hashes.frag_hash); + + if is_hud { + log::debug!( + "HUD pipeline detected → vert={} frag={}", + hashes + .vert_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + hashes + .frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + ); + // Device-level HUD detection (shared across all command buffers) + if !ds + .hud_detected_frame + .load(std::sync::atomic::Ordering::Relaxed) + { + ds.hud_detected_frame + .store(true, std::sync::atomic::Ordering::Relaxed); + ds.pending_capture_frame + .store(true, std::sync::atomic::Ordering::Relaxed); + } + } + + if is_skip { + log::debug!( + "Skip pipeline detected → frag={}", + hashes + .frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + ); + } + } + + /*log::trace!( + "pipeline bound → vert={} frag={}", + hashes + .vert_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + hashes + .frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + );*/ + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkCmdBeginRenderPass — resolve the current color attachment image. +// +// The framebuffer contains VkImageViews. We look up each view in the +// view_to_image map to get the VkImage, and use view_format for the format. +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdBeginRenderPass( + command_buffer: vk::CommandBuffer, + p_render_pass_begin: *const vk::RenderPassBeginInfo, + contents: vk::SubpassContents, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + let rpb = unsafe { &*p_render_pass_begin }; + let framebuffer = rpb.framebuffer; + + let (color_image, format, extent) = if let Some(views) = + ds.framebuffer_to_views.get(&framebuffer.as_raw()) + { + let view_keys: Vec = views.iter().copied().collect(); + let extent = ds + .framebuffer_extent + .get(&framebuffer.as_raw()) + .map(|r| *r) + .unwrap_or(vk::Extent2D { + width: rpb.render_area.extent.width, + height: rpb.render_area.extent.height, + }); + + if !view_keys.is_empty() { + let first_view = view_keys[0]; + let image = ds + .view_to_image + .get(&first_view) + .map(|r| vk::Image::from_raw(*r)); + let fmt = ds.view_format.get(&first_view).map(|r| *r); + log::debug!( + "vkCmdBeginRenderPass → fb={:#010x} view={:#010x} image={:?} format={} extent={}x{}", + framebuffer.as_raw(), + first_view, + image.map(|i| i.as_raw()), + fmt.map(|f| f.as_raw()).unwrap_or(0), + extent.width, + extent.height, + ); + (image, fmt, Some(extent)) + } else { + log::debug!( + "vkCmdBeginRenderPass → fb={:#010x} no views", + framebuffer.as_raw(), + ); + (None, None, Some(extent)) + } + } else { + log::debug!( + "vkCmdBeginRenderPass → fb={:#010x} NOT FOUND in framebuffer_to_views", + framebuffer.as_raw(), + ); + (None, None, None) + }; + + unsafe { + (ds.fp.cmd_begin_render_pass)(command_buffer, p_render_pass_begin, contents); + } + + if let Some(mut state) = CB_STATE.get_mut(&cb_key) { + state.current_color_image = color_image; + state.current_image_format = format; + state.current_image_extent = extent; + } else { + CB_STATE.insert( + cb_key, + crate::state::CbState { + device_key, + current_color_image: color_image, + current_image_format: format, + current_image_extent: extent, + ..Default::default() + }, + ); + } + + if let (Some(img), Some(ext)) = (color_image, extent) { + log::debug!( + "render pass begin → color attachment image {:#010x} extent {}x{}", + img.as_raw(), + ext.width, + ext.height, + ); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdEndRenderPass(command_buffer: vk::CommandBuffer) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + // Phase 4: track the largest extent we've seen (main framebuffer) + let current_extent = CB_STATE + .get(&cb_key) + .map(|r| r.current_image_extent) + .unwrap_or(None); + if let Some(ext) = current_extent { + let mut largest = ds.largest_extent.lock().unwrap(); + if ext.width * ext.height > largest.width * largest.height { + *largest = ext; + } + } + + // Phase 4: check if we need to inject a HUDless capture copy (device-level) + let needs_hudless_capture = ds + .pending_capture_frame + .load(std::sync::atomic::Ordering::Relaxed) + && !ds + .capture_injected_frame + .load(std::sync::atomic::Ordering::Relaxed); + + unsafe { + (ds.fp.cmd_end_render_pass)(command_buffer); + } + + // Inject HUDless capture if HUD was detected this frame + if needs_hudless_capture { + log::info!("injecting HUDless capture copy for cb {:#010x}", cb_key); + unsafe { + capture::inject_hudless_copy(command_buffer, device_key); + } + ds.pending_capture_frame + .store(false, std::sync::atomic::Ordering::Relaxed); + ds.capture_injected_frame + .store(true, std::sync::atomic::Ordering::Relaxed); + } + + let skipped = ds + .skipped_draws_frame + .swap(0, std::sync::atomic::Ordering::Relaxed); + if skipped > 0 { + log::info!("render pass end → {} draws skipped", skipped); + } + + if let Some(mut state) = CB_STATE.get_mut(&cb_key) { + state.current_color_image = None; + state.current_image_format = None; + state.current_image_extent = None; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkCmdBeginRenderingKHR — dynamic rendering path (DXVK 1.10+). +// +// Unlike the framebuffer path, VkRenderingInfoKHR gives VkImageViews directly +// in VkRenderingAttachmentInfoKHR. No framebuffer object is involved. +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdBeginRenderingKHR( + command_buffer: vk::CommandBuffer, + p_rendering_info: *const vk::RenderingInfo, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + let ri = unsafe { &*p_rendering_info }; + let extent = ri.render_area.extent; + + let (color_image, format) = + if ri.color_attachment_count > 0 && !ri.p_color_attachments.is_null() { + let first = unsafe { &*ri.p_color_attachments }; + if first.image_view != vk::ImageView::null() { + let image = ds + .view_to_image + .get(&first.image_view.as_raw()) + .map(|r| vk::Image::from_raw(*r)); + let fmt = ds.view_format.get(&first.image_view.as_raw()).map(|r| *r); + (image, fmt) + } else { + (None, None) + } + } else { + (None, None) + }; + + unsafe { + (ds.fp.cmd_begin_rendering_khr.unwrap())(command_buffer, p_rendering_info); + } + + if let Some(mut state) = CB_STATE.get_mut(&cb_key) { + state.current_color_image = color_image; + state.current_image_format = format; + state.current_image_extent = Some(extent); + } else { + CB_STATE.insert( + cb_key, + crate::state::CbState { + device_key, + current_color_image: color_image, + current_image_format: format, + current_image_extent: Some(extent), + ..Default::default() + }, + ); + } + + log::info!( + "vkCmdBeginRenderingKHR → color_image={:?} format={:?} extent={}x{}", + color_image.map(|i| i.as_raw()), + format.map(|f| f.as_raw()), + extent.width, + extent.height, + ); +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdEndRenderingKHR(command_buffer: vk::CommandBuffer) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + // Phase 4: check if we need to inject a HUDless capture copy (device-level) + let needs_hudless_capture = ds + .pending_capture_frame + .load(std::sync::atomic::Ordering::Relaxed) + && !ds + .capture_injected_frame + .load(std::sync::atomic::Ordering::Relaxed); + + unsafe { + (ds.fp.cmd_end_rendering_khr.unwrap())(command_buffer); + } + + // Inject HUDless capture if HUD was detected this frame + if needs_hudless_capture { + log::info!("injecting HUDless capture copy for cb {:#010x}", cb_key); + unsafe { + capture::inject_hudless_copy(command_buffer, device_key); + } + ds.pending_capture_frame + .store(false, std::sync::atomic::Ordering::Relaxed); + ds.capture_injected_frame + .store(true, std::sync::atomic::Ordering::Relaxed); + } + + let skipped = ds + .skipped_draws_frame + .swap(0, std::sync::atomic::Ordering::Relaxed); + if skipped > 0 { + log::info!("rendering end → {} draws skipped", skipped); + } + + if let Some(mut state) = CB_STATE.get_mut(&cb_key) { + state.current_color_image = None; + state.current_image_format = None; + state.current_image_extent = None; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Phase 6: vkCmdDraw* hooks — skip HUD/skip shaders, log for discovery +// +// When a config is loaded and the active pipeline matches a HUD or skip shader, +// the draw call is NOT forwarded to the driver. This prevents the HUD from +// being rendered to the color attachment. The nescapture capture at +// vkCmdEndRenderPass then copies the clean (HUD-free) attachment. +// +// In discovery mode (HUDLESS_DISCOVER=1), draws are never skipped. +// ───────────────────────────────────────────────────────────────────────────── + +/// Check if the currently bound pipeline should be suppressed. +/// Returns (should_skip, vert_hash, frag_hash) so the caller doesn't need +/// to re-acquire the CB_STATE lock. +unsafe fn should_skip_draw( + cb_key: u64, + ds: &std::sync::Arc, +) -> (bool, Option, Option) { + if discovery::is_discovery_mode() { + return (false, None, None); + } + let shader_set = match &ds.shader_hashes { + Some(s) => s, + None => return (false, None, None), + }; + let state = match CB_STATE.get(&cb_key) { + Some(s) => s, + None => return (false, None, None), + }; + let vh = state.active_vert_hash; + let fh = state.active_frag_hash; + let skip = shader_set.is_hud_shader(vh, fh) || shader_set.is_skip_shader(fh); + (skip, vh, fh) +} + +/// Record that a draw was skipped (for diagnostics). +unsafe fn record_skipped_draw( + ds: &crate::state::DeviceState, + vert_hash: Option, + frag_hash: Option, +) { + ds.skipped_draws_frame + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + log::info!( + "SKIPPING draw → vert={} frag={}", + vert_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + ); +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDraw( + command_buffer: vk::CommandBuffer, + vertex_count: u32, + instance_count: u32, + first_vertex: u32, + first_instance: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, vertex_count); + return; + } + + (ds.fp.cmd_draw)( + command_buffer, + vertex_count, + instance_count, + first_vertex, + first_instance, + ); + + discovery::record_draw(command_buffer, vertex_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndexed( + command_buffer: vk::CommandBuffer, + index_count: u32, + instance_count: u32, + first_index: u32, + vertex_offset: i32, + first_instance: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, index_count); + return; + } + + (ds.fp.cmd_draw_indexed)( + command_buffer, + index_count, + instance_count, + first_index, + vertex_offset, + first_instance, + ); + + discovery::record_draw(command_buffer, index_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndirect( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + draw_count: u32, + stride: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, draw_count); + return; + } + + (ds.fp.cmd_draw_indirect)(command_buffer, buffer, offset, draw_count, stride); + + discovery::record_draw(command_buffer, draw_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndexedIndirect( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + draw_count: u32, + stride: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, draw_count); + return; + } + + (ds.fp.cmd_draw_indexed_indirect)(command_buffer, buffer, offset, draw_count, stride); + + discovery::record_draw(command_buffer, draw_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndirectCount( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + count_buffer: vk::Buffer, + count_buffer_offset: vk::DeviceSize, + max_draw_count: u32, + stride: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, max_draw_count); + return; + } + + if let Some(f) = ds.fp.cmd_draw_indirect_count { + f( + command_buffer, + buffer, + offset, + count_buffer, + count_buffer_offset, + max_draw_count, + stride, + ); + } + + discovery::record_draw(command_buffer, max_draw_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndexedIndirectCount( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + count_buffer: vk::Buffer, + count_buffer_offset: vk::DeviceSize, + max_draw_count: u32, + stride: u32, +) { + let cb_key = command_buffer.as_raw(); + let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) { + Some(r) => *r, + None => return, + }; + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s.clone(), + None => return, + }; + + unsafe { + let (skip, vh, fh) = should_skip_draw(cb_key, &ds); + if skip { + record_skipped_draw(&ds, vh, fh); + discovery::record_draw(command_buffer, max_draw_count); + return; + } + + if let Some(f) = ds.fp.cmd_draw_indexed_indirect_count { + f( + command_buffer, + buffer, + offset, + count_buffer, + count_buffer_offset, + max_draw_count, + stride, + ); + } + + discovery::record_draw(command_buffer, max_draw_count); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndirectCountKHR( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + count_buffer: vk::Buffer, + count_buffer_offset: vk::DeviceSize, + max_draw_count: u32, + stride: u32, +) { + unsafe { + vkCmdDrawIndirectCount( + command_buffer, + buffer, + offset, + count_buffer, + count_buffer_offset, + max_draw_count, + stride, + ); + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCmdDrawIndexedIndirectCountKHR( + command_buffer: vk::CommandBuffer, + buffer: vk::Buffer, + offset: vk::DeviceSize, + count_buffer: vk::Buffer, + count_buffer_offset: vk::DeviceSize, + max_draw_count: u32, + stride: u32, +) { + unsafe { + vkCmdDrawIndexedIndirectCount( + command_buffer, + buffer, + offset, + count_buffer, + count_buffer_offset, + max_draw_count, + stride, + ); + } +} diff --git a/apps/nescapture/src/config.rs b/apps/nescapture/src/config.rs new file mode 100644 index 00000000..217dff6e --- /dev/null +++ b/apps/nescapture/src/config.rs @@ -0,0 +1,141 @@ +// ───────────────────────────────────────────────────────────────────────────── +// config.rs — Phase 3: per-game shader hash configuration +// +// Loads a TOML config file that maps game executables to sets of shader hashes: +// hud_fragment_shaders — fragment shaders used for HUD/UI elements +// hud_vertex_shaders — vertex shaders used for HUD/UI elements +// skip_fragment_shaders — fragment shaders to silently drop (motion blur, etc.) +// +// Environment variables: +// HUDLESS_CONFIG=/path/to/config.toml — explicit config path +// HUDLESS_GAME_NAME=Game.exe — override game name detection +// +// Config format: +// [game."Control_DX11.exe"] +// hud_fragment_shaders = ["0xaabbccddeeff0011"] +// hud_vertex_shaders = ["0x1a2b3c4d5e6f7890"] +// skip_fragment_shaders = ["0x1122334455667788"] +// ───────────────────────────────────────────────────────────────────────────── + +use serde::Deserialize; +use std::collections::HashSet; +use std::fs; +use std::path::Path; + +/// Parsed shader hash sets for a single game. +#[derive(Clone, Debug, Default)] +pub struct ShaderHashSet { + pub hud_fragment_shaders: HashSet, + pub hud_vertex_shaders: HashSet, + pub skip_fragment_shaders: HashSet, +} + +impl ShaderHashSet { + /// Check if the given vert/frag hash pair matches a HUD shader. + /// + /// A HUD match occurs when: + /// - frag_hash is in hud_fragment_shaders, OR + /// - vert_hash is in hud_vertex_shaders (if frag not set) + pub fn is_hud_shader(&self, vert_hash: Option, frag_hash: Option) -> bool { + match frag_hash { + Some(fh) => self.hud_fragment_shaders.contains(&fh), + None => vert_hash.is_some_and(|vh| self.hud_vertex_shaders.contains(&vh)), + } + } + + /// Check if the given frag_hash matches a skip shader. + pub fn is_skip_shader(&self, frag_hash: Option) -> bool { + if let Some(fh) = frag_hash { + return self.skip_fragment_shaders.contains(&fh); + } + false + } +} + +/// TOML deserialization structure. +#[derive(Deserialize)] +struct ConfigFile { + game: Option>, +} + +#[derive(Deserialize)] +struct GameConfig { + hud_fragment_shaders: Option>, + hud_vertex_shaders: Option>, + skip_fragment_shaders: Option>, +} + +/// Parse a hex string like "0xaabbccddeeff0011" into a u64. +fn parse_hex(s: &str) -> Option { + let trimmed = s.trim().trim_start_matches("0x").trim_start_matches("0X"); + u64::from_str_radix(trimmed, 16).ok() +} + +/// Load the config from the given path and return the ShaderHashSet for the +/// current game. +pub fn load_config(path: &Path) -> Option { + let content = match fs::read_to_string(path) { + Ok(c) => c, + Err(e) => { + log::warn!("failed to read config {:?}: {}", path, e); + return None; + } + }; + + let config: ConfigFile = match toml::from_str(&content) { + Ok(c) => c, + Err(e) => { + log::warn!("failed to parse config {:?}: {}", path, e); + return None; + } + }; + + let game_name = std::env::var("NESCAPTURE_GAME_NAME").ok()?; + let games = config.game.as_ref()?; + let game_config = games.get(&game_name)?; + + let mut set = ShaderHashSet::default(); + + if let Some(hashes) = &game_config.hud_fragment_shaders { + for h in hashes { + if let Some(v) = parse_hex(h) { + set.hud_fragment_shaders.insert(v); + } + } + } + + if let Some(hashes) = &game_config.hud_vertex_shaders { + for h in hashes { + if let Some(v) = parse_hex(h) { + set.hud_vertex_shaders.insert(v); + } + } + } + + if let Some(hashes) = &game_config.skip_fragment_shaders { + for h in hashes { + if let Some(v) = parse_hex(h) { + set.skip_fragment_shaders.insert(v); + } + } + } + + log::info!( + "loaded config for '{}' — {} hud_frag, {} hud_vert, {} skip_frag", + game_name, + set.hud_fragment_shaders.len(), + set.hud_vertex_shaders.len(), + set.skip_fragment_shaders.len(), + ); + + Some(set) +} + +/// Resolve the config file path from HUDLESS_CONFIG env var, or fall back to +/// a default location relative to the game executable. +pub fn resolve_config_path() -> Option { + if let Ok(path) = std::env::var("NESCAPTURE_CONFIG") { + return Some(std::path::PathBuf::from(path)); + } + None +} diff --git a/apps/nescapture/src/device.rs b/apps/nescapture/src/device.rs new file mode 100644 index 00000000..9a1dcff2 --- /dev/null +++ b/apps/nescapture/src/device.rs @@ -0,0 +1,528 @@ +// ───────────────────────────────────────────────────────────────────────────── +// device.rs — vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue +// ───────────────────────────────────────────────────────────────────────────── + +use crate::config; +use crate::dispatch::{NextDeviceFn, PFN_vkCreateDevice, PFN_vkGetInstanceProcAddr}; +use crate::state::{DEVICE_STATE, DeviceState, INSTANCE_STATE, QUEUE_TO_DEVICE_KEY}; +use crate::{ + VkLayerDeviceCreateInfo, dispatch_key, find_layer_link, load_device_fn, try_load_device_fn, +}; +use ash::vk::{self, Handle}; +use dashmap::DashMap; +use std::os::raw::c_void; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +const VK_LAYER_LINK_INFO: u32 = 0; + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateDevice( + physical_device: vk::PhysicalDevice, + p_create_info: *const vk::DeviceCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_device: *mut vk::Device, +) -> vk::Result { + let layer_info: *mut VkLayerDeviceCreateInfo = match unsafe { + find_layer_link((*p_create_info).p_next as *const c_void, VK_LAYER_LINK_INFO) + } { + Some(p) => p, + None => return vk::Result::ERROR_INITIALIZATION_FAILED, + }; + + let dev_link = unsafe { (*layer_info).u.pDeviceLayerInfo }; + let next_gipa: PFN_vkGetInstanceProcAddr = + match unsafe { (*dev_link).pfnNextGetInstanceProcAddr } { + Some(f) => f, + None => return vk::Result::ERROR_INITIALIZATION_FAILED, + }; + let next_gdpa = unsafe { + match (*dev_link).pfnNextGetDeviceProcAddr { + Some(f) => f, + None => return vk::Result::ERROR_INITIALIZATION_FAILED, + } + }; + unsafe { (*layer_info).u.pDeviceLayerInfo = (*dev_link).pNext }; + + let inst_key = unsafe { dispatch_key(physical_device.as_raw() as *const c_void) }; + let istate = match INSTANCE_STATE.get(&inst_key) { + Some(s) => s.clone(), + None => { + let next_create: PFN_vkCreateDevice = unsafe { + crate::load_instance_fn(next_gipa, vk::Instance::null(), b"vkCreateDevice\0") + }; + return unsafe { next_create(physical_device, p_create_info, p_allocator, p_device) }; + } + }; + + // ── Inject DMA-BUF extensions for zero-copy capture ────────────────── + let ci = unsafe { &*p_create_info }; + + // Collect the game's original extensions. + let original_extensions: Vec<*const libc::c_char> = + if ci.enabled_extension_count > 0 && !ci.pp_enabled_extension_names.is_null() { + unsafe { + std::slice::from_raw_parts( + ci.pp_enabled_extension_names, + ci.enabled_extension_count as usize, + ) + } + .to_vec() + } else { + Vec::new() + }; + + // Extensions we need — static byte strings so pointers stay valid. + const EXT_EXTERNAL_MEMORY: &[u8] = b"VK_KHR_external_memory\0"; + const EXT_EXTERNAL_MEMORY_FD: &[u8] = b"VK_KHR_external_memory_fd\0"; + const EXT_EXTERNAL_MEMORY_DMABUF: &[u8] = b"VK_EXT_external_memory_dma_buf\0"; + + let needed: &[&[u8]] = &[ + EXT_EXTERNAL_MEMORY, + EXT_EXTERNAL_MEMORY_FD, + EXT_EXTERNAL_MEMORY_DMABUF, + ]; + + // Build extended list: original + any of ours not already present. + let mut extended = original_extensions.clone(); + for &ext in needed { + let name_cstr = unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(ext) }; + let already = extended + .iter() + .any(|&ptr| unsafe { std::ffi::CStr::from_ptr(ptr) == name_cstr }); + if !already { + extended.push(ext.as_ptr() as *const libc::c_char); + } + } + + // ── Bump queue count for dedicated capture queue ────────────────── + // Add one extra queue to the first queue family so capture + // submissions don't compete with game rendering. + let mut capture_queue_index = 0u32; + let queue_infos: Vec = if ci.queue_create_info_count > 0 + && !ci.p_queue_create_infos.is_null() + { + let slice = unsafe { + std::slice::from_raw_parts(ci.p_queue_create_infos, ci.queue_create_info_count as usize) + }; + let mut qis = slice.to_vec(); + if let Some(first) = qis.first_mut() { + capture_queue_index = first.queue_count; // use the NEXT index + first.queue_count += 1; + } + qis + } else { + Vec::new() + }; + + // Try with injected extensions first. + let mut modified_ci = *ci; + modified_ci.enabled_extension_count = extended.len() as u32; + modified_ci.pp_enabled_extension_names = extended.as_ptr(); + modified_ci.queue_create_info_count = queue_infos.len() as u32; + modified_ci.p_queue_create_infos = queue_infos.as_ptr(); + + let mut dmabuf_available = true; + let result = + unsafe { (istate.create_device)(physical_device, &modified_ci, p_allocator, p_device) }; + + let result = if result != vk::Result::SUCCESS { + // Driver rejected our extensions — retry with original create info. + log::warn!( + "vkCreateDevice with DMA-BUF extensions failed ({:?}), \ + retrying without — CPU readback fallback will be used", + result + ); + dmabuf_available = false; + unsafe { (istate.create_device)(physical_device, p_create_info, p_allocator, p_device) } + } else { + log::info!("DMA-BUF extensions injected successfully"); + result + }; + if result != vk::Result::SUCCESS { + return result; + } + + if !dmabuf_available { + log::warn!("DMA-BUF extensions missing — will use CPU readback fallback (expensive!)"); + } + + let device = unsafe { *p_device }; + + // Cache physical device memory properties + let mut mem_props = vk::PhysicalDeviceMemoryProperties::default(); + unsafe { (istate.get_physical_device_memory_properties)(physical_device, &mut mem_props) }; + + macro_rules! load { + ($name:literal) => { + unsafe { load_device_fn(next_gdpa, device, $name) } + }; + } + macro_rules! try_load { + ($name:literal) => { + unsafe { try_load_device_fn(next_gdpa, device, $name) } + }; + } + + let fp = NextDeviceFn { + // Infrastructure + get_device_proc_addr: next_gdpa, + destroy_device: load!(b"vkDestroyDevice\0"), + get_device_queue: load!(b"vkGetDeviceQueue\0"), + queue_present_khr: try_load!(b"vkQueuePresentKHR\0"), + + // Phase 1 + create_shader_module: load!(b"vkCreateShaderModule\0"), + destroy_shader_module: load!(b"vkDestroyShaderModule\0"), + create_graphics_pipelines: load!(b"vkCreateGraphicsPipelines\0"), + destroy_pipeline: load!(b"vkDestroyPipeline\0"), + + // Phase 2 + create_image_view: load!(b"vkCreateImageView\0"), + destroy_image_view: load!(b"vkDestroyImageView\0"), + create_framebuffer: load!(b"vkCreateFramebuffer\0"), + destroy_framebuffer: load!(b"vkDestroyFramebuffer\0"), + allocate_command_buffers: load!(b"vkAllocateCommandBuffers\0"), + free_command_buffers: load!(b"vkFreeCommandBuffers\0"), + cmd_bind_pipeline: load!(b"vkCmdBindPipeline\0"), + cmd_begin_render_pass: load!(b"vkCmdBeginRenderPass\0"), + cmd_end_render_pass: load!(b"vkCmdEndRenderPass\0"), + cmd_begin_rendering_khr: try_load!(b"vkCmdBeginRenderingKHR\0"), + cmd_end_rendering_khr: try_load!(b"vkCmdEndRenderingKHR\0"), + + // Phase 4 — capture images + create_image: load!(b"vkCreateImage\0"), + destroy_image: load!(b"vkDestroyImage\0"), + allocate_memory: load!(b"vkAllocateMemory\0"), + free_memory: load!(b"vkFreeMemory\0"), + bind_image_memory: load!(b"vkBindImageMemory\0"), + get_image_memory_requirements: load!(b"vkGetImageMemoryRequirements\0"), + map_memory: load!(b"vkMapMemory\0"), + unmap_memory: load!(b"vkUnmapMemory\0"), + cmd_pipeline_barrier: load!(b"vkCmdPipelineBarrier\0"), + cmd_copy_image: load!(b"vkCmdCopyImage\0"), + get_image_subresource_layout: load!(b"vkGetImageSubresourceLayout\0"), + get_memory_fd_khr: try_load!(b"vkGetMemoryFdKHR\0"), + + // Phase 4 — synchronisation + create_fence: load!(b"vkCreateFence\0"), + destroy_fence: load!(b"vkDestroyFence\0"), + create_command_pool: load!(b"vkCreateCommandPool\0"), + destroy_command_pool: load!(b"vkDestroyCommandPool\0"), + reset_command_pool: load!(b"vkResetCommandPool\0"), + begin_command_buffer: load!(b"vkBeginCommandBuffer\0"), + end_command_buffer: load!(b"vkEndCommandBuffer\0"), + reset_command_buffer: load!(b"vkResetCommandBuffer\0"), // needed for double-buffered capture + queue_submit: load!(b"vkQueueSubmit\0"), + wait_for_fences: load!(b"vkWaitForFences\0"), + reset_fences: load!(b"vkResetFences\0"), + + // Phase 4 — swapchain + create_swapchain_khr: try_load!(b"vkCreateSwapchainKHR\0"), + destroy_swapchain_khr: try_load!(b"vkDestroySwapchainKHR\0"), + get_swapchain_images_khr: try_load!(b"vkGetSwapchainImagesKHR\0"), + + // Phase 6 — draw commands + cmd_draw: load!(b"vkCmdDraw\0"), + cmd_draw_indexed: load!(b"vkCmdDrawIndexed\0"), + cmd_draw_indirect: load!(b"vkCmdDrawIndirect\0"), + cmd_draw_indexed_indirect: load!(b"vkCmdDrawIndexedIndirect\0"), + cmd_draw_indirect_count: try_load!(b"vkCmdDrawIndirectCount\0"), + cmd_draw_indexed_indirect_count: try_load!(b"vkCmdDrawIndexedIndirectCount\0"), + }; + + // ── Retrieve capture queue from the bumped slot ─────────────────── + let mut capture_queue = vk::Queue::null(); + if queue_infos + .first() + .map(|q| q.queue_count > 1) + .unwrap_or(false) + { + let qi = &queue_infos[0]; + unsafe { + (fp.get_device_queue)( + device, + qi.queue_family_index, + capture_queue_index, + &mut capture_queue, + ); + } + log::info!( + "capture queue: family={} index={capture_queue_index}", + qi.queue_family_index + ); + } + + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + + // Phase 3: load shader hash config + let shader_hashes = config::resolve_config_path() + .as_ref() + .and_then(|p| config::load_config(p)); + + if let Some(ref set) = shader_hashes { + log::info!( + "config loaded — {} hud_frag, {} hud_vert, {} skip_frag", + set.hud_fragment_shaders.len(), + set.hud_vertex_shaders.len(), + set.skip_fragment_shaders.len(), + ); + } else { + log::warn!("no config — draw suppression disabled"); + } + + let dev_state = Arc::new(DeviceState { + raw: device, + physical_device, + fp, + + shader_registry: DashMap::new(), + pipeline_registry: DashMap::new(), + pipeline_state: DashMap::new(), + + view_to_image: DashMap::new(), + view_format: DashMap::new(), + framebuffer_to_views: DashMap::new(), + framebuffer_extent: DashMap::new(), + + shader_hashes, + + hudless_image: std::sync::Mutex::new(None), + hudless_memory: std::sync::Mutex::new(None), + hudless_size: std::sync::Mutex::new((0, 0, vk::Format::UNDEFINED)), + final_image: std::sync::Mutex::new(None), + final_memory: std::sync::Mutex::new(None), + final_size: std::sync::Mutex::new((0, 0, vk::Format::UNDEFINED)), + final_stride: std::sync::atomic::AtomicU32::new(0), + swapchain: std::sync::Mutex::new(None), + swapchain_images: std::sync::Mutex::new(Vec::new()), + swapchain_format: std::sync::Mutex::new(vk::Format::UNDEFINED), + swapchain_extent: std::sync::Mutex::new(vk::Extent2D { + width: 0, + height: 0, + }), + swapchain_colorspace: std::sync::atomic::AtomicU32::new(0), + frame_counter: std::sync::atomic::AtomicU64::new(0), + largest_extent: std::sync::Mutex::new(vk::Extent2D { + width: 0, + height: 0, + }), + + hud_detected_frame: std::sync::atomic::AtomicBool::new(false), + pending_capture_frame: std::sync::atomic::AtomicBool::new(false), + capture_injected_frame: std::sync::atomic::AtomicBool::new(false), + skipped_draws_frame: std::sync::atomic::AtomicU32::new(0), + + encoder: std::sync::Mutex::new(None), + + capture_resources: std::sync::Mutex::new(None), + capture_queue: std::sync::Mutex::new(capture_queue), + fake_images: std::sync::Mutex::new(Vec::new()), + fake_memories: std::sync::Mutex::new(Vec::new()), + fake_fds: std::sync::Mutex::new(Vec::new()), + fake_strides: std::sync::Mutex::new(Vec::new()), + fake_available: std::sync::Mutex::new(Vec::new()), + fake_image_count: std::sync::atomic::AtomicU32::new(0), + fake_swapchain: std::sync::Mutex::new(None), + signal_queue: std::sync::Mutex::new(vk::Queue::null()), + next_acquire: std::sync::atomic::AtomicU32::new(0), + memory_properties: std::sync::Mutex::new(mem_props), + acquire_dummy_pool: std::sync::Mutex::new(vk::CommandPool::null()), + acquire_dummy_cb: std::sync::Mutex::new(vk::CommandBuffer::null()), + cached_dmabuf_fd: std::sync::atomic::AtomicI32::new(-1), + + target_fps: std::sync::atomic::AtomicU32::new(0), + last_capture_time: std::sync::Mutex::new(None), + capture_tx: std::sync::Mutex::new(None), + }); + + DEVICE_STATE.insert(key, dev_state); + + log::info!("vkCreateDevice OK — key {:#x}", key); + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyDevice( + device: vk::Device, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.remove(&key) { + Some((_, ds)) => ds, + None => return, + }; + + log::info!( + "vkDestroyDevice — {} shaders, {} pipelines evicted", + ds.shader_registry.len(), + ds.pipeline_registry.len(), + ); + + // ── 1. Shut down encoder pipeline (unblocks encoder + RTP threads) ─── + { + let mut enc_guard = ds.encoder.lock().unwrap(); + if let Some(handle) = enc_guard.take() { + handle.shutdown(); + // `handle` is dropped here → drops `frame_tx` → encoder thread's + // recv_timeout returns Disconnected → encoder thread drops + // `encoded_tx` → RTP thread exits too. + // + // Give threads a moment to drain. In production you'd join the + // JoinHandles, but since we don't store them, a short sleep + + // the AtomicBool shutdown flag is sufficient. + log::info!("encoder pipeline shutdown signaled"); + } + } + // Brief yield to let threads notice the disconnect. + std::thread::sleep(std::time::Duration::from_millis(50)); + + // ── 2. Clean up capture resources (double-buffered cmd pool + fences) ─ + { + let mut res_guard = ds.capture_resources.lock().unwrap(); + if let Some(res) = res_guard.take() { + unsafe { + // Wait for any in-flight capture commands to finish before + // destroying the fences / command pool. + let _ = (ds.fp.wait_for_fences)( + ds.raw, + res.fences.len() as u32, + res.fences.as_ptr(), + vk::TRUE, + 5_000_000_000, // 5 seconds — should be instant + ); + for &f in &res.fences { + (ds.fp.destroy_fence)(ds.raw, f, std::ptr::null()); + } + (ds.fp.destroy_command_pool)(ds.raw, res.command_pool, std::ptr::null()); + } + log::debug!("capture resources destroyed"); + } + } + + // ── 3. Free final_image / final_memory ──────────────────────────────── + { + let img = ds.final_image.lock().unwrap().take(); + let mem = ds.final_memory.lock().unwrap().take(); + if let Some(i) = img { + unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) }; + } + if let Some(m) = mem { + unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) }; + } + } + + // ── 4. Free hudless_image / hudless_memory ──────────────────────────── + { + let img = ds.hudless_image.lock().unwrap().take(); + let mem = ds.hudless_memory.lock().unwrap().take(); + if let Some(i) = img { + unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) }; + } + if let Some(m) = mem { + unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) }; + } + } + + // ── 5. Close cached DMA-BUF fd ──────────────────────────────────────── + { + let fd = ds.cached_dmabuf_fd.load(Ordering::Relaxed); + if fd >= 0 { + unsafe { libc::close(fd) }; + log::debug!("cached DMA-BUF fd {} closed", fd); + } + } + + // ── 6. Clean up queue → device key mappings for this device ─────────── + QUEUE_TO_DEVICE_KEY.retain(|_, dk| *dk != key); + + // ── 7. Call the real vkDestroyDevice ────────────────────────────────── + unsafe { (ds.fp.destroy_device)(device, p_allocator) }; + + log::info!("vkDestroyDevice complete"); +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkGetDeviceQueue( + device: vk::Device, + queue_family_index: u32, + queue_index: u32, + p_queue: *mut vk::Queue, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + unsafe { (ds.fp.get_device_queue)(device, queue_family_index, queue_index, p_queue) }; + let queue = unsafe { *p_queue }; + QUEUE_TO_DEVICE_KEY.insert(queue.as_raw(), key); + // Store first queue for acquire semaphore signaling + let mut sq = ds.signal_queue.lock().unwrap(); + if *sq == vk::Queue::null() { + *sq = queue; + } + } +} + +/// Enumerate device extensions supported by the physical device. +unsafe fn enumerate_device_extensions( + istate: &crate::dispatch::NextInstanceFn, + physical_device: vk::PhysicalDevice, +) -> Vec { + // We need vkEnumerateDeviceExtensionProperties. Load it from the + // instance dispatch since it's a physical-device-level function. + // For simplicity, use ash's raw function signature. + type PFN_vkEnumerateDeviceExtensionProperties = unsafe extern "system" fn( + vk::PhysicalDevice, + *const libc::c_char, + *mut u32, + *mut vk::ExtensionProperties, + ) -> vk::Result; + + let func: Option = { + let raw = unsafe { + (istate.get_instance_proc_addr)( + vk::Instance::null(), + b"vkEnumerateDeviceExtensionProperties\0".as_ptr() as *const libc::c_char, + ) + }; + raw.map(|f| unsafe { std::mem::transmute(f) }) + }; + + let Some(enumerate) = func else { + log::warn!("could not load vkEnumerateDeviceExtensionProperties"); + return Vec::new(); + }; + + let mut count = 0u32; + if unsafe { + enumerate( + physical_device, + std::ptr::null(), + &mut count, + std::ptr::null_mut(), + ) + } != vk::Result::SUCCESS + { + return Vec::new(); + } + + let mut props = vec![vk::ExtensionProperties::default(); count as usize]; + if unsafe { + enumerate( + physical_device, + std::ptr::null(), + &mut count, + props.as_mut_ptr(), + ) + } != vk::Result::SUCCESS + { + return Vec::new(); + } + + props + .iter() + .filter_map(|p| { + let cstr = unsafe { std::ffi::CStr::from_ptr(p.extension_name.as_ptr()) }; + Some(cstr.to_owned()) + }) + .collect() +} diff --git a/apps/nescapture/src/discovery.rs b/apps/nescapture/src/discovery.rs new file mode 100644 index 00000000..8e410ae6 --- /dev/null +++ b/apps/nescapture/src/discovery.rs @@ -0,0 +1,165 @@ +// ───────────────────────────────────────────────────────────────────────────── +// discovery.rs — Phase 6: per-draw logging for shader hash discovery +// +// When HUDLESS_DISCOVER=1 is set, every vkCmdDraw* call is logged to: +// /tmp/hudless_discover_$EXE.log +// +// Log format: +// frame=0001 draw=00042 vert=0x1a2b3c4d5e6f7890 frag=0xaabbccddeeff0011 verts=6 blend=true depth=false +// +// HUD suspect heuristics: +// - blend=true && depth_test=false → strong suspect (UI quads) +// - verts <= 6 → two triangles = one quad +// - Same pipeline reused many times per frame at low vertex count +// ───────────────────────────────────────────────────────────────────────────── + +use crate::state::{CB_STATE, DEVICE_STATE}; +use ash::vk::{self, Handle}; +use std::fs::OpenOptions; +use std::io::Write; +use std::sync::Mutex; + +static LOG_FILE: Mutex> = Mutex::new(None); + +pub fn is_discovery_mode() -> bool { + std::env::var("NESCAPTURE_DISCOVER") + .map(|v| v == "1") + .unwrap_or(false) +} + +fn ensure_log_file() -> bool { + let mut guard = LOG_FILE.lock().unwrap(); + if guard.is_some() { + return true; + } + + let exe_name = std::env::var("NESCAPTURE_GAME_NAME").unwrap_or_else(|_| { + std::fs::read_link("/proc/self/exe") + .ok() + .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) + .unwrap_or_else(|| "unknown".to_string()) + }); + + let path = format!("/tmp/hudless_discover_{}.log", exe_name); + match OpenOptions::new().create(true).append(true).open(&path) { + Ok(f) => { + log::info!("discovery logging to {}", path); + *guard = Some(f); + true + } + Err(e) => { + log::warn!("failed to open discovery log {}: {}", path, e); + false + } + } +} + +pub fn log_draw( + frame: u64, + draw: u32, + vert_hash: Option, + frag_hash: Option, + vert_count: u32, + blend_enabled: bool, + depth_test_enabled: bool, +) { + if !ensure_log_file() { + return; + } + + let mut guard = match LOG_FILE.lock() { + Ok(g) => g, + Err(_) => return, + }; + let file = match guard.as_mut() { + Some(f) => f, + None => return, + }; + + let vert_str = vert_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()); + let frag_str = frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()); + + let is_hud_suspect = blend_enabled && !depth_test_enabled && vert_count <= 6; + + let suspect_marker = if is_hud_suspect { " [SUSPECT]" } else { "" }; + + let line = format!( + "frame={:04} draw={:05} vert={} frag={} verts={:<5} blend={:<5} depth={}{}\n", + frame, + draw, + vert_str, + frag_str, + vert_count, + blend_enabled, + depth_test_enabled, + suspect_marker + ); + + let _ = file.write_all(line.as_bytes()); + let _ = file.flush(); +} + +pub unsafe fn record_draw(cb: vk::CommandBuffer, vert_count: u32) { + if !is_discovery_mode() { + return; + } + + let cb_key = cb.as_raw(); + let device_key = match CB_STATE.get(&cb_key) { + Some(r) => r.device_key, + None => return, + }; + + let ds = match DEVICE_STATE.get(&device_key) { + Some(s) => s, + None => return, + }; + + let frame = ds.frame_counter.load(std::sync::atomic::Ordering::Relaxed); + + let (vert_hash, frag_hash, blend_enabled, depth_test_enabled) = { + let state = match CB_STATE.get_mut(&cb_key) { + Some(s) => s, + None => return, + }; + let draw = state.draw_counter + 1; + + let vert_hash = state.active_vert_hash; + let frag_hash = state.active_frag_hash; + + let (blend_enabled, depth_test_enabled) = if let Some(active_frag) = frag_hash { + let mut blend = false; + let mut depth = false; + for entry in ds.pipeline_state.iter() { + if let Some(ph) = ds.pipeline_registry.get(entry.key()) { + if ph.frag_hash == Some(active_frag) { + blend = entry.value().blend_enabled; + depth = entry.value().depth_test_enabled; + break; + } + } + } + (blend, depth) + } else { + (false, false) + }; + + drop(state); + + log_draw( + frame, + draw, + vert_hash, + frag_hash, + vert_count, + blend_enabled, + depth_test_enabled, + ); + + (vert_hash, frag_hash, blend_enabled, depth_test_enabled) + }; +} diff --git a/apps/nescapture/src/dispatch.rs b/apps/nescapture/src/dispatch.rs new file mode 100644 index 00000000..b33af6b3 --- /dev/null +++ b/apps/nescapture/src/dispatch.rs @@ -0,0 +1,363 @@ +// ───────────────────────────────────────────────────────────────────────────── +// dispatch.rs — raw Vulkan function-pointer type aliases and dispatch tables +// ───────────────────────────────────────────────────────────────────────────── + +use ash::vk; +use std::os::raw::c_char; + +pub type RawFn = unsafe extern "system" fn(); + +// ── Instance-level ──────────────────────────────────────────────────────────── + +pub type PFN_vkGetInstanceProcAddr = + unsafe extern "system" fn(vk::Instance, *const c_char) -> Option; +pub type PFN_vkGetDeviceProcAddr = + unsafe extern "system" fn(vk::Device, *const c_char) -> Option; + +pub type PFN_vkCreateInstance = unsafe extern "system" fn( + *const vk::InstanceCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Instance, +) -> vk::Result; +pub type PFN_vkDestroyInstance = + unsafe extern "system" fn(vk::Instance, *const vk::AllocationCallbacks); + +pub type PFN_vkGetPhysicalDeviceMemoryProperties = + unsafe extern "system" fn(vk::PhysicalDevice, *mut vk::PhysicalDeviceMemoryProperties); + +pub type PFN_vkCreateDevice = unsafe extern "system" fn( + vk::PhysicalDevice, + *const vk::DeviceCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Device, +) -> vk::Result; + +// ── Device infrastructure ───────────────────────────────────────────────────── + +pub type PFN_vkDestroyDevice = + unsafe extern "system" fn(vk::Device, *const vk::AllocationCallbacks); + +pub type PFN_vkGetDeviceQueue = unsafe extern "system" fn(vk::Device, u32, u32, *mut vk::Queue); + +pub type PFN_vkQueuePresentKHR = + unsafe extern "system" fn(vk::Queue, *const vk::PresentInfoKHR) -> vk::Result; + +// ── Phase 1: Shader modules ─────────────────────────────────────────────────── + +pub type PFN_vkCreateShaderModule = unsafe extern "system" fn( + vk::Device, + *const vk::ShaderModuleCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::ShaderModule, +) -> vk::Result; + +pub type PFN_vkDestroyShaderModule = + unsafe extern "system" fn(vk::Device, vk::ShaderModule, *const vk::AllocationCallbacks); + +// ── Phase 1: Graphics pipelines ────────────────────────────────────────────── + +pub type PFN_vkCreateGraphicsPipelines = unsafe extern "system" fn( + vk::Device, + vk::PipelineCache, + u32, + *const vk::GraphicsPipelineCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Pipeline, +) -> vk::Result; + +pub type PFN_vkDestroyPipeline = + unsafe extern "system" fn(vk::Device, vk::Pipeline, *const vk::AllocationCallbacks); + +// ── Phase 2: Image views and framebuffers ───────────────────────────────────── + +pub type PFN_vkCreateImageView = unsafe extern "system" fn( + vk::Device, + *const vk::ImageViewCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::ImageView, +) -> vk::Result; + +pub type PFN_vkDestroyImageView = + unsafe extern "system" fn(vk::Device, vk::ImageView, *const vk::AllocationCallbacks); + +pub type PFN_vkCreateFramebuffer = unsafe extern "system" fn( + vk::Device, + *const vk::FramebufferCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Framebuffer, +) -> vk::Result; + +pub type PFN_vkDestroyFramebuffer = + unsafe extern "system" fn(vk::Device, vk::Framebuffer, *const vk::AllocationCallbacks); + +pub type PFN_vkAllocateCommandBuffers = unsafe extern "system" fn( + vk::Device, + *const vk::CommandBufferAllocateInfo, + *mut vk::CommandBuffer, +) -> vk::Result; + +pub type PFN_vkFreeCommandBuffers = + unsafe extern "system" fn(vk::Device, vk::CommandPool, u32, *const vk::CommandBuffer); + +// ── Phase 2: Render pass and rendering ─────────────────────────────────────── + +pub type PFN_vkCmdBindPipeline = + unsafe extern "system" fn(vk::CommandBuffer, vk::PipelineBindPoint, vk::Pipeline); + +pub type PFN_vkCmdBeginRenderPass = unsafe extern "system" fn( + vk::CommandBuffer, + *const vk::RenderPassBeginInfo, + vk::SubpassContents, +); + +pub type PFN_vkCmdEndRenderPass = unsafe extern "system" fn(vk::CommandBuffer); + +pub type PFN_vkCmdBeginRenderingKHR = + unsafe extern "system" fn(vk::CommandBuffer, *const vk::RenderingInfo); + +pub type PFN_vkCmdEndRenderingKHR = unsafe extern "system" fn(vk::CommandBuffer); + +// ── Phase 4: Capture images and memory ─────────────────────────────────────── + +pub type PFN_vkCreateImage = unsafe extern "system" fn( + vk::Device, + *const vk::ImageCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Image, +) -> vk::Result; + +pub type PFN_vkDestroyImage = + unsafe extern "system" fn(vk::Device, vk::Image, *const vk::AllocationCallbacks); + +pub type PFN_vkAllocateMemory = unsafe extern "system" fn( + vk::Device, + *const vk::MemoryAllocateInfo, + *const vk::AllocationCallbacks, + *mut vk::DeviceMemory, +) -> vk::Result; + +pub type PFN_vkFreeMemory = + unsafe extern "system" fn(vk::Device, vk::DeviceMemory, *const vk::AllocationCallbacks); + +pub type PFN_vkBindImageMemory = unsafe extern "system" fn( + vk::Device, + vk::Image, + vk::DeviceMemory, + vk::DeviceSize, +) -> vk::Result; + +pub type PFN_vkGetImageMemoryRequirements = + unsafe extern "system" fn(vk::Device, vk::Image, *mut vk::MemoryRequirements); + +pub type PFN_vkMapMemory = unsafe extern "system" fn( + vk::Device, + vk::DeviceMemory, + vk::DeviceSize, + vk::DeviceSize, + vk::MemoryMapFlags, + *mut *mut std::os::raw::c_void, +) -> vk::Result; + +pub type PFN_vkUnmapMemory = unsafe extern "system" fn(vk::Device, vk::DeviceMemory); + +pub type PFN_vkCmdPipelineBarrier = unsafe extern "system" fn( + vk::CommandBuffer, + vk::PipelineStageFlags, + vk::PipelineStageFlags, + vk::DependencyFlags, + u32, + *const vk::MemoryBarrier, + u32, + *const vk::BufferMemoryBarrier, + u32, + *const vk::ImageMemoryBarrier, +); + +pub type PFN_vkCmdCopyImage = unsafe extern "system" fn( + vk::CommandBuffer, + vk::Image, + vk::ImageLayout, + vk::Image, + vk::ImageLayout, + u32, + *const vk::ImageCopy, +); + +pub type PFN_vkGetImageSubresourceLayout = unsafe extern "system" fn( + vk::Device, + vk::Image, + *const vk::ImageSubresource, + *mut vk::SubresourceLayout, +); + +// DMA-BUF fd export (used to share final_image with pixelforge zero-copy) +pub type PFN_vkGetMemoryFdKHR = unsafe extern "system" fn( + vk::Device, + *const vk::MemoryGetFdInfoKHR, + *mut std::os::raw::c_int, +) -> vk::Result; + +// ── Phase 4: Synchronisation ───────────────────────────────────────────────── + +pub type PFN_vkCreateFence = unsafe extern "system" fn( + vk::Device, + *const vk::FenceCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::Fence, +) -> vk::Result; + +pub type PFN_vkDestroyFence = + unsafe extern "system" fn(vk::Device, vk::Fence, *const vk::AllocationCallbacks); + +pub type PFN_vkCreateCommandPool = unsafe extern "system" fn( + vk::Device, + *const vk::CommandPoolCreateInfo, + *const vk::AllocationCallbacks, + *mut vk::CommandPool, +) -> vk::Result; + +pub type PFN_vkDestroyCommandPool = + unsafe extern "system" fn(vk::Device, vk::CommandPool, *const vk::AllocationCallbacks); + +pub type PFN_vkResetCommandPool = + unsafe extern "system" fn(vk::Device, vk::CommandPool, vk::CommandPoolResetFlags) -> vk::Result; + +pub type PFN_vkBeginCommandBuffer = + unsafe extern "system" fn(vk::CommandBuffer, *const vk::CommandBufferBeginInfo) -> vk::Result; + +pub type PFN_vkEndCommandBuffer = unsafe extern "system" fn(vk::CommandBuffer) -> vk::Result; + +pub type PFN_vkQueueSubmit = + unsafe extern "system" fn(vk::Queue, u32, *const vk::SubmitInfo, vk::Fence) -> vk::Result; + +pub type PFN_vkWaitForFences = + unsafe extern "system" fn(vk::Device, u32, *const vk::Fence, vk::Bool32, u64) -> vk::Result; + +pub type PFN_vkResetFences = + unsafe extern "system" fn(vk::Device, u32, *const vk::Fence) -> vk::Result; + +// ── Phase 4: Swapchain tracking ────────────────────────────────────────────── + +pub type PFN_vkCreateSwapchainKHR = unsafe extern "system" fn( + vk::Device, + *const vk::SwapchainCreateInfoKHR, + *const vk::AllocationCallbacks, + *mut vk::SwapchainKHR, +) -> vk::Result; + +pub type PFN_vkDestroySwapchainKHR = + unsafe extern "system" fn(vk::Device, vk::SwapchainKHR, *const vk::AllocationCallbacks); + +pub type PFN_vkGetSwapchainImagesKHR = + unsafe extern "system" fn(vk::Device, vk::SwapchainKHR, *mut u32, *mut vk::Image) -> vk::Result; + +// ── Phase 6: Draw commands ─────────────────────────────────────────────────── + +pub type PFN_vkCmdDraw = unsafe extern "system" fn(vk::CommandBuffer, u32, u32, u32, u32); +pub type PFN_vkCmdDrawIndexed = + unsafe extern "system" fn(vk::CommandBuffer, u32, u32, u32, i32, u32); +pub type PFN_vkCmdDrawIndirect = + unsafe extern "system" fn(vk::CommandBuffer, vk::Buffer, vk::DeviceSize, u32, u32); +pub type PFN_vkCmdDrawIndexedIndirect = + unsafe extern "system" fn(vk::CommandBuffer, vk::Buffer, vk::DeviceSize, u32, u32); +pub type PFN_vkCmdDrawIndirectCount = unsafe extern "system" fn( + vk::CommandBuffer, + vk::Buffer, + vk::DeviceSize, + vk::Buffer, + vk::DeviceSize, + u32, + u32, +); +pub type PFN_vkCmdDrawIndexedIndirectCount = unsafe extern "system" fn( + vk::CommandBuffer, + vk::Buffer, + vk::DeviceSize, + vk::Buffer, + vk::DeviceSize, + u32, + u32, +); + +pub type PFN_vkResetCommandBuffer = + unsafe extern "system" fn(vk::CommandBuffer, vk::CommandBufferResetFlags) -> vk::Result; + +// ── Dispatch table structs ──────────────────────────────────────────────────── + +pub struct NextInstanceFn { + pub get_instance_proc_addr: PFN_vkGetInstanceProcAddr, + pub destroy_instance: PFN_vkDestroyInstance, + pub get_physical_device_memory_properties: PFN_vkGetPhysicalDeviceMemoryProperties, + pub create_device: PFN_vkCreateDevice, +} + +#[derive(Clone, Copy)] +pub struct NextDeviceFn { + // Infrastructure + pub get_device_proc_addr: PFN_vkGetDeviceProcAddr, + pub destroy_device: PFN_vkDestroyDevice, + pub get_device_queue: PFN_vkGetDeviceQueue, + pub queue_present_khr: Option, + + // Phase 1 + pub create_shader_module: PFN_vkCreateShaderModule, + pub destroy_shader_module: PFN_vkDestroyShaderModule, + pub create_graphics_pipelines: PFN_vkCreateGraphicsPipelines, + pub destroy_pipeline: PFN_vkDestroyPipeline, + + // Phase 2 + pub create_image_view: PFN_vkCreateImageView, + pub destroy_image_view: PFN_vkDestroyImageView, + pub create_framebuffer: PFN_vkCreateFramebuffer, + pub destroy_framebuffer: PFN_vkDestroyFramebuffer, + pub allocate_command_buffers: PFN_vkAllocateCommandBuffers, + pub free_command_buffers: PFN_vkFreeCommandBuffers, + pub cmd_bind_pipeline: PFN_vkCmdBindPipeline, + pub cmd_begin_render_pass: PFN_vkCmdBeginRenderPass, + pub cmd_end_render_pass: PFN_vkCmdEndRenderPass, + pub cmd_begin_rendering_khr: Option, + pub cmd_end_rendering_khr: Option, + + // Phase 4 — capture images + pub create_image: PFN_vkCreateImage, + pub destroy_image: PFN_vkDestroyImage, + pub allocate_memory: PFN_vkAllocateMemory, + pub free_memory: PFN_vkFreeMemory, + pub bind_image_memory: PFN_vkBindImageMemory, + pub get_image_memory_requirements: PFN_vkGetImageMemoryRequirements, + pub map_memory: PFN_vkMapMemory, + pub unmap_memory: PFN_vkUnmapMemory, + pub cmd_pipeline_barrier: PFN_vkCmdPipelineBarrier, + pub cmd_copy_image: PFN_vkCmdCopyImage, + pub get_image_subresource_layout: PFN_vkGetImageSubresourceLayout, + /// `None` when `VK_KHR_external_memory_fd` is unavailable. + /// Required for DMA-BUF export to pixelforge's VkDevice. + pub get_memory_fd_khr: Option, + + // Phase 4 — synchronisation + pub create_fence: PFN_vkCreateFence, + pub destroy_fence: PFN_vkDestroyFence, + pub create_command_pool: PFN_vkCreateCommandPool, + pub destroy_command_pool: PFN_vkDestroyCommandPool, + pub reset_command_pool: PFN_vkResetCommandPool, + pub begin_command_buffer: PFN_vkBeginCommandBuffer, + pub end_command_buffer: PFN_vkEndCommandBuffer, + pub queue_submit: PFN_vkQueueSubmit, + pub wait_for_fences: PFN_vkWaitForFences, + pub reset_fences: PFN_vkResetFences, + + // Phase 4 — swapchain + pub create_swapchain_khr: Option, + pub destroy_swapchain_khr: Option, + pub get_swapchain_images_khr: Option, + + // Phase 6 — draw commands + pub cmd_draw: PFN_vkCmdDraw, + pub cmd_draw_indexed: PFN_vkCmdDrawIndexed, + pub cmd_draw_indirect: PFN_vkCmdDrawIndirect, + pub cmd_draw_indexed_indirect: PFN_vkCmdDrawIndexedIndirect, + pub cmd_draw_indirect_count: Option, + pub cmd_draw_indexed_indirect_count: Option, + + pub reset_command_buffer: PFN_vkResetCommandBuffer, +} diff --git a/apps/nescapture/src/dmabuf_import.rs b/apps/nescapture/src/dmabuf_import.rs new file mode 100644 index 00000000..1d42394a --- /dev/null +++ b/apps/nescapture/src/dmabuf_import.rs @@ -0,0 +1,242 @@ +//! DMA-BUF import support for zero-copy video encoding. +//! +//! This module provides the ability to import Linux DMA-BUF file descriptors as +//! Vulkan images for direct video encoding without CPU-side copies. +//! +//! `DmaBufImporter` caches imported Vulkan resources per compositor buffer index +//! so that pre-allocated GBM buffers are imported only once. Subsequent frames +//! from the same buffer reuse the cached `VkImage` and `VkDeviceMemory`, +//! eliminating per-frame Vulkan object creation and layout transitions. + +use anyhow::Result; +use ash::vk; +use log::debug; +use pixelforge::VideoContext; +use std::os::fd::RawFd; +use std::os::unix::io::{BorrowedFd, IntoRawFd}; + +/// Information about a single DMA-BUF plane. +#[derive(Debug, Clone, Copy)] +pub struct DmaBufPlane { + /// File descriptor for the DMA-BUF. + pub fd: RawFd, + /// Offset within the DMA-BUF to the start of this plane. + pub offset: u32, + /// Row stride in bytes. + pub stride: u32, + /// DRM format modifier. + pub modifier: u64, +} + +/// Cached Vulkan resources for a single compositor buffer slot. +struct CachedImport { + image: vk::Image, + memory: vk::DeviceMemory, +} + +/// Importer for DMA-BUF file descriptors into Vulkan images. +/// +/// Owns a per-buffer-index cache of `VkImage` + `VkDeviceMemory`. +/// Layout transitions are deferred to the consumer (e.g. `ColorConverter`) +/// to avoid a separate GPU submission per first-time import. +pub struct DmaBufImporter { + context: VideoContext, + external_memory_fd: ash::khr::external_memory_fd::Device, + /// Per-buffer-index cache. Index corresponds to `ExportedFrame::buffer_index`. + cached_imports: Vec>, +} + +impl DmaBufImporter { + /// Create a new DMA-BUF importer. + pub fn new(context: VideoContext) -> Result { + let external_memory_fd = + ash::khr::external_memory_fd::Device::load(context.instance(), context.device()); + + Ok(Self { + context, + external_memory_fd, + cached_imports: Vec::new(), + }) + } + + /// Import a DMA-BUF as a Vulkan image, reusing a cached import when + /// the same `buffer_index` has been seen before. + /// + /// The `format` parameter specifies the Vulkan format matching the DMA-BUF + /// pixel format (e.g. `B8G8R8A8_UNORM` for SDR, `A2B10G10R10_UNORM_PACK32` + /// for 10-bit HDR, `R16G16B16A16_SFLOAT` for FP16 HDR). + /// + /// Returns `(image, needs_transition)` where `needs_transition` is `true` + /// for first-time imports whose image is still in `UNDEFINED` layout. + /// The caller is responsible for transitioning the image (e.g. by passing + /// the appropriate `src_layout` to `ColorConverter::convert`). + pub fn import_or_reuse( + &mut self, + buffer_index: usize, + width: u32, + height: u32, + format: vk::Format, + planes: &[DmaBufPlane], + ) -> Result<(vk::Image, bool)> { + // Grow the cache vector if needed. + if self.cached_imports.len() <= buffer_index { + self.cached_imports.resize_with(buffer_index + 1, || None); + } + + if let Some(cached) = &self.cached_imports[buffer_index] { + return Ok((cached.image, false)); + } + + // First time seeing this buffer — full import. + debug!( + "First import for buffer {buffer_index}: {}x{}, format={:?}, fd={}, stride={}, modifier={:#x}", + width, height, format, planes[0].fd, planes[0].stride, planes[0].modifier + ); + + let (image, memory) = self.import_internal(width, height, format, planes)?; + + self.cached_imports[buffer_index] = Some(CachedImport { image, memory }); + Ok((image, true)) + } + + /// Perform the raw Vulkan import of a DMA-BUF with the specified format. + /// + /// Returns the `(VkImage, VkDeviceMemory)` pair. The image is in + /// `UNDEFINED` layout; the caller must transition it. + fn import_internal( + &self, + width: u32, + height: u32, + format: vk::Format, + planes: &[DmaBufPlane], + ) -> Result<(vk::Image, vk::DeviceMemory)> { + if planes.is_empty() { + return Err(anyhow::anyhow!("At least one DMA-BUF plane is required")); + } + + let device = self.context.device(); + + // Build DRM format modifier plane layouts for all planes. + // AMD modifiers (e.g. tiled/DCC) may require multiple planes; + // the layout count must match the modifier's expected plane count. + let plane_layouts: Vec = planes + .iter() + .map(|p| { + vk::SubresourceLayout::default() + .offset(p.offset as u64) + .row_pitch(p.stride as u64) + }) + .collect(); + + let modifier = planes[0].modifier; + let mut drm_format_modifier_info = + vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default() + .drm_format_modifier(modifier) + .plane_layouts(&plane_layouts); + + let mut external_memory_info = vk::ExternalMemoryImageCreateInfo::default() + .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); + external_memory_info.p_next = &mut drm_format_modifier_info + as *mut vk::ImageDrmFormatModifierExplicitCreateInfoEXT + as *mut _; + + let mut image_create_info = vk::ImageCreateInfo::default() + .image_type(vk::ImageType::TYPE_2D) + .format(format) + .extent(vk::Extent3D { + width, + height, + depth: 1, + }) + .mip_levels(1) + .array_layers(1) + .samples(vk::SampleCountFlags::TYPE_1) + .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) + .usage(vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::SAMPLED) + .sharing_mode(vk::SharingMode::EXCLUSIVE) + .initial_layout(vk::ImageLayout::UNDEFINED); + image_create_info.p_next = + &mut external_memory_info as *mut vk::ExternalMemoryImageCreateInfo as *mut _; + + let image = unsafe { device.create_image(&image_create_info, None) } + .map_err(|e| anyhow::anyhow!("DMA-BUF image creation: {e}"))?; + + // Memory requirements. + let mem_requirements = unsafe { device.get_image_memory_requirements(image) }; + + // FD memory properties. + let mut memory_fd_properties = vk::MemoryFdPropertiesKHR::default(); + unsafe { + self.external_memory_fd.get_memory_fd_properties( + vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, + planes[0].fd, + &mut memory_fd_properties, + ) + } + .map_err(|e| anyhow::anyhow!("Failed to get memory FD properties: {e}"))?; + + // Duplicate the FD — vkAllocateMemory consumes it. + let fd = unsafe { BorrowedFd::borrow_raw(planes[0].fd) } + .try_clone_to_owned() + .map_err(|e| anyhow::anyhow!("Failed to duplicate DMA-BUF FD: {e}"))? + .into_raw_fd(); + + let mut import_memory_fd_info = vk::ImportMemoryFdInfoKHR::default() + .handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT) + .fd(fd); + + let memory_type_bits = + mem_requirements.memory_type_bits & memory_fd_properties.memory_type_bits; + + debug!( + "Memory allocation: size={}, image_type_bits={:#x}, fd_type_bits={:#x}, combined={:#x}", + mem_requirements.size, + mem_requirements.memory_type_bits, + memory_fd_properties.memory_type_bits, + memory_type_bits + ); + + let memory_type_index = self + .context + .find_memory_type(memory_type_bits, vk::MemoryPropertyFlags::empty()) + .ok_or_else(|| anyhow::anyhow!("No suitable memory type for DMA-BUF import"))?; + + // Dedicated allocation (required by many drivers for external memory). + let mut dedicated_alloc_info = vk::MemoryDedicatedAllocateInfo::default().image(image); + import_memory_fd_info.p_next = + &mut dedicated_alloc_info as *mut vk::MemoryDedicatedAllocateInfo as *mut _; + + let mut alloc_info = vk::MemoryAllocateInfo::default() + .allocation_size(mem_requirements.size) + .memory_type_index(memory_type_index); + alloc_info.p_next = &mut import_memory_fd_info as *mut vk::ImportMemoryFdInfoKHR as *mut _; + + let memory = unsafe { device.allocate_memory(&alloc_info, None) }.map_err(|e| { + unsafe { device.destroy_image(image, None) }; + anyhow::anyhow!("DMA-BUF memory import: {e}") + })?; + + if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } { + unsafe { + device.free_memory(memory, None); + device.destroy_image(image, None); + } + return Err(anyhow::anyhow!("DMA-BUF memory bind: {e}")); + } + + Ok((image, memory)) + } +} + +impl Drop for DmaBufImporter { + fn drop(&mut self) { + let device = self.context.device(); + unsafe { + // Clean up cached imports. + for cached in self.cached_imports.drain(..).flatten() { + device.destroy_image(cached.image, None); + device.free_memory(cached.memory, None); + } + } + } +} diff --git a/apps/nescapture/src/encode.rs b/apps/nescapture/src/encode.rs new file mode 100644 index 00000000..0349a97f --- /dev/null +++ b/apps/nescapture/src/encode.rs @@ -0,0 +1,1105 @@ +// ───────────────────────────────────────────────────────────────────────────── +// encode.rs — Vulkan Video hardware encoding + IPC transmission to nestri-guest-hub +// +// ┌─────────────────── Zero-copy GPU pipeline ──────────────────────────────┐ +// │ │ +// │ Game VkDevice (intercepted by nescapture layer) │ +// │ vkCmdCopyImage(swapchain → final_image) ← GPU, no CPU │ +// │ get_dmabuf_fd(final_memory) ← export fd │ +// │ │ +// │ pixelforge VkDevice (separate, video-encode queue) │ +// │ DmaBufImporter::import_or_reuse(fd, ...) ← import as vk::Image │ +// │ ColorConverter::convert(bgra_img, ← GPU compute shader │ +// │ encoder.input_image()) BGRA/RGB10/FP16 │ +// │ → NV12/P010/YUV444 │ +// │ Encoder::encode(encoder.input_image()) ← Vulkan Video encode │ +// │ IPC send to nestri-guest-hub │ +// └─────────────────────────────────────────────────────────────────────────┘ +// +// Environment variables +// ────────────────────── +// NESCAPTURE_CODEC "h264" | "h265" | "av1" (default: best available) +// NESCAPTURE_FORMAT "yuv420" | "yuv444" (default: yuv420) +// NESCAPTURE_DEPTH "8" | "10" (default: auto from VkFormat) +// NESCAPTURE_BITRATE CBR target kbps (default: 10000) +// NESCAPTURE_QP Constant QP (overrides BITRATE) (default: unset) +// NESCAPTURE_FPS Frame rate (default: 60) +// NESCAPTURE_IDR_INTERVAL Force IDR every N seconds (default: 4) +// NESCAPTURE_TUNE "highquality" | "lowlatency" | "ultralowlatency" | "lossless" (default: unset) +// NESCAPTURE_IPC_PATH Unix socket path for hub IPC (default: /tmp/nestri-video.sock) +// ───────────────────────────────────────────────────────────────────────────── + +use anyhow::Result; +use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::unix::net::UnixDatagram; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering}; +use std::sync::{Arc, mpsc}; +use std::thread; +use std::time::Instant; + +use nesprotocol::{ + CODEC_AV1, CODEC_H264, CODEC_H265, CODEC_KEEP, FLAG_KEYFRAME, FLAG_RECONFIG, + MSG_ENCODE_SETTINGS, MSG_IDR_REQUEST, STREAM_VIDEO, decode_encode_settings, encode_ipc_frame, +}; +use pixelforge::{ + Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorSpace, EncodeBitDepth, + EncodeConfig, EncodeContentHint, EncodeFuture, EncodeUsageHint, Encoder, EncoderTuningMode, + InputFormat, OutputFormat, PixelFormat, RateControlMode, VideoContextBuilder, +}; + +use crate::dmabuf_import::{DmaBufImporter, DmaBufPlane}; + +// ── VkColorSpaceKHR constants (raw values, matches ash/Vulkan spec) ─────────── +const VK_COLOR_SPACE_SRGB_NONLINEAR_KHR: u32 = 0; +const VK_COLOR_SPACE_HDR10_ST2084_EXT: u32 = 1_000_104_002; +const VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT: u32 = 1_000_104_003; +const VK_COLOR_SPACE_BT2020_LINEAR_EXT: u32 = 1_000_104_007; +const VK_COLOR_SPACE_DOLBYVISION_EXT: u32 = 1_000_104_009; +const VK_COLOR_SPACE_HDR10_HLG_EXT: u32 = 1_000_104_010; + +pub fn vk_format_to_input_format(vk_format: u32) -> Option { + match vk_format { + 44..=50 => Some(InputFormat::BGRA), + 37..=43 => Some(InputFormat::RGBA), + 64 => Some(InputFormat::ABGR2101010), + 97 => Some(InputFormat::RGBA16F), + other => { + log::warn!("unsupported VkFormat {other} for color conversion — defaulting to BGRA"); + Some(InputFormat::BGRA) + } + } +} + +pub fn vk_colorspace_to_color_space(vk_colorspace: u32) -> ColorSpace { + match vk_colorspace { + VK_COLOR_SPACE_HDR10_ST2084_EXT + | VK_COLOR_SPACE_DOLBYVISION_EXT + | VK_COLOR_SPACE_HDR10_HLG_EXT => ColorSpace::Bt2020, + + VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT | VK_COLOR_SPACE_BT2020_LINEAR_EXT => { + ColorSpace::SrgbToBt2020Pq + } + + _ => ColorSpace::Bt709, + } +} + +pub fn vk_format_to_bit_depth(vk_format: u32) -> EncodeBitDepth { + match vk_format { + 64 | 58 | 97 => EncodeBitDepth::Ten, + _ => EncodeBitDepth::Eight, + } +} + +pub fn vk_colorspace_to_color_description(vk_colorspace: u32) -> Option { + match vk_colorspace { + VK_COLOR_SPACE_HDR10_ST2084_EXT + | VK_COLOR_SPACE_DOLBYVISION_EXT + | VK_COLOR_SPACE_HDR10_HLG_EXT => Some(ColorDescription::bt2020_pq()), + _ => Some(ColorDescription::bt709()), + } +} + +pub fn output_format(pixel_fmt: PixelFormat, bit_depth: EncodeBitDepth) -> OutputFormat { + match (pixel_fmt, bit_depth) { + (PixelFormat::Yuv420, EncodeBitDepth::Eight) => OutputFormat::NV12, + (PixelFormat::Yuv420, EncodeBitDepth::Ten) => OutputFormat::P010, + (PixelFormat::Yuv444, EncodeBitDepth::Eight) => OutputFormat::YUV444, + (PixelFormat::Yuv444, EncodeBitDepth::Ten) => OutputFormat::YUV444P10, + _ => OutputFormat::NV12, + } +} + +// ── Captured frame (sent from present.rs to encoder thread) ────────────────── + +pub struct CapturedFrame { + pub source: FrameSource, + pub width: u32, + pub height: u32, + pub vk_format: u32, + pub vk_colorspace: u32, +} + +pub enum FrameSource { + DmaBuf { + fd: RawFd, + stride: u32, + modifier: u64, + }, + Pixels(Vec), +} +impl Drop for FrameSource { + fn drop(&mut self) { + if let FrameSource::DmaBuf { fd, .. } = self { + if *fd >= 0 { + unsafe { libc::close(*fd) }; + } + } + } +} + +// ── Codec probing ───────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HwCodec { + H264, + H265, + AV1, +} + +impl HwCodec { + fn to_pixelforge(self) -> Codec { + match self { + Self::H264 => Codec::H264, + Self::H265 => Codec::H265, + Self::AV1 => Codec::AV1, + } + } + + fn to_protocol_codec(self) -> u8 { + match self { + Self::H264 => CODEC_H264, + Self::H265 => CODEC_H265, + Self::AV1 => CODEC_AV1, + } + } +} + +fn probe_any() -> Option<(HwCodec, pixelforge::VideoContext)> { + probe_specific(HwCodec::AV1) + .or_else(|| probe_specific(HwCodec::H265)) + .or_else(|| probe_specific(HwCodec::H264)) +} + +fn probe_specific(codec: HwCodec) -> Option<(HwCodec, pixelforge::VideoContext)> { + let ctx = VideoContextBuilder::new() + .app_name("nescapture") + .enable_validation(false) + .require_encode(codec.to_pixelforge()) + .build() + .ok()?; + if ctx.supports_encode(codec.to_pixelforge()) { + log::info!("hardware {:?} encode available", codec); + Some((codec, ctx)) + } else { + None + } +} + +fn resolve_codec(requested: Option<&str>) -> Option<(HwCodec, pixelforge::VideoContext)> { + match requested { + Some("av1") => probe_specific(HwCodec::AV1).or_else(|| { + log::warn!("AV1 unavailable — falling back to H.264"); + probe_specific(HwCodec::H264) + }), + Some("h265" | "hevc") => probe_specific(HwCodec::H265).or_else(|| { + log::warn!("H.265 unavailable — falling back to H.264"); + probe_specific(HwCodec::H264) + }), + Some("h264" | "avc") => probe_specific(HwCodec::H264), + Some(other) => { + log::warn!("unknown NESCAPTURE_CODEC={other} — probing best available"); + probe_any() + } + None => probe_any(), + } +} + +// ── Pipeline config ─────────────────────────────────────────────────────────── + +pub struct PipelineConfig { + pub width: u32, + pub height: u32, + pub fps: u32, + pub bitrate_kbps: Option, + pub qp: Option, + pub idr_interval: u32, + pub encoder_tuning_mode: EncoderTuningMode, + pub pixel_format: PixelFormat, + pub codec_request: Option, + pub ipc_path: std::path::PathBuf, + pub physical_device: Option, +} +impl PipelineConfig { + pub fn from_env(width: u32, height: u32) -> Option { + let pixel_format = match std::env::var("NESCAPTURE_FORMAT").as_deref() { + Ok("yuv444") => PixelFormat::Yuv444, + _ => PixelFormat::Yuv420, + }; + + let ipc_path = std::env::var("NESCAPTURE_IPC_PATH") + .unwrap_or_else(|_| "/tmp/nestri-video.sock".to_string()) + .into(); + + let mut bitrate: Option = None; + if std::env::var("NESCAPTURE_QP").is_err() { + bitrate = Some(env_u64("NESCAPTURE_BITRATE", 10_000) as u32); + } + + let encoder_tuning_mode = match std::env::var("NESCAPTURE_TUNE").as_deref() { + Ok("highquality") => EncoderTuningMode::HighQuality, + Ok("lowlatency") => EncoderTuningMode::LowLatency, + Ok("ultralowlatency") => EncoderTuningMode::UltraLowLatency, + Ok("lossless") => EncoderTuningMode::Lossless, + _ => EncoderTuningMode::Default, + }; + + Some(Self { + width, + height, + fps: env_u64("NESCAPTURE_FPS", 60) as u32, + bitrate_kbps: bitrate, + qp: std::env::var("NESCAPTURE_QP") + .ok() + .and_then(|s| s.parse().ok()), + idr_interval: (env_u64("NESCAPTURE_FPS", 60) * env_u64("NESCAPTURE_IDR_INTERVAL", 4)) + as u32, + encoder_tuning_mode, + pixel_format, + codec_request: std::env::var("NESCAPTURE_CODEC").ok(), + ipc_path, + physical_device: None, + }) + } +} + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(default) +} + +// ── Pipeline handle ─────────────────────────────────────────────────────────── + +pub struct PipelineHandle { + frame_tx: mpsc::SyncSender, + idr_requested: Arc, + shutdown: Arc, + pub codec: HwCodec, + pub capture_fps: Arc, + pub encode_avg_ms: Arc, + pub capture_ms: Arc, + pub dropped_frames: Arc, + pub present_attempts: Arc, + pub capture_attempts: Arc, +} + +impl PipelineHandle { + pub fn new(config: PipelineConfig) -> Result { + let (codec, ctx) = resolve_codec(config.codec_request.as_deref()) + .ok_or_else(|| "no hardware video encoder found on this GPU".to_string())?; + + let (frame_tx, frame_rx) = mpsc::sync_channel::(2); + let (encoded_tx, encoded_rx) = mpsc::sync_channel::(2); + let (reconfig_tx, reconfig_rx) = mpsc::channel::(); + let shutdown = Arc::new(AtomicBool::new(false)); + let idr_requested = Arc::new(AtomicBool::new(false)); + let capture_fps = Arc::new(AtomicU32::new(0)); + let encode_avg_ms = Arc::new(AtomicU32::new(0)); + let capture_ms = Arc::new(AtomicU32::new(0)); + let dropped_frames = Arc::new(AtomicU32::new(0)); + let present_attempts = Arc::new(AtomicU32::new(0)); + let capture_attempts = Arc::new(AtomicU32::new(0)); + let current_codec = Arc::new(AtomicU8::new(codec.to_protocol_codec())); + let needs_reconfig_flag = Arc::new(AtomicBool::new(false)); + + let enc_shutdown = shutdown.clone(); + let ipc_shutdown = shutdown.clone(); + + let enc_cfg = EncoderConfig { + width: config.width, + height: config.height, + fps: config.fps, + bitrate_kbps: config.bitrate_kbps, + qp: config.qp, + idr_interval: config.idr_interval, + encoder_tuning_mode: config.encoder_tuning_mode, + pixel_format: config.pixel_format, + codec, + ctx, + idr_requested: idr_requested.clone(), + reconfig_rx, + current_codec: current_codec.clone(), + wanted_depth_override: None, + needs_reconfig_flag: needs_reconfig_flag.clone(), + }; + thread::Builder::new() + .name("nescapture-encoder".into()) + .spawn(move || encoder_thread(enc_cfg, frame_rx, encoded_tx, enc_shutdown)) + .map_err(|e| format!("spawn encoder: {e}"))?; + + let ipc_path = config.ipc_path.clone(); + let ipc_cfg = IpcConfig { + ipc_path: config.ipc_path, + current_codec: current_codec.clone(), + needs_reconfig_flag: needs_reconfig_flag, + width: config.width as u16, + height: config.height as u16, + encode_ms: encode_avg_ms.clone(), + }; + thread::Builder::new() + .name("nescapture-ipc".into()) + .spawn(move || ipc_send_thread(ipc_cfg, encoded_rx, ipc_shutdown)) + .map_err(|e| format!("spawn ipc: {e}"))?; + + // Spawn periodic stats sender + let stats_ipc = ipc_path.with_file_name("nestri-stats.sock"); + let stats_cap_fps = capture_fps.clone(); + let stats_enc_ms = encode_avg_ms.clone(); + let stats_cap_ms = capture_ms.clone(); + let stats_drop = dropped_frames.clone(); + let stats_shutdown = shutdown.clone(); + let pa = present_attempts.clone(); + let ca = capture_attempts.clone(); + thread::Builder::new() + .name("nescapture-stats".into()) + .spawn(move || { + stats_sender_thread( + stats_cap_fps, + stats_enc_ms, + stats_cap_ms, + stats_drop, + pa, + ca, + stats_ipc, + stats_shutdown, + ) + }) + .map_err(|e| format!("spawn stats: {e}"))?; + + // Spawn IDR command listener (separate thread, blocks on recv) + let idr_thread = idr_requested.clone(); + thread::Builder::new() + .name("nescapture-idr".into()) + .spawn(move || { + let cmd_path = std::path::PathBuf::from("/tmp/nescapture-cmd.sock"); + let _ = std::fs::remove_file(&cmd_path); + let sock = match std::os::unix::net::UnixDatagram::bind(&cmd_path) { + Ok(s) => { + let _ = std::fs::set_permissions( + &cmd_path, + std::os::unix::fs::PermissionsExt::from_mode(0o666), + ); + log::info!("cmd listener on {}", cmd_path.display()); + s + } + Err(e) => { + log::warn!("cmd socket bind failed: {e}"); + return; + } + }; + let mut buf = [0u8; 128]; + loop { + match sock.recv(&mut buf) { + Ok(1) if buf[0] == MSG_IDR_REQUEST => { + log::info!("IDR requested by client"); + idr_thread.store(true, Ordering::Relaxed); + } + Ok(n) if n >= 2 && buf[0] == MSG_ENCODE_SETTINGS => { + if let Some((codec_id, rc, value, depth)) = + decode_encode_settings(&buf[1..n]) + { + let codec = match codec_id { + CODEC_KEEP => None, + CODEC_H264 => Some(HwCodec::H264), + CODEC_H265 => Some(HwCodec::H265), + CODEC_AV1 => Some(HwCodec::AV1), + _ => { + log::warn!( + "unknown codec id {codec_id} in encode settings" + ); + continue; + } + }; + let rate_control = match rc { + 0 => RateControlMode::Cbr, + _ => RateControlMode::Cqp, + }; + let bit_depth = depth.and_then(|d| match d { + 0 => Some(EncodeBitDepth::Eight), + 1 => Some(EncodeBitDepth::Ten), + _ => None, + }); + let change = EncodeSettingsChange { + codec, + rate_control_mode: rate_control, + value, + bit_depth, + }; + if reconfig_tx.send(change).is_err() { + log::warn!("reconfig channel closed, stopping cmd listener"); + break; + } + } + } + Err(e) => { + log::warn!("cmd listener error: {e}"); + break; + } + _ => {} + } + } + }) + .map_err(|e| format!("spawn idr: {e}"))?; + + log::info!( + "pipeline ready — {:?} {}x{} @ {}FPS {} -> {}", + codec, + config.width, + config.height, + config.fps, + (if config.bitrate_kbps.is_some() { + std::format!("- CBR: {}kbps", config.bitrate_kbps.unwrap()) + } else if config.qp.is_some() { + std::format!("- QP: {}", config.qp.unwrap()) + } else { + "".to_string() + }), + ipc_path.display(), + ); + Ok(Self { + frame_tx, + idr_requested, + shutdown, + codec, + capture_fps, + encode_avg_ms, + capture_ms, + dropped_frames, + present_attempts, + capture_attempts, + }) + } + + pub fn push_frame(&self, frame: CapturedFrame) -> bool { + self.capture_fps.fetch_add(1, Ordering::Relaxed); + let ok = self.frame_tx.try_send(frame).is_ok(); + if !ok { + self.dropped_frames.fetch_add(1, Ordering::Relaxed); + } + ok + } + + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::SeqCst); + } + + pub fn request_idr(&self) { + self.idr_requested.store(true, Ordering::Relaxed); + } +} +impl Drop for PipelineHandle { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::SeqCst); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Encoder thread +// ───────────────────────────────────────────────────────────────────────────── + +struct EncoderConfig { + width: u32, + height: u32, + fps: u32, + bitrate_kbps: Option, + qp: Option, + idr_interval: u32, + encoder_tuning_mode: EncoderTuningMode, + pixel_format: PixelFormat, + codec: HwCodec, + ctx: pixelforge::VideoContext, + idr_requested: Arc, + reconfig_rx: mpsc::Receiver, + current_codec: Arc, + wanted_depth_override: Option, + needs_reconfig_flag: Arc, +} + +struct EncodedPacket { + data: Vec, + is_key_frame: bool, + frame_number: u32, +} + +#[derive(Debug, Clone)] +pub struct EncodeSettingsChange { + pub codec: Option, + pub rate_control_mode: RateControlMode, + pub value: u32, + pub bit_depth: Option, +} + +fn encoder_thread( + mut cfg: EncoderConfig, + frame_rx: mpsc::Receiver, + encoded_tx: mpsc::SyncSender, + shutdown: Arc, +) { + let ctx = cfg.ctx; + + let mut encoder_state: Option = None; + + let mut dmabuf_importer = match DmaBufImporter::new(ctx.clone()) { + Ok(i) => Some(i), + Err(e) => { + log::warn!("DmaBufImporter init failed: {e} — GPU path unavailable"); + None + } + }; + + let mut frame_number = 0u32; + + let wanted_depth = std::env::var("NESCAPTURE_DEPTH"); + + loop { + if shutdown.load(Ordering::Relaxed) { + break; + } + + // Check for dynamic encode settings changes + if let Ok(change) = cfg.reconfig_rx.try_recv() { + log::info!( + "reconfig: codec={:?}, rc={:?}, value={}", + change.codec, + change.rate_control_mode, + change.value, + ); + match change.rate_control_mode { + RateControlMode::Cbr => { + cfg.bitrate_kbps = Some(change.value); + cfg.qp = None; + } + RateControlMode::Cqp => { + cfg.bitrate_kbps = None; + cfg.qp = Some(change.value); + } + _ => { + log::warn!( + "unsupported rate control mode {:?}, keeping current", + change.rate_control_mode + ); + } + } + if let Some(codec) = change.codec { + cfg.codec = codec; + } + if let Some(depth) = change.bit_depth { + cfg.wanted_depth_override = Some(depth); + } + // Drop old encoder state to force re-creation with new settings + encoder_state = None; + // Signal IPC thread to set FLAG_RECONFIG on next frame + cfg.needs_reconfig_flag.store(true, Ordering::Relaxed); + // Update IPC thread with new codec + cfg.current_codec + .store(cfg.codec.to_protocol_codec(), Ordering::Relaxed); + // Request IDR so first frame after reconfig has new SPS/PPS + cfg.idr_requested.store(true, Ordering::Relaxed); + } + + let mut raw = match frame_rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(frame) => frame, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + + let bit_depth = if let Some(ov) = cfg.wanted_depth_override { + ov + } else { + match wanted_depth.as_deref() { + Ok("10") => EncodeBitDepth::Ten, + Ok("8") => EncodeBitDepth::Eight, + _ => vk_format_to_bit_depth(raw.vk_format), + } + }; + let input_fmt = vk_format_to_input_format(raw.vk_format).unwrap_or(InputFormat::BGRA); + let color_space = vk_colorspace_to_color_space(raw.vk_colorspace); + let out_fmt = output_format(cfg.pixel_format, bit_depth); + + let state = match encoder_state.as_mut() { + Some(s) if s.bit_depth == bit_depth && s.pixel_format == cfg.pixel_format => s, + _ => { + let color_desc = vk_colorspace_to_color_description(raw.vk_colorspace); + match PerFrameEncoder::new( + &ctx, + cfg.codec.to_pixelforge(), + cfg.width, + cfg.height, + cfg.fps, + cfg.bitrate_kbps, + cfg.qp, + cfg.idr_interval, + cfg.encoder_tuning_mode, + cfg.pixel_format, + bit_depth, + color_desc, + input_fmt, + out_fmt, + color_space, + ) { + Ok(s) => { + encoder_state = Some(s); + encoder_state.as_mut().unwrap() + } + Err(e) => { + log::error!("encoder (re)init: {e}"); + frame_number += 1; + continue; + } + } + } + }; + + let mut force_idr = cfg.idr_requested.swap(false, Ordering::Relaxed); + if cfg.idr_interval > 0 { + force_idr = force_idr || frame_number % cfg.idr_interval == 0; + } + if force_idr { + state.encoder.request_idr(); + } + + let result = match &mut raw.source { + FrameSource::DmaBuf { + fd, + stride, + modifier, + } => { + let owned_fd = *fd; + *fd = -1; + match dmabuf_importer.as_mut() { + Some(importer) => gpu_encode_frame( + importer, + &mut state.converter, + &mut state.encoder, + owned_fd, + *stride, + *modifier, + raw.width, + raw.height, + raw.vk_format, + frame_number, + ), + None => { + unsafe { libc::close(owned_fd) }; + log::warn!( + "DmaBuf fd available but importer is gone — skipping frame {frame_number}" + ); + frame_number += 1; + continue; + } + } + } + FrameSource::Pixels(pixels) => cpu_encode_frame( + &ctx, + &mut state.encoder, + pixels, + raw.width, + raw.height, + raw.vk_format, + ), + }; + + match result { + Err(e) => log::warn!("encode frame {frame_number}: {e}"), + Ok(future) => { + let _ = encoded_tx.try_send(future); + } + } + + frame_number += 1; + } + + // Flush + if let Some(state) = encoder_state.as_mut() { + let _ = state.encoder.flush(); + } + + log::info!("encoder thread exited"); +} + +struct PerFrameEncoder { + encoder: Encoder, + converter: ColorConverter, + bit_depth: EncodeBitDepth, + pixel_format: PixelFormat, +} + +impl PerFrameEncoder { + #[allow(clippy::too_many_arguments)] + fn new( + ctx: &pixelforge::VideoContext, + codec: Codec, + width: u32, + height: u32, + fps: u32, + bitrate_kbps: Option, + qp: Option, + idr_interval: u32, + encoder_tuning_mode: EncoderTuningMode, + pixel_format: PixelFormat, + bit_depth: EncodeBitDepth, + color_desc: Option, + input_fmt: InputFormat, + out_fmt: OutputFormat, + color_space: ColorSpace, + ) -> Result { + log::info!( + "(re)init encoder: {:?} {:?} {:?} {:?} → {:?}", + codec, + pixel_format, + bit_depth, + color_space, + out_fmt + ); + + let mut enc_cfg = match codec { + Codec::H264 => EncodeConfig::h264(width, height), + Codec::H265 => EncodeConfig::h265(width, height), + Codec::AV1 => EncodeConfig::av1(width, height), + }; + enc_cfg = enc_cfg + .with_frame_rate(fps, 1) + .with_gop_size(idr_interval) + .with_b_frames(0) + .with_pixel_format(pixel_format) + .with_bit_depth(bit_depth) + .with_encode_usage_hint(EncodeUsageHint::Streaming) + .with_encode_content_hint(EncodeContentHint::Rendered) + .with_encoder_tuning_mode(encoder_tuning_mode); + if let Some(desc) = color_desc { + enc_cfg = enc_cfg.with_color_description(desc); + } else { + // GPU framebuffer captures are always full-range — use BT.709 full-range + // so the decoder doesn't apply limited‑range expansion. + enc_cfg = enc_cfg.with_color_description(ColorDescription::bt709()); + } + enc_cfg = if let Some(q) = qp { + enc_cfg + .with_rate_control(RateControlMode::Cqp) + .with_quality_level(q) + } else { + if let Some(bitrate) = bitrate_kbps { + enc_cfg + .with_rate_control(RateControlMode::Cbr) + .with_target_bitrate(bitrate * 1_000) + } else { + enc_cfg + .with_rate_control(RateControlMode::Cbr) + .with_target_bitrate(1000 * 1_000) + } + }; + + let encoder = + Encoder::new(ctx.clone(), enc_cfg).map_err(|e| format!("Encoder::new: {e}"))?; + + let conv_cfg = + ColorConverterConfig::new(width, height, /*color_space,*/ input_fmt, out_fmt); + + let mut converter = ColorConverter::new(ctx.clone(), conv_cfg) + .map_err(|e| format!("ColorConverter::new: {e}"))?; + + converter.set_full_range(true); + + Ok(Self { + encoder, + converter, + bit_depth, + pixel_format, + }) + } +} + +fn gpu_encode_frame( + importer: &mut DmaBufImporter, + converter: &mut ColorConverter, + encoder: &mut Encoder, + fd: RawFd, + stride: u32, + modifier: u64, + width: u32, + height: u32, + vk_format: u32, + frame_number: u32, +) -> Result { + use ash::vk; + + let bgra_vk_fmt = map_vk_format_raw(vk_format); + + let plane = DmaBufPlane { + fd, + offset: 0, + stride, + modifier, + }; + + let (imported_image, needs_layout_transition) = importer + .import_or_reuse(0, width, height, bgra_vk_fmt, &[plane]) + .map_err(|e| anyhow::anyhow!("DmaBufImporter: {e}"))?; + + unsafe { libc::close(fd) }; + + let src_layout = if needs_layout_transition { + vk::ImageLayout::UNDEFINED + } else { + vk::ImageLayout::GENERAL + }; + + converter + .convert(imported_image, src_layout, encoder.input_image()) + .map_err(|e| anyhow::anyhow!("ColorConverter::convert frame {frame_number}: {e}"))?; + + encoder + .encode(encoder.input_image()) + .map_err(|e| anyhow::anyhow!("Encoder::encode frame {frame_number}: {e}")) +} + +fn map_vk_format_raw(vk_format: u32) -> ash::vk::Format { + ash::vk::Format::from_raw(vk_format as i32) +} + +fn cpu_encode_frame( + ctx: &pixelforge::VideoContext, + encoder: &mut Encoder, + pixels: &[u8], + width: u32, + height: u32, + vk_format: u32, +) -> Result { + use pixelforge::{EncodeBitDepth, InputImage}; + + let yuv = bgra_to_yuv420(pixels, width, height, vk_format); + + let mut input_image = InputImage::new( + ctx.clone(), + Codec::H264, + width, + height, + EncodeBitDepth::Eight, + PixelFormat::Yuv420, + ) + .map_err(|e| anyhow::anyhow!("InputImage::new: {e}"))?; + + input_image + .upload_yuv420_to(encoder.input_image(), &yuv) + .map_err(|e| anyhow::anyhow!("upload_yuv420_to: {e}"))?; + + encoder + .encode(encoder.input_image()) + .map_err(|e| anyhow::anyhow!("Encoder::encode (CPU path): {e}")) +} + +fn bgra_to_yuv420(pixels: &[u8], width: u32, height: u32, vk_format: u32) -> Vec { + let w = width as usize; + let h = height as usize; + let bgra = (44..=50).contains(&vk_format); + let luma = w * h; + let chroma = (w / 2) * (h / 2); + let mut yuv = vec![0u8; luma + 2 * chroma]; + let (y, uv) = yuv.split_at_mut(luma); + let (u, v) = uv.split_at_mut(chroma); + for row in 0..h { + for col in 0..w { + let i = (row * w + col) * 4; + if i + 2 >= pixels.len() { + break; + } + let (r, g, b) = if bgra { + (pixels[i + 2] as i32, pixels[i + 1] as i32, pixels[i] as i32) + } else { + (pixels[i] as i32, pixels[i + 1] as i32, pixels[i + 2] as i32) + }; + y[row * w + col] = (((66 * r + 129 * g + 25 * b + 128) >> 8) + 16).clamp(16, 235) as u8; + if row % 2 == 0 && col % 2 == 0 { + let ci = (row / 2) * (w / 2) + col / 2; + u[ci] = (((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128).clamp(16, 240) as u8; + v[ci] = (((112 * r - 94 * g - 18 * b + 128) >> 8) + 128).clamp(16, 240) as u8; + } + } + } + yuv +} + +// ───────────────────────────────────────────────────────────────────────────── +// IPC send thread +// ───────────────────────────────────────────────────────────────────────────── + +struct IpcConfig { + ipc_path: std::path::PathBuf, + current_codec: Arc, + needs_reconfig_flag: Arc, + width: u16, + height: u16, + encode_ms: Arc, +} + +fn ipc_send_thread( + cfg: IpcConfig, + encoded_rx: mpsc::Receiver, + shutdown: Arc, +) { + let socket = match UnixDatagram::unbound() { + Ok(s) => { + let so_sndbuf: libc::c_int = 4 * 1024 * 1024; + unsafe { + libc::setsockopt( + s.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_SNDBUF, + &so_sndbuf as *const _ as *const libc::c_void, + std::mem::size_of::() as libc::socklen_t, + ); + } + s + } + Err(e) => { + log::error!("IPC socket create: {e}"); + return; + } + }; + + let start_time = Instant::now(); + let mut frame_count: u64 = 0; + + 'outer: loop { + if shutdown.load(Ordering::Relaxed) { + break; + } + + // Connect to hub with retry + loop { + if shutdown.load(Ordering::Relaxed) { + break 'outer; + } + match socket.connect(&cfg.ipc_path) { + Ok(()) => { + log::info!("IPC connected → {}", cfg.ipc_path.display(),); + break; + } + Err(e) => { + log::warn!( + "IPC connect to {} failed (retrying in 2s): {e}", + cfg.ipc_path.display() + ); + std::thread::sleep(std::time::Duration::from_secs(2)); + } + } + } + + // Send loop + let mut last_warn = Instant::now(); + let mut error_count: u64 = 0; + + loop { + if shutdown.load(Ordering::Relaxed) { + break 'outer; + } + let result = match encoded_rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(p) => pollster::block_on(p), + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break 'outer, + }; + let pkt = match result { + Ok(p) => p, + Err(e) => { + log::warn!("encoder future receive: {e}"); + continue; + } + }; + + if let Some(stats) = &pkt.stats { + let enc_ms: f32 = (std::time::Duration::from_nanos(stats.gpu_time_ns).as_secs_f64() + * 1000.0) as f32; + cfg.encode_ms.store(enc_ms.to_bits(), Ordering::Relaxed); + } + + let timestamp_ms = start_time.elapsed().as_millis() as u32; + let mut flags = if pkt.is_key_frame { FLAG_KEYFRAME } else { 0 }; + // Set FLAG_RECONFIG on the first frame after an encoder reconfig. + // Clear it after setting so only the first frame is marked. + if cfg.needs_reconfig_flag.swap(false, Ordering::Relaxed) { + flags |= FLAG_RECONFIG; + } + let protocol_codec = cfg.current_codec.load(Ordering::Relaxed); + + let ipc_frame = encode_ipc_frame( + STREAM_VIDEO, + protocol_codec, + flags, + timestamp_ms, + cfg.width, + cfg.height, + &pkt.data, + ); + + if let Err(e) = socket.send(&ipc_frame) { + error_count += 1; + if last_warn.elapsed() > std::time::Duration::from_secs(5) { + log::warn!("IPC send failed ({} frames dropped): {e}", error_count); + last_warn = Instant::now(); + } + // Socket disconnected — reconnect + log::warn!("IPC disconnected, reconnecting..."); + break; + } + + frame_count += 1; + if frame_count % 300 == 0 { + log::trace!("IPC sent {frame_count} frames"); + } + } + } + + log::info!("IPC thread exited ({frame_count} frames)"); +} + +fn stats_sender_thread( + capture_fps: Arc, + encode_avg_ms: Arc, + capture_ms: Arc, + dropped_frames: Arc, + present_attempts: Arc, + capture_attempts: Arc, + ipc_path: std::path::PathBuf, + shutdown: Arc, +) { + let socket = match std::os::unix::net::UnixDatagram::unbound() { + Ok(s) => s, + Err(e) => { + log::error!("stats socket create: {e}"); + return; + } + }; + if socket.connect(&ipc_path).is_err() { + log::warn!("stats socket connect failed, stats unavailable"); + return; + } + + log::info!("stats sender → {}", ipc_path.display()); + + loop { + if shutdown.load(Ordering::Relaxed) { + break; + } + std::thread::sleep(std::time::Duration::from_secs(1)); + + let raw_fps = capture_fps.load(Ordering::Relaxed); + let fps = raw_fps.min(255) as u8; + capture_fps.store(0, Ordering::Relaxed); + let enc_bits = encode_avg_ms.swap(0, Ordering::Relaxed); + let enc_ms = f32::from_bits(enc_bits); + let cap_ms = f32::from_bits(capture_ms.swap(0, Ordering::Relaxed)); + let dropped = dropped_frames.swap(0, Ordering::Relaxed); + let pa = present_attempts.swap(0, Ordering::Relaxed); + let ca = capture_attempts.swap(0, Ordering::Relaxed); + + let mut buf = Vec::with_capacity(22); + nesprotocol::stats::encode_hudless_stats( + &mut buf, fps, enc_ms, dropped, pa, ca, cap_ms, + ); + let _ = socket.send(&buf); + } + + log::info!("stats sender exited"); +} diff --git a/apps/nescapture/src/framebuffer.rs b/apps/nescapture/src/framebuffer.rs new file mode 100644 index 00000000..dfcf9f7d --- /dev/null +++ b/apps/nescapture/src/framebuffer.rs @@ -0,0 +1,193 @@ +// ───────────────────────────────────────────────────────────────────────────── +// framebuffer.rs — Phase 2: image view and framebuffer tracking +// +// We need to know which VkImage is bound as a color attachment at draw time. +// The chain is: VkFramebuffer → VkImageView → VkImage + VkFormat. +// +// vkCreateImageView → view_to_image[view] = image, view_format[view] = fmt +// vkCreateFramebuffer → framebuffer_to_views[fb] = views, framebuffer_extent[fb] = extent +// vkAllocateCommandBuffers → cmd_buf_to_device_key[cmd_buf] = device_key +// vkFreeCommandBuffers → remove from CB_STATE and cmd_buf_to_device_key +// ───────────────────────────────────────────────────────────────────────────── + +use crate::dispatch_key; +use crate::state::{CB_STATE, CMD_BUF_TO_DEVICE_KEY, DEVICE_STATE}; +use ash::vk::{self, Handle}; +use std::os::raw::c_void; + +// ───────────────────────────────────────────────────────────────────────────── +// vkCreateImageView — map view → image and view → format +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateImageView( + device: vk::Device, + p_create_info: *const vk::ImageViewCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_view: *mut vk::ImageView, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let result = unsafe { (ds.fp.create_image_view)(device, p_create_info, p_allocator, p_view) }; + if result != vk::Result::SUCCESS { + return result; + } + + let ci = unsafe { &*p_create_info }; + let view = unsafe { *p_view }; + let image = ci.image; + + ds.view_to_image.insert(view.as_raw(), image.as_raw()); + ds.view_format.insert(view.as_raw(), ci.format); + + /*log::trace!( + "image view {:#010x} → image {:#010x} format={}", + view.as_raw(), + image.as_raw(), + ci.format.as_raw(), + );*/ + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyImageView( + device: vk::Device, + image_view: vk::ImageView, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + ds.view_to_image.remove(&image_view.as_raw()); + ds.view_format.remove(&image_view.as_raw()); + unsafe { (ds.fp.destroy_image_view)(device, image_view, p_allocator) }; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkCreateFramebuffer — map framebuffer → views and framebuffer → extent +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateFramebuffer( + device: vk::Device, + p_create_info: *const vk::FramebufferCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_framebuffer: *mut vk::Framebuffer, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let result = + unsafe { (ds.fp.create_framebuffer)(device, p_create_info, p_allocator, p_framebuffer) }; + if result != vk::Result::SUCCESS { + return result; + } + + let ci = unsafe { &*p_create_info }; + let fb = unsafe { *p_framebuffer }; + let views: Vec = + unsafe { std::slice::from_raw_parts(ci.p_attachments, ci.attachment_count as usize) } + .iter() + .map(|v| v.as_raw()) + .collect(); + + ds.framebuffer_to_views.insert(fb.as_raw(), views.clone()); + ds.framebuffer_extent.insert( + fb.as_raw(), + vk::Extent2D { + width: ci.width, + height: ci.height, + }, + ); + + log::trace!( + "framebuffer {:#010x} → {} views, extent {}x{}", + fb.as_raw(), + ci.attachment_count, + ci.width, + ci.height, + ); + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyFramebuffer( + device: vk::Device, + framebuffer: vk::Framebuffer, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + ds.framebuffer_to_views.remove(&framebuffer.as_raw()); + ds.framebuffer_extent.remove(&framebuffer.as_raw()); + unsafe { (ds.fp.destroy_framebuffer)(device, framebuffer, p_allocator) }; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkAllocateCommandBuffers — register command buffer → device key mapping +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkAllocateCommandBuffers( + device: vk::Device, + p_allocate_info: *const vk::CommandBufferAllocateInfo, + p_command_buffers: *mut vk::CommandBuffer, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let result = + unsafe { (ds.fp.allocate_command_buffers)(device, p_allocate_info, p_command_buffers) }; + if result != vk::Result::SUCCESS { + return result; + } + + let ai = unsafe { &*p_allocate_info }; + let count = ai.command_buffer_count; + let buffers = unsafe { std::slice::from_raw_parts(p_command_buffers, count as usize) }; + + for &cb in buffers { + CMD_BUF_TO_DEVICE_KEY.insert(cb.as_raw(), key); + } + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkFreeCommandBuffers( + device: vk::Device, + command_pool: vk::CommandPool, + command_buffer_count: u32, + p_command_buffers: *const vk::CommandBuffer, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + let buffers = + unsafe { std::slice::from_raw_parts(p_command_buffers, command_buffer_count as usize) }; + for &cb in buffers { + CMD_BUF_TO_DEVICE_KEY.remove(&cb.as_raw()); + CB_STATE.remove(&cb.as_raw()); + } + unsafe { + (ds.fp.free_command_buffers)( + device, + command_pool, + command_buffer_count, + p_command_buffers, + ) + }; + } +} diff --git a/apps/nescapture/src/instance.rs b/apps/nescapture/src/instance.rs new file mode 100644 index 00000000..3a47b4d4 --- /dev/null +++ b/apps/nescapture/src/instance.rs @@ -0,0 +1,80 @@ +// ───────────────────────────────────────────────────────────────────────────── +// instance.rs — vkCreateInstance and vkDestroyInstance +// ───────────────────────────────────────────────────────────────────────────── + +use crate::dispatch::NextInstanceFn; +use crate::state::INSTANCE_STATE; +use crate::{VkLayerInstanceCreateInfo, dispatch_key, find_layer_link, load_instance_fn}; +use ash::vk::{self, Handle}; +use std::os::raw::c_void; +use std::sync::Arc; + +use crate::dispatch::PFN_vkCreateInstance; + +const VK_LAYER_LINK_INFO: u32 = 0; + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateInstance( + p_create_info: *const vk::InstanceCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_instance: *mut vk::Instance, +) -> vk::Result { + crate::init_logger(); + + // ── Walk the pNext chain to find the loader's layer link ───────────────── + let layer_info: *mut VkLayerInstanceCreateInfo = match unsafe { + find_layer_link((*p_create_info).p_next as *const c_void, VK_LAYER_LINK_INFO) + } { + Some(p) => p, + None => return vk::Result::ERROR_INITIALIZATION_FAILED, + }; + + let layer_link = unsafe { (*layer_info).u.pLayerInfo }; + let next_gipa = match unsafe { (*layer_link).pfnNextGetInstanceProcAddr } { + Some(f) => f, + None => return vk::Result::ERROR_INITIALIZATION_FAILED, + }; + // Advance the chain before calling through so the next layer gets its link. + unsafe { (*layer_info).u.pLayerInfo = (*layer_link).pNext }; + + // ── Call the next layer / loader's vkCreateInstance ────────────────────── + let next_create: PFN_vkCreateInstance = + unsafe { load_instance_fn(next_gipa, vk::Instance::null(), b"vkCreateInstance\0") }; + + let result = unsafe { next_create(p_create_info, p_allocator, p_instance) }; + if result != vk::Result::SUCCESS { + return result; + } + + let instance = unsafe { *p_instance }; + let key = unsafe { dispatch_key(instance.as_raw() as *const c_void) }; + + // ── Build and store per-instance dispatch table ─────────────────────────── + let istate = Arc::new(NextInstanceFn { + get_instance_proc_addr: next_gipa, + destroy_instance: unsafe { load_instance_fn(next_gipa, instance, b"vkDestroyInstance\0") }, + get_physical_device_memory_properties: unsafe { + load_instance_fn( + next_gipa, + instance, + b"vkGetPhysicalDeviceMemoryProperties\0", + ) + }, + create_device: unsafe { load_instance_fn(next_gipa, instance, b"vkCreateDevice\0") }, + }); + + INSTANCE_STATE.insert(key, istate); + log::debug!("vkCreateInstance OK (enabled={})", crate::enabled()); + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyInstance( + instance: vk::Instance, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(instance.as_raw() as *const c_void) }; + if let Some((_, state)) = INSTANCE_STATE.remove(&key) { + unsafe { (state.destroy_instance)(instance, p_allocator) }; + } +} diff --git a/apps/nescapture/src/lib.rs b/apps/nescapture/src/lib.rs new file mode 100644 index 00000000..927bc732 --- /dev/null +++ b/apps/nescapture/src/lib.rs @@ -0,0 +1,353 @@ +// ───────────────────────────────────────────────────────────────────────────── +// lib.rs — nescapture Vulkan implicit layer +// +// Entry points: +// vkNegotiateLoaderLayerInterfaceVersion +// vkGetInstanceProcAddr +// vkGetDeviceProcAddr +// +// To add a new hook: +// 1. Implement it in the relevant module. +// 2. Add it to `match_device_fn` below. +// 3. Export it with #[unsafe(no_mangle)] in the module. +// ───────────────────────────────────────────────────────────────────────────── + +#![allow( + non_snake_case, + non_camel_case_types, + dead_code, + unused_variables, + clippy::missing_safety_doc, + clippy::too_many_arguments +)] + +mod capture; +mod commands; +mod config; +mod device; +mod discovery; +mod dispatch; +mod dmabuf_import; +mod encode; +mod framebuffer; +mod instance; +mod pipeline; +mod present; +mod shader; +mod state; +mod swapchain; + +use commands::{ + vkCmdBeginRenderPass, vkCmdBeginRenderingKHR, vkCmdBindPipeline, vkCmdDraw, vkCmdDrawIndexed, + vkCmdDrawIndexedIndirect, vkCmdDrawIndexedIndirectCount, vkCmdDrawIndexedIndirectCountKHR, + vkCmdDrawIndirect, vkCmdDrawIndirectCount, vkCmdDrawIndirectCountKHR, vkCmdEndRenderPass, + vkCmdEndRenderingKHR, +}; +use device::{vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue}; +use framebuffer::{ + vkAllocateCommandBuffers, vkCreateFramebuffer, vkCreateImageView, vkDestroyFramebuffer, + vkDestroyImageView, vkFreeCommandBuffers, +}; +use instance::{vkCreateInstance, vkDestroyInstance}; +use pipeline::{vkCreateGraphicsPipelines, vkDestroyPipeline}; +use present::vkQueuePresentKHR; +use shader::{vkCreateShaderModule, vkDestroyShaderModule}; +use swapchain::{vkCreateSwapchainKHR, vkDestroySwapchainKHR, vkGetSwapchainImagesKHR}; + +use dispatch::{PFN_vkGetDeviceProcAddr, PFN_vkGetInstanceProcAddr, RawFn}; +use state::{DEVICE_STATE, INSTANCE_STATE}; + +use ash::vk::{self, Handle}; +use once_cell::sync::OnceCell; +use std::ffi::CStr; +use std::os::raw::{c_char, c_void}; + +const ENABLE_ENV: &str = "NESCAPTURE_ENABLE"; + +pub(crate) fn enabled() -> bool { + std::env::var(ENABLE_ENV).map(|v| v == "1").unwrap_or(false) +} + +static LOGGER: OnceCell<()> = OnceCell::new(); + +pub(crate) fn init_logger() { + LOGGER.get_or_init(|| { + env_logger::Builder::from_default_env().init(); + }); +} + +// ── Vulkan loader structs (not in ash) ──────────────────────────────────────── + +#[repr(C)] +pub(crate) struct VkLayerInstanceLink { + pNext: *mut VkLayerInstanceLink, + pfnNextGetInstanceProcAddr: Option, + pfnNextGetPhysicalDeviceProcAddr: Option, +} + +#[repr(C)] +pub(crate) struct VkLayerDeviceLink { + pNext: *mut VkLayerDeviceLink, + pfnNextGetInstanceProcAddr: Option, + pfnNextGetDeviceProcAddr: Option, +} + +#[repr(C)] +pub(crate) union VkLayerCreateInfoU { + pub pLayerInfo: *mut VkLayerInstanceLink, + pub pDeviceLayerInfo: *mut VkLayerDeviceLink, +} + +const VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO: i32 = 47; +const VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO: i32 = 48; + +#[repr(C)] +pub(crate) struct VkLayerInstanceCreateInfo { + pub sType: i32, + pub pNext: *const c_void, + pub function: u32, + pub u: VkLayerCreateInfoU, +} + +pub(crate) type VkLayerDeviceCreateInfo = VkLayerInstanceCreateInfo; + +pub(crate) unsafe fn dispatch_key(handle: *const c_void) -> usize { + unsafe { *(handle as *const usize) } +} + +pub(crate) unsafe fn find_layer_link(p_next: *const c_void, function: u32) -> Option<*mut T> { + let mut current = p_next; + while !current.is_null() { + let header = current as *const VkLayerInstanceCreateInfo; + let s = unsafe { (*header).sType }; + if (s == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO + || s == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO) + && unsafe { (*header).function } == function + { + return Some(header as *mut T); + } + current = unsafe { (*header).pNext }; + } + None +} + +pub(crate) unsafe fn load_device_fn( + get: PFN_vkGetDeviceProcAddr, + device: vk::Device, + name: &[u8], +) -> T { + let raw = unsafe { + get(device, name.as_ptr() as *const c_char) + .unwrap_or_else(|| panic!("missing device fn: {}", core::str::from_utf8(name).unwrap())) + }; + unsafe { std::mem::transmute_copy(&raw) } +} + +pub(crate) unsafe fn try_load_device_fn( + get: PFN_vkGetDeviceProcAddr, + device: vk::Device, + name: &[u8], +) -> Option { + let raw = unsafe { get(device, name.as_ptr() as *const c_char) }?; + Some(unsafe { std::mem::transmute_copy(&raw) }) +} + +pub(crate) unsafe fn load_instance_fn( + get: PFN_vkGetInstanceProcAddr, + instance: vk::Instance, + name: &[u8], +) -> T { + let raw = unsafe { + get(instance, name.as_ptr() as *const c_char).unwrap_or_else(|| { + panic!( + "missing instance fn: {}", + core::str::from_utf8(name).unwrap() + ) + }) + }; + unsafe { std::mem::transmute_copy(&raw) } +} + +#[inline] +pub(crate) unsafe fn to_raw(addr: usize) -> RawFn { + unsafe { std::mem::transmute(addr) } +} + +macro_rules! layer_fn { + ($f:ident) => { + return Some(unsafe { to_raw($f as *const () as usize) }) + }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkGetInstanceProcAddr +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkGetInstanceProcAddr( + instance: vk::Instance, + p_name: *const c_char, +) -> Option { + let name = unsafe { CStr::from_ptr(p_name).to_bytes() }; + + match name { + b"vkGetInstanceProcAddr" => layer_fn!(vkGetInstanceProcAddr), + b"vkCreateInstance" => layer_fn!(vkCreateInstance), + b"vkDestroyInstance" => layer_fn!(vkDestroyInstance), + b"vkCreateDevice" => layer_fn!(vkCreateDevice), + _ => {} + } + + if let Some(f) = unsafe { match_device_fn(name) } { + return Some(f); + } + + if instance.as_raw() == 0 { + return None; + } + + let ikey = unsafe { dispatch_key(instance.as_raw() as *const c_void) }; + if let Some(istate) = INSTANCE_STATE.get(&ikey) { + return unsafe { (istate.get_instance_proc_addr)(instance, p_name) }; + } + None +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkGetDeviceProcAddr +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkGetDeviceProcAddr( + device: vk::Device, + p_name: *const c_char, +) -> Option { + let name = unsafe { CStr::from_ptr(p_name).to_bytes() }; + + if let Some(f) = unsafe { match_device_fn(name) } { + return Some(f); + } + + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + return unsafe { (ds.fp.get_device_proc_addr)(device, p_name) }; + } + None +} + +unsafe fn match_device_fn(name: &[u8]) -> Option { + let name_str = std::str::from_utf8(name).unwrap_or(""); + // Only log swapchain/acquire/present functions to keep noise down + let log_this = name_str.contains("Swapchain") + || name_str.contains("AcquireNextImage") + || name_str.contains("QueuePresent") + || name_str.contains("CreateDevice") + || name_str.contains("GetSwapchain"); + if log_this { + log::debug!("gdpa: {}", name_str); + } + unsafe { + match name { + b"vkGetDeviceProcAddr" => Some(to_raw(vkGetDeviceProcAddr as *const () as usize)), + b"vkCreateDevice" => Some(to_raw(vkCreateDevice as *const () as usize)), + b"vkDestroyDevice" => Some(to_raw(vkDestroyDevice as *const () as usize)), + b"vkGetDeviceQueue" => Some(to_raw(vkGetDeviceQueue as *const () as usize)), + b"vkQueuePresentKHR" => Some(to_raw(vkQueuePresentKHR as *const () as usize)), + + b"vkCreateShaderModule" => Some(to_raw(vkCreateShaderModule as *const () as usize)), + b"vkDestroyShaderModule" => Some(to_raw(vkDestroyShaderModule as *const () as usize)), + b"vkCreateGraphicsPipelines" => { + Some(to_raw(vkCreateGraphicsPipelines as *const () as usize)) + } + b"vkDestroyPipeline" => Some(to_raw(vkDestroyPipeline as *const () as usize)), + + b"vkCreateImageView" => Some(to_raw(vkCreateImageView as *const () as usize)), + b"vkDestroyImageView" => Some(to_raw(vkDestroyImageView as *const () as usize)), + b"vkCreateFramebuffer" => Some(to_raw(vkCreateFramebuffer as *const () as usize)), + b"vkDestroyFramebuffer" => Some(to_raw(vkDestroyFramebuffer as *const () as usize)), + b"vkAllocateCommandBuffers" => { + Some(to_raw(vkAllocateCommandBuffers as *const () as usize)) + } + b"vkFreeCommandBuffers" => Some(to_raw(vkFreeCommandBuffers as *const () as usize)), + + b"vkCmdBindPipeline" => Some(to_raw(vkCmdBindPipeline as *const () as usize)), + b"vkCmdBeginRenderPass" => Some(to_raw(vkCmdBeginRenderPass as *const () as usize)), + b"vkCmdEndRenderPass" => Some(to_raw(vkCmdEndRenderPass as *const () as usize)), + b"vkCmdBeginRenderingKHR" => Some(to_raw(vkCmdBeginRenderingKHR as *const () as usize)), + b"vkCmdEndRenderingKHR" => Some(to_raw(vkCmdEndRenderingKHR as *const () as usize)), + + b"vkCmdDraw" => Some(to_raw(vkCmdDraw as *const () as usize)), + b"vkCmdDrawIndexed" => Some(to_raw(vkCmdDrawIndexed as *const () as usize)), + b"vkCmdDrawIndirect" => Some(to_raw(vkCmdDrawIndirect as *const () as usize)), + b"vkCmdDrawIndexedIndirect" => { + Some(to_raw(vkCmdDrawIndexedIndirect as *const () as usize)) + } + b"vkCmdDrawIndirectCount" => Some(to_raw(vkCmdDrawIndirectCount as *const () as usize)), + b"vkCmdDrawIndexedIndirectCount" => { + Some(to_raw(vkCmdDrawIndexedIndirectCount as *const () as usize)) + } + b"vkCmdDrawIndirectCountKHR" => { + Some(to_raw(vkCmdDrawIndirectCountKHR as *const () as usize)) + } + b"vkCmdDrawIndexedIndirectCountKHR" => Some(to_raw( + vkCmdDrawIndexedIndirectCountKHR as *const () as usize, + )), + + b"vkCreateSwapchainKHR" => { + if log_this { + log::debug!("gdpa: → our vkCreateSwapchainKHR"); + } + Some(to_raw(vkCreateSwapchainKHR as *const () as usize)) + } + b"vkDestroySwapchainKHR" => { + if log_this { + log::debug!("gdpa: → our vkDestroySwapchainKHR"); + } + Some(to_raw(vkDestroySwapchainKHR as *const () as usize)) + } + b"vkGetSwapchainImagesKHR" => { + if log_this { + log::debug!("gdpa: → our vkGetSwapchainImagesKHR"); + } + Some(to_raw(vkGetSwapchainImagesKHR as *const () as usize)) + } + + _ => { + if log_this { + log::debug!("gdpa: → passthrough (not intercepted)"); + } + None + } + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// vkNegotiateLoaderLayerInterfaceVersion +// ───────────────────────────────────────────────────────────────────────────── + +#[repr(C)] +pub struct VkNegotiateLayerInterface { + pub sType: u32, + pub pNext: *mut c_void, + pub loaderLayerInterfaceVersion: u32, + pub pfnGetInstanceProcAddr: Option, + pub pfnGetDeviceProcAddr: Option, + pub pfnGetPhysicalDeviceProcAddr: + Option Option>, +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkNegotiateLoaderLayerInterfaceVersion( + p_version_struct: *mut VkNegotiateLayerInterface, +) -> vk::Result { + init_logger(); + let s = unsafe { &mut *p_version_struct }; + if s.loaderLayerInterfaceVersion > 2 { + s.loaderLayerInterfaceVersion = 2; + } + s.pfnGetInstanceProcAddr = Some(vkGetInstanceProcAddr); + s.pfnGetDeviceProcAddr = Some(vkGetDeviceProcAddr); + s.pfnGetPhysicalDeviceProcAddr = None; + vk::Result::SUCCESS +} diff --git a/apps/nescapture/src/pipeline.rs b/apps/nescapture/src/pipeline.rs new file mode 100644 index 00000000..24ea817f --- /dev/null +++ b/apps/nescapture/src/pipeline.rs @@ -0,0 +1,144 @@ +// ───────────────────────────────────────────────────────────────────────────── +// pipeline.rs — Phase 1: graphics pipeline tracking +// +// When the game calls vkCreateGraphicsPipelines, we: +// 1. Let the call through to the next layer / driver. +// 2. For each created pipeline, walk its shader stages. +// 3. Look up each stage's VkShaderModule in shader_registry to get its hash. +// 4. Store (VkPipeline → PipelineHashes{vert_hash, frag_hash}) in +// DeviceState::pipeline_registry. +// +// vkDestroyPipeline removes the entry to keep the map bounded. +// +// Phases 3+ use pipeline_registry in vkCmdBindPipeline to know which +// shader hashes are active when draw calls are issued. +// ───────────────────────────────────────────────────────────────────────────── + +use crate::dispatch_key; +use crate::state::{DEVICE_STATE, PipelineHashes, PipelineState}; +use ash::vk::{self, Handle}; +use std::os::raw::c_void; + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateGraphicsPipelines( + device: vk::Device, + pipeline_cache: vk::PipelineCache, + create_info_count: u32, + p_create_infos: *const vk::GraphicsPipelineCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_pipelines: *mut vk::Pipeline, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + // Call through first. On success, p_pipelines is populated. + let result = unsafe { + (ds.fp.create_graphics_pipelines)( + device, + pipeline_cache, + create_info_count, + p_create_infos, + p_allocator, + p_pipelines, + ) + }; + if result != vk::Result::SUCCESS { + return result; + } + + let infos = unsafe { std::slice::from_raw_parts(p_create_infos, create_info_count as usize) }; + let pipelines = unsafe { std::slice::from_raw_parts(p_pipelines, create_info_count as usize) }; + + for (ci, &pipeline) in infos.iter().zip(pipelines.iter()) { + if pipeline == vk::Pipeline::null() { + // Can happen when VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT is set. + continue; + } + + let stages = unsafe { std::slice::from_raw_parts(ci.p_stages, ci.stage_count as usize) }; + + let mut vert_hash: Option = None; + let mut frag_hash: Option = None; + + for stage in stages { + let module_hash = ds.shader_registry.get(&stage.module.as_raw()).map(|r| *r); + + match stage.stage { + vk::ShaderStageFlags::VERTEX => vert_hash = module_hash, + vk::ShaderStageFlags::FRAGMENT => frag_hash = module_hash, + _ => {} + } + } + + // Phase 6: extract blend and depth state for discovery mode + let mut blend_enabled = false; + if !ci.p_color_blend_state.is_null() { + let blend = unsafe { &*ci.p_color_blend_state }; + for i in 0..blend.attachment_count as usize { + let attachment = unsafe { &*blend.p_attachments.add(i) }; + if attachment.blend_enable != vk::FALSE { + blend_enabled = true; + break; + } + } + } + + let mut depth_test_enabled = false; + let mut depth_write_enabled = false; + if !ci.p_depth_stencil_state.is_null() { + let ds_state = unsafe { &*ci.p_depth_stencil_state }; + depth_test_enabled = ds_state.depth_test_enable != vk::FALSE; + depth_write_enabled = ds_state.depth_write_enable != vk::FALSE; + } + + /*log::trace!( + "pipeline {:#010x} → vert={} frag={} blend={} depth_test={} depth_write={}", + pipeline.as_raw(), + vert_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + frag_hash + .map(|h| format!("{:#018x}", h)) + .unwrap_or_else(|| "none".to_string()), + blend_enabled, + depth_test_enabled, + depth_write_enabled, + );*/ + + ds.pipeline_registry.insert( + pipeline.as_raw(), + PipelineHashes { + vert_hash, + frag_hash, + }, + ); + + ds.pipeline_state.insert( + pipeline.as_raw(), + PipelineState { + blend_enabled, + depth_test_enabled, + depth_write_enabled, + }, + ); + } + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyPipeline( + device: vk::Device, + pipeline: vk::Pipeline, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + ds.pipeline_registry.remove(&pipeline.as_raw()); + ds.pipeline_state.remove(&pipeline.as_raw()); + unsafe { (ds.fp.destroy_pipeline)(device, pipeline, p_allocator) }; + } +} diff --git a/apps/nescapture/src/present.rs b/apps/nescapture/src/present.rs new file mode 100644 index 00000000..c08b5bbf --- /dev/null +++ b/apps/nescapture/src/present.rs @@ -0,0 +1,214 @@ +use crate::capture; +use crate::encode::{CapturedFrame, FrameSource, PipelineConfig, PipelineHandle}; +use crate::state::{DEVICE_STATE, QUEUE_TO_DEVICE_KEY}; +use ash::vk::{self, Handle}; +use std::os::raw::c_void; +use std::sync::atomic::Ordering; +use std::sync::mpsc; + +pub struct CaptureJob { + pub queue: vk::Queue, + pub sc_image: vk::Image, + pub sc_fmt: vk::Format, + pub sc_ext: vk::Extent2D, + pub frame: u64, + pub ds_key: usize, +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkQueuePresentKHR( + queue: vk::Queue, + p_present_info: *const vk::PresentInfoKHR, +) -> vk::Result { + let ds = { + let dk = unsafe { crate::dispatch_key(queue.as_raw() as *const c_void) }; + DEVICE_STATE + .get(&dk) + .map(|r| r.clone()) + .or_else(|| { + QUEUE_TO_DEVICE_KEY + .get(&queue.as_raw()) + .and_then(|dk| DEVICE_STATE.get(dk.value()).map(|r| r.clone())) + }) + .or_else(|| DEVICE_STATE.iter().next().map(|e| e.value().clone())) + }; + let ds = match ds { + Some(d) => d, + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let frame = ds.frame_counter.fetch_add(1, Ordering::Relaxed); + ds.hud_detected_frame.store(false, Ordering::Relaxed); + ds.pending_capture_frame.store(false, Ordering::Relaxed); + ds.capture_injected_frame.store(false, Ordering::Relaxed); + ds.skipped_draws_frame.store(0, Ordering::Relaxed); + + if let Ok(enc) = ds.encoder.lock() { + if let Some(ref h) = *enc { + h.present_attempts.fetch_add(1, Ordering::Relaxed); + } + } + + let pi = unsafe { &*p_present_info }; + if pi.swapchain_count > 0 && !pi.p_swapchains.is_null() && !pi.p_image_indices.is_null() { + let idx = unsafe { *pi.p_image_indices as usize }; + let (sc_image, sc_fmt, sc_ext) = { + let images = ds.swapchain_images.lock().unwrap(); + let fmt = *ds.swapchain_format.lock().unwrap(); + let ext = *ds.swapchain_extent.lock().unwrap(); + if idx < images.len() && ext.width > 0 && ext.height > 0 { + (Some(images[idx]), fmt, ext) + } else { + (None, fmt, ext) + } + }; + + if let Some(sc_image) = sc_image { + // No time-based throttle — let the encoder channel provide natural backpressure + let should = true; + + if should { + if let Ok(enc) = ds.encoder.lock() { + if let Some(ref h) = *enc { + h.capture_attempts.fetch_add(1, Ordering::Relaxed); + } + } + // Ensure capture worker is running + { + let mut ctx = ds.capture_tx.lock().unwrap(); + if ctx.is_none() { + let (tx, rx) = mpsc::channel(); + let key = unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) }; + start_capture_worker(key, rx); + *ctx = Some(tx); + } + } + // Queue job to worker thread — don't block present + let job = CaptureJob { + queue, + sc_image, + sc_fmt, + sc_ext, + frame, + ds_key: unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) }, + }; + if let Ok(capture_tx) = ds.capture_tx.lock() { + let _ = capture_tx.as_ref().unwrap().send(job); + } + } + } + } + + match ds.fp.queue_present_khr { + Some(f) => unsafe { f(queue, p_present_info) }, + None => vk::Result::ERROR_EXTENSION_NOT_PRESENT, + } +} + +pub fn start_capture_worker(ds_key: usize, capture_rx: mpsc::Receiver) { + std::thread::Builder::new() + .name("nescapture-capture".into()) + .spawn(move || { + while let Ok(job) = capture_rx.recv() { + let t0 = std::time::Instant::now(); + let ds = match DEVICE_STATE.get(&job.ds_key) { + Some(s) => s.clone(), + None => { + log::error!("capture worker: device state gone"); + break; + } + }; + // Do the blit on the worker's own time + unsafe { + capture::capture_final_frame( + &ds, + job.queue, + job.sc_image, + job.sc_fmt, + job.sc_ext, + job.frame, + ) + }; + + // Export DMA-BUF and push to encoder + let source = { + let mem_guard = ds.final_memory.lock().unwrap(); + let stride = ds.final_stride.load(Ordering::Relaxed); + if let Some(mem) = *mem_guard { + unsafe { try_make_dmabuf_source(&ds, mem, stride) }.unwrap_or_else(|| { + let (w, h, _) = *ds.final_size.lock().unwrap(); + let img = ds.final_image.lock().unwrap().unwrap(); + unsafe { capture::read_frame_pixels(&ds, img, mem, w, h) } + .map(FrameSource::Pixels) + .unwrap_or(FrameSource::Pixels(Vec::new())) + }) + } else { + continue; + } + }; + + let (w, h, _) = *ds.final_size.lock().unwrap(); + if !matches!(&source, FrameSource::Pixels(p) if p.is_empty()) { + // Lazy-init encoder + { + let mut enc = ds.encoder.lock().unwrap(); + if enc.is_none() { + if let Some(cfg) = PipelineConfig::from_env(w, h) { + ds.target_fps.store(cfg.fps, Ordering::Relaxed); + match PipelineHandle::new(cfg) { + Ok(h) => *enc = Some(h), + Err(e) => panic!("{e}"), + } + } + } + } + let enc_guard = ds.encoder.lock().unwrap(); + if let Some(ref encoder) = *enc_guard { + let capture_elapsed = t0.elapsed().as_secs_f32() * 1000.0; + encoder + .capture_ms + .store(capture_elapsed.to_bits(), Ordering::Relaxed); + encoder.push_frame(CapturedFrame { + source, + width: w, + height: h, + vk_format: job.sc_fmt.as_raw() as u32, + vk_colorspace: ds.swapchain_colorspace.load(Ordering::Relaxed), + }); + } + } + } + log::info!("capture worker exiting"); + }) + .ok(); +} + +unsafe fn try_make_dmabuf_source( + ds: &crate::state::DeviceState, + mem: vk::DeviceMemory, + stride: u32, +) -> Option { + let cached = ds.cached_dmabuf_fd.load(Ordering::Relaxed); + let fd = if cached >= 0 { + let duped = unsafe { libc::dup(cached) }; + if duped >= 0 { + duped + } else { + let fresh = unsafe { capture::get_dmabuf_fd(ds, mem)? }; + ds.cached_dmabuf_fd.store(fresh, Ordering::Relaxed); + unsafe { libc::dup(fresh) } + } + } else { + let fresh = unsafe { capture::get_dmabuf_fd(ds, mem)? }; + ds.cached_dmabuf_fd.store(fresh, Ordering::Relaxed); + unsafe { libc::dup(fresh) } + }; + if fd < 0 { + return None; + } + Some(FrameSource::DmaBuf { + fd, + stride, + modifier: 0, + }) +} diff --git a/apps/nescapture/src/shader.rs b/apps/nescapture/src/shader.rs new file mode 100644 index 00000000..91c9cefc --- /dev/null +++ b/apps/nescapture/src/shader.rs @@ -0,0 +1,92 @@ +// ───────────────────────────────────────────────────────────────────────────── +// shader.rs — Phase 1: shader module interception and SPIR-V fingerprinting +// +// When the game (or DXVK/VKD3D) calls vkCreateShaderModule, we: +// 1. Let the call through to the next layer / driver. +// 2. Hash the raw SPIR-V bytecode with SHA-256, truncated to 64 bits. +// 3. Store (VkShaderModule → hash) in DeviceState::shader_registry. +// +// vkDestroyShaderModule removes the entry to keep the map bounded. +// +// The hash is stable for a fixed (game version, DXVK version) pair. +// It is looked up in vkCreateGraphicsPipelines (pipeline.rs) to tag each +// pipeline with the hashes of its vertex and fragment shaders. +// ───────────────────────────────────────────────────────────────────────────── + +use crate::dispatch_key; +use crate::state::DEVICE_STATE; +use ash::vk::{self, Handle}; +use sha2::{Digest, Sha256}; +use std::os::raw::c_void; + +/// Compute a stable 64-bit fingerprint of a SPIR-V module. +/// +/// SPIR-V is a sequence of u32 words. We hash the raw bytes in native endian — +/// endianness consistency is all that matters since the hash is only compared +/// on the same machine within the same session. +/// +/// SHA-256 is collision-resistant. Truncating to 64 bits is safe because any +/// single game has hundreds of shaders at most, making collisions astronomically +/// unlikely. +fn hash_spirv(words: &[u32]) -> u64 { + let bytes: &[u8] = bytemuck::cast_slice(words); + let digest = Sha256::digest(bytes); + // Take the first 8 bytes as a little-endian u64. + u64::from_le_bytes(digest[..8].try_into().unwrap()) +} + +// ───────────────────────────────────────────────────────────────────────────── + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateShaderModule( + device: vk::Device, + p_create_info: *const vk::ShaderModuleCreateInfo, + p_allocator: *const vk::AllocationCallbacks, + p_shader_module: *mut vk::ShaderModule, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + // Call through first so the driver creates the real object. + let result = unsafe { + (ds.fp.create_shader_module)(device, p_create_info, p_allocator, p_shader_module) + }; + if result != vk::Result::SUCCESS { + return result; + } + + // Hash the SPIR-V bytecode. + let ci = unsafe { &*p_create_info }; + // code_size is in bytes; p_code points to u32 words. + let word_count = ci.code_size / std::mem::size_of::(); + let words = unsafe { std::slice::from_raw_parts(ci.p_code, word_count) }; + let hash = hash_spirv(words); + + let module = unsafe { *p_shader_module }; + ds.shader_registry.insert(module.as_raw(), hash); + + /*log::trace!( + "shader module {:#010x} → spir-v hash {:#018x} ({} words)", + module.as_raw(), + hash, + word_count, + );*/ + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroyShaderModule( + device: vk::Device, + shader_module: vk::ShaderModule, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + ds.shader_registry.remove(&shader_module.as_raw()); + unsafe { (ds.fp.destroy_shader_module)(device, shader_module, p_allocator) }; + } +} diff --git a/apps/nescapture/src/state.rs b/apps/nescapture/src/state.rs new file mode 100644 index 00000000..62e7b254 --- /dev/null +++ b/apps/nescapture/src/state.rs @@ -0,0 +1,155 @@ +// ───────────────────────────────────────────────────────────────────────────── +// state.rs — global state shared across all hooks +// ───────────────────────────────────────────────────────────────────────────── + +use crate::config::ShaderHashSet; +use crate::dispatch::NextDeviceFn; +use crate::encode::PipelineHandle; +use ash::vk; +use dashmap::DashMap; +use once_cell::sync::Lazy; +use std::sync::Arc; + +// Holds capture-specific resources instead of re-creating them *each frame* +pub struct CaptureResources { + pub command_pool: vk::CommandPool, + pub command_buffers: [vk::CommandBuffer; 4], + pub fences: [vk::Fence; 4], + pub current: usize, +} + +// ── Per-pipeline records ────────────────────────────────────────────────────── + +#[derive(Clone, Debug, Default)] +pub struct PipelineHashes { + pub vert_hash: Option, + pub frag_hash: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct PipelineState { + pub blend_enabled: bool, + pub depth_test_enabled: bool, + pub depth_write_enabled: bool, +} + +// ── Per-device state ────────────────────────────────────────────────────────── + +pub struct DeviceState { + pub raw: vk::Device, + pub physical_device: vk::PhysicalDevice, + pub fp: NextDeviceFn, + + // Phase 1: shader / pipeline + pub shader_registry: DashMap, + pub pipeline_registry: DashMap, + pub pipeline_state: DashMap, + + // Phase 2: view / framebuffer tracking + pub view_to_image: DashMap, + pub view_format: DashMap, + pub framebuffer_to_views: DashMap>, + pub framebuffer_extent: DashMap, + + // Phase 3: shader hash config + pub shader_hashes: Option, + + // Phase 4: HUDless capture (optional — HUDLESS_CAPTURE_HUDLESS=1) + pub hudless_image: std::sync::Mutex>, + pub hudless_memory: std::sync::Mutex>, + pub hudless_size: std::sync::Mutex<(u32, u32, vk::Format)>, + + // Phase 4: final-frame capture (DMA-BUF exportable) + pub final_image: std::sync::Mutex>, + pub final_memory: std::sync::Mutex>, + pub final_size: std::sync::Mutex<(u32, u32, vk::Format)>, + /// Row stride in bytes of `final_image`, queried once after allocation. + /// Zero until the first frame is captured. + pub final_stride: std::sync::atomic::AtomicU32, + + // Phase 4: swapchain tracking + pub swapchain: std::sync::Mutex>, + pub swapchain_images: std::sync::Mutex>, + pub swapchain_format: std::sync::Mutex, + /// Raw `VkColorSpaceKHR` value, captured from vkCreateSwapchainKHR. + /// Used to derive color space for the encoder (SDR vs HDR10 etc.). + pub swapchain_colorspace: std::sync::atomic::AtomicU32, + pub swapchain_extent: std::sync::Mutex, + + pub frame_counter: std::sync::atomic::AtomicU64, + pub largest_extent: std::sync::Mutex, + + // Phase 3/4: per-frame HUD detection flags + pub hud_detected_frame: std::sync::atomic::AtomicBool, + pub pending_capture_frame: std::sync::atomic::AtomicBool, + pub capture_injected_frame: std::sync::atomic::AtomicBool, + pub skipped_draws_frame: std::sync::atomic::AtomicU32, + + // Phase 7: encode + IPC pipeline (lazy-init on first frame) + pub encoder: std::sync::Mutex>, + + // Re-usable capture resources (double-buffered) + pub capture_resources: std::sync::Mutex>, + /// Dedicated queue for capture submissions (separate from game rendering). + pub capture_queue: std::sync::Mutex, + // Fake swapchain pool (headless — no real present) + pub fake_images: std::sync::Mutex>, + pub fake_memories: std::sync::Mutex>, + pub fake_fds: std::sync::Mutex>, + pub fake_strides: std::sync::Mutex>, + pub fake_available: std::sync::Mutex>, + pub fake_image_count: std::sync::atomic::AtomicU32, + pub fake_swapchain: std::sync::Mutex>, + pub signal_queue: std::sync::Mutex, + pub next_acquire: std::sync::atomic::AtomicU32, + pub memory_properties: std::sync::Mutex, + pub acquire_dummy_pool: std::sync::Mutex, + pub acquire_dummy_cb: std::sync::Mutex, + /// Cached DMA-BUF fd for final_memory (-1 = not cached). + /// Avoids a kernel ioctl per frame. + pub cached_dmabuf_fd: std::sync::atomic::AtomicI32, + + // ── Frame-rate throttle ─────────────────────────────────────────── + /// Target FPS for capture throttling (set from HUDLESS_FPS on pipeline init). + pub target_fps: std::sync::atomic::AtomicU32, + /// Timestamp of last captured frame (for rate limiting). + pub last_capture_time: std::sync::Mutex>, + /// Channel for threaded capture worker (present → worker). + pub capture_tx: std::sync::Mutex>>, +} + +// ── Per-command-buffer state ────────────────────────────────────────────────── + +#[derive(Default, Clone)] +pub struct CbState { + pub device_key: usize, + + pub current_color_image: Option, + pub current_image_format: Option, + pub current_image_extent: Option, + + pub active_vert_hash: Option, + pub active_frag_hash: Option, + pub hud_captured: bool, + + pub pending_capture: bool, + pub capture_injected: bool, + pub hud_detected: bool, + + pub draw_counter: u32, +} + +// ── Global state ────────────────────────────────────────────────────────────── + +pub static INSTANCE_STATE: Lazy>> = + Lazy::new(DashMap::new); + +pub static DEVICE_STATE: Lazy>> = Lazy::new(DashMap::new); + +pub static CB_STATE: Lazy> = Lazy::new(DashMap::new); + +/// VkQueue → device dispatch key +pub static QUEUE_TO_DEVICE_KEY: Lazy> = Lazy::new(DashMap::new); + +/// VkCommandBuffer → device dispatch key +pub static CMD_BUF_TO_DEVICE_KEY: Lazy> = Lazy::new(DashMap::new); diff --git a/apps/nescapture/src/swapchain.rs b/apps/nescapture/src/swapchain.rs new file mode 100644 index 00000000..46a9f491 --- /dev/null +++ b/apps/nescapture/src/swapchain.rs @@ -0,0 +1,121 @@ +// ───────────────────────────────────────────────────────────────────────────── +// swapchain.rs — Phase 4: swapchain tracking +// +// Captures format, extent AND color space from vkCreateSwapchainKHR so the +// encoder can automatically determine the correct color space pipeline +// (SDR/BT.709 vs HDR10/BT.2020-PQ vs FP16) without any manual configuration. +// ───────────────────────────────────────────────────────────────────────────── + +use crate::dispatch_key; +use crate::state::DEVICE_STATE; +use ash::vk::{self, Handle}; +use std::os::raw::c_void; +use std::sync::atomic::Ordering; + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkCreateSwapchainKHR( + device: vk::Device, + p_create_info: *const vk::SwapchainCreateInfoKHR, + p_allocator: *const vk::AllocationCallbacks, + p_swapchain: *mut vk::SwapchainKHR, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let create_fn = match ds.fp.create_swapchain_khr { + Some(f) => f, + None => return vk::Result::ERROR_EXTENSION_NOT_PRESENT, + }; + + let ci = unsafe { &*p_create_info }; + + // Add TRANSFER_SRC so we can blit from swapchain images. + let mut modified_ci = *ci; + modified_ci.image_usage = ci.image_usage | vk::ImageUsageFlags::TRANSFER_SRC; + + let result = unsafe { create_fn(device, &modified_ci, p_allocator, p_swapchain) }; + // If the driver rejects TRANSFER_SRC (e.g. composited window), try without. + let result = if result != vk::Result::SUCCESS { + unsafe { create_fn(device, ci, p_allocator, p_swapchain) } + } else { + result + }; + if result != vk::Result::SUCCESS { + return result; + } + + *ds.swapchain.lock().unwrap() = Some(unsafe { *p_swapchain }); + *ds.swapchain_format.lock().unwrap() = ci.image_format; + *ds.swapchain_extent.lock().unwrap() = ci.image_extent; + ds.swapchain_colorspace + .store(ci.image_color_space.as_raw() as u32, Ordering::Relaxed); + + log::debug!( + "swapchain created — format={:?} colorspace={:?} extent={}x{}", + ci.image_format, + ci.image_color_space, + ci.image_extent.width, + ci.image_extent.height, + ); + + vk::Result::SUCCESS +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkDestroySwapchainKHR( + device: vk::Device, + swapchain: vk::SwapchainKHR, + p_allocator: *const vk::AllocationCallbacks, +) { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + if let Some(ds) = DEVICE_STATE.get(&key) { + *ds.swapchain.lock().unwrap() = None; + *ds.swapchain_images.lock().unwrap() = Vec::new(); + if let Some(destroy_fn) = ds.fp.destroy_swapchain_khr { + unsafe { destroy_fn(device, swapchain, p_allocator) }; + } + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "system" fn vkGetSwapchainImagesKHR( + device: vk::Device, + swapchain: vk::SwapchainKHR, + p_swapchain_image_count: *mut u32, + p_swapchain_images: *mut vk::Image, +) -> vk::Result { + let key = unsafe { dispatch_key(device.as_raw() as *const c_void) }; + let ds = match DEVICE_STATE.get(&key) { + Some(s) => s.clone(), + None => return vk::Result::ERROR_DEVICE_LOST, + }; + + let get_fn = match ds.fp.get_swapchain_images_khr { + Some(f) => f, + None => return vk::Result::ERROR_EXTENSION_NOT_PRESENT, + }; + + let result = unsafe { + get_fn( + device, + swapchain, + p_swapchain_image_count, + p_swapchain_images, + ) + }; + if result != vk::Result::SUCCESS { + return result; + } + + if !p_swapchain_images.is_null() { + let count = unsafe { *p_swapchain_image_count as usize }; + let images = unsafe { std::slice::from_raw_parts(p_swapchain_images, count) }; + *ds.swapchain_images.lock().unwrap() = images.to_vec(); + log::debug!("swapchain images — {} images", count); + } + + vk::Result::SUCCESS +}