github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2026-09-19 17:25:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat(nescapture): open the capture layer

Browse Source 
A Vulkan implicit layer that captures frames from inside the workload's own
process and encodes them on the GPU they were drawn on. Fourth and last of this
batch, imported as a tree from `nestrilabs/nescapture` on the same terms.

Filed under `apps/` rather than `crates/` despite building a cdylib. The rule
here is what a thing *is*, not what it compiles to: this is a finished artefact
that gets installed into an image beside its layer manifest, not a library
another crate in this tree depends on. `crates/` is for the latter, and putting
this there would make the distinction useless the first time someone looked.

Wired to the workspace, `nesprotocol` by path. Its description named the
transport component; that reads better as what it actually is — where the frames
go — so it says that instead.

Whole workspace builds and tests: 21 across four members.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Wanjohi
2026-08-26 18:04:02 +03:00
parent 06b844b961
commit 6164e0c636
22 changed files with 5958 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

335
Cargo.lock generated
View File

@@ -92,6 +92,14 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "ash"
version = "0.38.0+1.4.329"
source = "git+https://github.com/ash-rs/ash?rev=55dd56906bbb5760e9e9e6c56f45be67f67e0649#55dd56906bbb5760e9e9e6c56f45be67f67e0649"
dependencies = [
"libloading",
]
[[package]] [[package]]
name = "atomic_float" name = "atomic_float"
version = "1.1.0" version = "1.1.0"
@@ -111,7 +119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [ dependencies = [
"annotate-snippets", "annotate-snippets",
"bitflags", "bitflags 2.13.1",
"cexpr", "cexpr",
"clang-sys", "clang-sys",
"itertools", "itertools",
@@ -123,6 +131,12 @@ dependencies = [
"syn 2.0.119", "syn 2.0.119",
] ]
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.13.1" version = "2.13.1"
@@ -164,7 +178,7 @@ version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"nix", "nix",
"polling", "polling",
"rustix 1.1.4", "rustix 1.1.4",
@@ -344,6 +358,51 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f"
[[package]]
name = "dashmap"
version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
name = "defmt"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
dependencies = [
"bitflags 1.3.2",
"defmt-macros",
]
[[package]]
name = "defmt-macros"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
dependencies = [
"defmt-parser",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "defmt-parser"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
dependencies = [
"thiserror 2.0.20",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.10.7" version = "0.10.7"
@@ -366,7 +425,7 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"bytemuck", "bytemuck",
"drm-ffi", "drm-ffi",
"drm-fourcc", "drm-fourcc",
@@ -415,6 +474,29 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "env_filter"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
dependencies = [
"log",
"regex",
]
[[package]]
name = "env_logger"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
dependencies = [
"anstream",
"anstyle",
"env_filter",
"jiff",
"log",
]
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.2" version = "1.0.2"
@@ -443,13 +525,28 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
[[package]]
name = "futures-channel"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
[[package]] [[package]]
name = "gbm" name = "gbm"
version = "0.18.0" version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"drm", "drm",
"drm-fourcc", "drm-fourcc",
"gbm-sys", "gbm-sys",
@@ -525,6 +622,12 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.17.1" version = "0.17.1"
@@ -556,7 +659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown", "hashbrown 0.17.1",
] ]
[[package]] [[package]]
@@ -565,7 +668,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbdc09524a91f9cacd26f16734ff63d7dc650daffadd2b6f84d17a285bd875a9" checksum = "fbdc09524a91f9cacd26f16734ff63d7dc650daffadd2b6f84d17a285bd875a9"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"input-sys", "input-sys",
"libc", "libc",
"log", "log",
@@ -604,6 +707,42 @@ dependencies = [
"either", "either",
] ]
[[package]]
name = "jiff"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc"
dependencies = [
"defmt",
"jiff-core",
"jiff-static",
"log",
"portable-atomic",
"portable-atomic-util",
"serde_core",
]
[[package]]
name = "jiff-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
dependencies = [
"defmt",
]
[[package]]
name = "jiff-static"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204"
dependencies = [
"jiff-core",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]] [[package]]
name = "khronos_api" name = "khronos_api"
version = "3.1.0" version = "3.1.0"
@@ -638,7 +777,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882f7427e7989dcc9d388b7f05c4630390a1d7696f9ffa469cd4a7a48f0b4c40" checksum = "882f7427e7989dcc9d388b7f05c4630390a1d7696f9ffa469cd4a7a48f0b4c40"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"cc", "cc",
"cookie-factory", "cookie-factory",
"libc", "libc",
@@ -687,6 +826,15 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.34" version = "0.4.34"
@@ -723,6 +871,26 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "nescapture"
version = "0.2.0"
dependencies = [
"anyhow",
"ash",
"bytemuck",
"dashmap",
"env_logger",
"libc",
"log",
"nesprotocol",
"once_cell",
"pixelforge",
"pollster",
"serde",
"sha2",
"toml 0.8.23",
]
[[package]] [[package]]
name = "nescope" name = "nescope"
version = "0.1.0" version = "0.1.0"
@@ -767,7 +935,7 @@ version = "0.31.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"cfg-if", "cfg-if",
"cfg_aliases", "cfg_aliases",
"libc", "libc",
@@ -831,6 +999,19 @@ dependencies = [
"cmake", "cmake",
] ]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]] [[package]]
name = "paste" name = "paste"
version = "1.0.15" version = "1.0.15"
@@ -849,7 +1030,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bde71084c4e25959d68f1ea54daa75e5ecdb338e5caf0b5510143b79baa32d5c" checksum = "bde71084c4e25959d68f1ea54daa75e5ecdb338e5caf0b5510143b79baa32d5c"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"libc", "libc",
"libspa", "libspa",
"libspa-sys", "libspa-sys",
@@ -868,6 +1049,17 @@ dependencies = [
"system-deps", "system-deps",
] ]
[[package]]
name = "pixelforge"
version = "0.7.1"
source = "git+https://github.com/hgaiser/pixelforge.git?rev=85c8654d383e194b984e27b319de96e4335e2b4f#85c8654d383e194b984e27b319de96e4335e2b4f"
dependencies = [
"ash",
"futures-channel",
"thiserror 1.0.69",
"tracing",
]
[[package]] [[package]]
name = "pixman" name = "pixman"
version = "0.2.1" version = "0.2.1"
@@ -906,6 +1098,27 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "pollster"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3"
[[package]]
name = "portable-atomic"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
[[package]]
name = "portable-atomic-util"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
dependencies = [
"portable-atomic",
]
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.21" version = "0.2.21"
@@ -1002,6 +1215,15 @@ dependencies = [
"getrandom 0.3.4", "getrandom 0.3.4",
] ]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags 2.13.1",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.13.1" version = "1.13.1"
@@ -1043,7 +1265,7 @@ version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.4.15", "linux-raw-sys 0.4.15",
@@ -1056,7 +1278,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.12.1", "linux-raw-sys 0.12.1",
@@ -1069,6 +1291,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]] [[package]]
name = "serde_core" name = "serde_core"
version = "1.0.229" version = "1.0.229"
@@ -1089,6 +1321,15 @@ dependencies = [
"syn 3.0.4", "syn 3.0.4",
] ]
[[package]]
name = "serde_spanned"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "serde_spanned" name = "serde_spanned"
version = "1.1.1" version = "1.1.1"
@@ -1150,7 +1391,7 @@ checksum = "740cea6927892bc182d5bf70c8f79806c8bc9f68f2fb96e55a30be171b63af98"
dependencies = [ dependencies = [
"appendlist", "appendlist",
"atomic_float", "atomic_float",
"bitflags", "bitflags 2.13.1",
"calloop", "calloop",
"cc", "cc",
"cgmath", "cgmath",
@@ -1223,7 +1464,7 @@ dependencies = [
"cfg-expr", "cfg-expr",
"heck", "heck",
"pkg-config", "pkg-config",
"toml", "toml 1.1.4+spec-1.1.0",
"version-compare", "version-compare",
] ]
@@ -1295,6 +1536,18 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "toml"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
"toml_edit",
]
[[package]] [[package]]
name = "toml" name = "toml"
version = "1.1.4+spec-1.1.0" version = "1.1.4+spec-1.1.0"
@@ -1303,11 +1556,20 @@ checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"serde_core", "serde_core",
"serde_spanned", "serde_spanned 1.1.1",
"toml_datetime", "toml_datetime 1.1.1+spec-1.1.0",
"toml_parser", "toml_parser",
"toml_writer", "toml_writer",
"winnow", "winnow 1.0.4",
]
[[package]]
name = "toml_datetime"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
] ]
[[package]] [[package]]
@@ -1319,15 +1581,35 @@ dependencies = [
"serde_core", "serde_core",
] ]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
"toml_write",
"winnow 0.7.15",
]
[[package]] [[package]]
name = "toml_parser" name = "toml_parser"
version = "1.1.3+spec-1.1.0" version = "1.1.3+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
dependencies = [ dependencies = [
"winnow", "winnow 1.0.4",
] ]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]] [[package]]
name = "toml_writer" name = "toml_writer"
version = "1.1.2+spec-1.1.0" version = "1.1.2+spec-1.1.0"
@@ -1478,7 +1760,7 @@ version = "0.31.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"rustix 1.1.4", "rustix 1.1.4",
"wayland-backend", "wayland-backend",
"wayland-scanner", "wayland-scanner",
@@ -1490,7 +1772,7 @@ version = "0.32.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"wayland-backend", "wayland-backend",
"wayland-client", "wayland-client",
"wayland-scanner", "wayland-scanner",
@@ -1503,7 +1785,7 @@ version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"wayland-backend", "wayland-backend",
"wayland-protocols", "wayland-protocols",
"wayland-scanner", "wayland-scanner",
@@ -1516,7 +1798,7 @@ version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"wayland-backend", "wayland-backend",
"wayland-protocols", "wayland-protocols",
"wayland-scanner", "wayland-scanner",
@@ -1540,7 +1822,7 @@ version = "0.31.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75" checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.13.1",
"downcast-rs", "downcast-rs",
"rustix 1.1.4", "rustix 1.1.4",
"wayland-backend", "wayland-backend",
@@ -1710,6 +1992,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "1.0.4" version = "1.0.4"

1
Cargo.toml
View File

@@ -13,6 +13,7 @@
[workspace] [workspace]
resolver = "3" resolver = "3"
members = [ members = [
"apps/nescapture",
"apps/nescope", "apps/nescope",
"apps/neswire", "apps/neswire",
"crates/nesprotocol", "crates/nesprotocol",

4
apps/nescapture/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.vscode/
.idea/
.zed/
/target/

45
apps/nescapture/Cargo.toml Normal file
View File

@@ -0,0 +1,45 @@
[package]
name = "nescapture"
version = "0.2.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "Vulkan implicit layer: direct frame capture, Vulkan Video encode, and IPC out to the session transport"
[lib]
name = "nescapture_layer"
crate-type = ["cdylib"]
[dependencies]
# Vulkan bindings
ash = { git = "https://github.com/ash-rs/ash", rev = "55dd56906bbb5760e9e9e6c56f45be67f67e0649" }
# Shader fingerprinting
sha2 = "0.10"
bytemuck = "1"
# Concurrent state maps
dashmap = "6"
once_cell = "1"
pollster = "0.4.0"
# Logging
log = "0.4"
env_logger = "0.11"
anyhow = "1"
# Per-game shader-hash config
toml = "0.8"
serde = { version = "1", features = ["derive"] }
# Vulkan Video hardware encoding.
pixelforge = { git = "https://github.com/hgaiser/pixelforge.git", rev = "85c8654d383e194b984e27b319de96e4335e2b4f", features = ["dmabuf"] }
# libc for DMA-BUF OS primitives
libc = "0.2"
# Shared IPC protocol
nesprotocol = { path = "../../crates/nesprotocol" }
[profile.release]
opt-level = 3
lto = "thin"

209
apps/nescapture/README.md Normal file
View File

@@ -0,0 +1,209 @@
# nescapture
A Vulkan implicit layer that captures frames from a running game, encodes them
with **Vulkan Video** hardware acceleration (H.264 / H.265 / AV1), packetizes with
Reed-Solomon FEC, and streams over RTP/UDP to a Moonlight-compatible client —
all with **zero CPU copies** — fully GPU from game rendering to RTP output.
---
## Architecture
```
Game process
│ Vulkan calls
▼
┌───────────────────────────────────────────────┐
│ nescapture Vulkan implicit layer │
│ │
│ vkCreateShaderModule → SHA-256 hash │
│ vkCreateGraphicsPipelines → track hashes │
│ vkCmdBindPipeline → detect HUD shaders │
│ vkCmdEndRenderPass/Rendering → inject copy │
│ vkQueuePresentKHR → capture + encode │
└───────────────────────────────────────────────┘
│ GPU blit (same device)
▼
final_image (DMA-BUF exportable)
│ get_dmabuf_fd(final_memory)
▼
DmaBufImporter (pixelforge VkDevice)
│ import_or_reuse() → vk::Image
▼
ColorConverter (GPU compute shader)
│ BGRA/RGB10/FP16 → NV12/P010/YUV444
▼
Encoder (Vulkan Video, hardware H.264/H.265)
│ encode()
▼
EncodedPacket (Annex-B)
│
▼
Packetizer (Moonlight wire format, Reed-Solomon FEC)
│ UDP datagrams
▼
Moonlight client (or any RTP/UDP receiver)
CPU fallback: only when DMA-BUF export unavailable (rare driver config)
```
---
## Quick start
```bash
# 1. Build
cargo build --release
# 2. Install the layer manifest
sudo cp manifest/VK_LAYER_nescapture.json /usr/share/vulkan/implicit_layer.d/
# Edit the manifest's `library_path` to point at target/release/libnescapture.so
# 3. Configure and launch a game
export NESCAPTURE_ENABLE=1
export NESCAPTURE_RTP_HOST=192.168.1.50 # Moonlight / receiver IP
export NESCAPTURE_RTP_PORT=47998 # default
export NESCAPTURE_CODEC=h265 # h264 | h265 | av1 (auto-probes if unset)
export NESCAPTURE_BITRATE=10000 # kbps (CBR; ignored if NESCAPTURE_QP is set)
export NESCAPTURE_FPS=60
export RUST_LOG=info # or NESCAPTURE_LOG=debug
wine MyGame.exe # or native Vulkan game
```
---
## Environment variables
| Variable | Default | Description |
| ------------------------- | -------------- | ------------------------------------------------------ |
| `NESCAPTURE_ENABLE` | _(unset)_ | Set to `1` to activate the layer |
| `NESCAPTURE_RTP_HOST` | _(required)_ | Destination IP / hostname for RTP stream |
| `NESCAPTURE_RTP_PORT` | `47998` | Destination UDP port |
| `NESCAPTURE_CODEC` | auto | `h264`, `h265` or `av1` — falls back to h264 if unsupported |
| `NESCAPTURE_BITRATE` | `10000` | CBR target bitrate in kbps |
| `NESCAPTURE_QP` | _(unset)_ | If set, use CQP with this quality level instead of CBR |
| `NESCAPTURE_FPS` | `60` | Target frame rate |
| `NESCAPTURE_IDR_INTERVAL` | `120` | Force an IDR keyframe every N frames |
| `NESCAPTURE_FEC_PCT` | `20` | Reed-Solomon FEC percentage |
| `NESCAPTURE_MIN_FEC` | `2` | Minimum FEC packets per block |
| `NESCAPTURE_PACKET_SIZE` | `1392` | Max UDP payload size (bytes) |
| `NESCAPTURE_CTRL_PORT` | `47999` | UDP port for the control stream |
| `NESCAPTURE_CAPTURE_HUDLESS` | _(unset)_ | Set to `1` to also capture HUDless frames |
| `NESCAPTURE_CONFIG` | _(unset)_ | Path to per-game shader-hash TOML config |
| `NESCAPTURE_GAME_NAME` | (exe basename) | Override game identification |
| `NESCAPTURE_DISCOVER` | _(unset)_ | Set to `1` to enable discovery mode (logs all draws) |
| `NESCAPTURE_LOG` | `info` | Log level (`error`, `warn`, `info`, `debug`, `trace`) |
| `NESCAPTURE_RTP_FORMAT` | `moonlight` | `standard` for RFC 6184/7798 RTP (GStreamer/FFmpeg compatible), `moonlight` for Moonlight wire format with FEC |
---
## Control stream
Send single-byte UDP datagrams to `NESCAPTURE_CTRL_PORT` (default 47999):
| Byte | Command |
| ------ | -------------------- |
| `0x01` | Start streaming |
| `0x02` | Stop streaming |
| `0x03` | Request IDR keyframe |
Example with netcat:
```bash
# Request IDR
printf '\x03' | nc -u -q1 localhost 47999
```
The control handle wiring in `present.rs` is currently a commented-out stub.
See `control.rs` for the full wiring instructions.
---
## Per-game shader-hash config
HUD shader detection requires a per-game config. Run the game once with
`NESCAPTURE_DISCOVER=1` to log all shaders, then identify HUD pipelines by the
`[SUSPECT]` marker (blend=true, depth=false, ≤6 vertices).
```toml
# ~/.config/nescapture/games.toml
[game."GameName.exe"]
hud_fragment_shaders = ["0xaabbccddeeff0011"]
hud_vertex_shaders = ["0x1a2b3c4d5e6f7890"]
skip_fragment_shaders = ["0x1122334455667788"]
```
```bash
export NESCAPTURE_CONFIG=~/.config/nescapture/games.toml
export NESCAPTURE_GAME_NAME=GameName.exe
```
---
## Dependencies
| Crate | Purpose |
| ---------------------- | -------------------------------------------- |
| `pixelforge` | Vulkan Video hardware encode (H.264 / H.265) |
| `ash` | Vulkan bindings |
| `reed-solomon-erasure` | FEC for RTP packetizer |
| `sha2` + `bytemuck` | SPIR-V shader fingerprinting |
| `dashmap` | Lock-free concurrent state maps |
| `serde` + `toml` | Per-game shader config |
---
## Zero-copy GPU pipeline
The full GPU path is implemented — no CPU color conversion:
1. `final_image` allocated with `DMA_BUF_EXT` external memory (`capture.rs`).
2. `get_dmabuf_fd()` exports the DMA-BUF fd (`capture.rs`).
3. `DmaBufImporter::import_or_reuse()` imports the fd as a `vk::Image` (`dmabuf_import.rs`).
4. `ColorConverter::convert()` runs a GPU compute shader:
BGRA/RGBA/RGB10/FP16 → NV12/P010/YUV444 (`encode.rs`).
5. `Encoder::encode()` encodes from the converter's output directly.
6. RTP packetizer + UDP send.
CPU fallback exists only for rare driver configurations that don't support
DMA-BUF external memory export.
---
## File structure
```
nescapture/
├── Cargo.toml
├── README.md
├── ARCHITECTURE.md
├── manifest/
│ └── VK_LAYER_nescapture.json
└── src/
├── lib.rs — entry points, dispatch routing
├── dispatch.rs — Vulkan function-pointer types + tables
├── state.rs — global DashMaps, DeviceState, CbState
├── instance.rs — vkCreateInstance / vkDestroyInstance
├── device.rs — vkCreateDevice / vkDestroyDevice
├── shader.rs — SPIR-V hashing
├── pipeline.rs — graphics pipeline tracking
├── framebuffer.rs — image view / framebuffer tracking
├── commands.rs — vkCmdBind*, vkCmdDraw*, vkCmdBeginRenderPass
├── swapchain.rs — vkCreateSwapchainKHR, image enumeration
├── capture.rs — GPU blit to capture image, DMA-BUF export
├── present.rs — vkQueuePresentKHR, encode dispatch
├── encode.rs — pixelforge pipeline, codec probing, RTP send
├── dmabuf_import.rs — DmaBufImporter (cross-device zero-copy import)
├── packetizer.rs — Moonlight RTP packetizer (Reed-Solomon FEC)
├── shard_batch.rs — zero-alloc shard buffer
├── control.rs — UDP control stream (IDR / start / stop)
├── config.rs — per-game TOML shader-hash config
└── discovery.rs — draw-call logging for shader discovery
```
---
## License
TBD

22
apps/nescapture/manifest/VK_LAYER_nescapture.json Normal file
View File

@@ -0,0 +1,22 @@
{
"file_format_version": "1.2.0",
"layer": {
"name": "VK_LAYER_nescapture",
"type": "GLOBAL",
"library_path": "/usr/lib/libnescapture_layer.so",
"api_version": "1.3.290",
"implementation_version": "1",
"description": "direct game frame capture for Nestri",
"functions": {
"vkGetInstanceProcAddr": "vkGetInstanceProcAddr",
"vkGetDeviceProcAddr": "vkGetDeviceProcAddr",
"vkNegotiateLoaderLayerInterfaceVersion": "vkNegotiateLoaderLayerInterfaceVersion"
},
"enable_environment": {
"NESCAPTURE_ENABLE": "1"
},
"disable_environment": {
"DISABLE_NESCAPTURE": "1"
}
}
}

737
apps/nescapture/src/capture.rs Normal file
View File

@@ -0,0 +1,737 @@
// ─────────────────────────────────────────────────────────────────────────────
// capture.rs — Frame capture helpers
//
// final_image is allocated with VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT
// so that after the GPU blit we can export an fd and import it into pixelforge's
// separate VkDevice for zero-copy hardware encoding via DmaBufImporter.
//
// After ensure_final_image allocates (or re-allocates) the image, we query
// its SubresourceLayout and cache the row stride in DeviceState::final_stride.
// The stride is needed by the encoder to correctly import the LINEAR image.
// ─────────────────────────────────────────────────────────────────────────────
use crate::state::{CB_STATE, CaptureResources, DEVICE_STATE};
use ash::vk::{self, Handle};
use std::os::raw::c_int;
use std::sync::atomic::Ordering;
fn make_subresource_range() -> vk::ImageSubresourceRange {
vk::ImageSubresourceRange {
aspect_mask: vk::ImageAspectFlags::COLOR,
base_mip_level: 0,
level_count: 1,
base_array_layer: 0,
layer_count: 1,
}
}
fn make_subresource_layers() -> vk::ImageSubresourceLayers {
vk::ImageSubresourceLayers {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
base_array_layer: 0,
layer_count: 1,
}
}
macro_rules! image_barrier {
($src:expr, $dst:expr, $old:expr, $new:expr, $img:expr) => {
vk::ImageMemoryBarrier {
s_type: vk::StructureType::IMAGE_MEMORY_BARRIER,
p_next: std::ptr::null(),
src_access_mask: $src,
dst_access_mask: $dst,
old_layout: $old,
new_layout: $new,
src_queue_family_index: vk::QUEUE_FAMILY_IGNORED,
dst_queue_family_index: vk::QUEUE_FAMILY_IGNORED,
image: $img,
subresource_range: make_subresource_range(),
_marker: std::marker::PhantomData,
}
};
}
// ── Memory helper ─────────────────────────────────────────────────────────────
unsafe fn find_host_coherent_mt(ds: &crate::state::DeviceState, bits: u32) -> u32 {
let mut mp = vk::PhysicalDeviceMemoryProperties::default();
let k = unsafe { crate::dispatch_key(ds.physical_device.as_raw() as *const std::ffi::c_void) };
if let Some(i) = crate::state::INSTANCE_STATE.get(&k) {
unsafe { (i.get_physical_device_memory_properties)(ds.physical_device, &mut mp) };
}
(0..mp.memory_type_count)
.find(|&i| {
(bits & (1 << i)) != 0
&& mp.memory_types[i as usize].property_flags.contains(
vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT,
)
})
.unwrap_or(0)
}
// ── Image allocators ──────────────────────────────────────────────────────────
/// Plain HOST_VISIBLE image (nescapture capture — no cross-device sharing needed).
unsafe fn allocate_host_image(
ds: &crate::state::DeviceState,
w: u32,
h: u32,
fmt: vk::Format,
label: &str,
) -> Option<(vk::Image, vk::DeviceMemory)> {
let ci = vk::ImageCreateInfo {
s_type: vk::StructureType::IMAGE_CREATE_INFO,
p_next: std::ptr::null(),
flags: vk::ImageCreateFlags::empty(),
image_type: vk::ImageType::TYPE_2D,
format: fmt,
extent: vk::Extent3D {
width: w,
height: h,
depth: 1,
},
mip_levels: 1,
array_layers: 1,
samples: vk::SampleCountFlags::TYPE_1,
tiling: vk::ImageTiling::LINEAR,
usage: vk::ImageUsageFlags::TRANSFER_DST,
sharing_mode: vk::SharingMode::EXCLUSIVE,
queue_family_index_count: 0,
p_queue_family_indices: std::ptr::null(),
initial_layout: vk::ImageLayout::UNDEFINED,
_marker: std::marker::PhantomData,
};
unsafe { alloc_image(ds, &ci, None, label) }
}
/// DMA-BUF exportable image (final capture — imported into pixelforge for encoding).
///
/// Falls back to a plain host image if the driver rejects external memory.
/// In that case `get_dmabuf_fd` will return `None` and the encoder will use
/// the CPU pixel-readback fallback.
unsafe fn allocate_dmabuf_image(
ds: &crate::state::DeviceState,
w: u32,
h: u32,
fmt: vk::Format,
label: &str,
) -> Option<(vk::Image, vk::DeviceMemory)> {
let ext_img = vk::ExternalMemoryImageCreateInfo {
s_type: vk::StructureType::EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
p_next: std::ptr::null_mut(),
handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
let ci = vk::ImageCreateInfo {
s_type: vk::StructureType::IMAGE_CREATE_INFO,
p_next: &ext_img as *const _ as *const _,
flags: vk::ImageCreateFlags::empty(),
image_type: vk::ImageType::TYPE_2D,
format: fmt,
extent: vk::Extent3D {
width: w,
height: h,
depth: 1,
},
mip_levels: 1,
array_layers: 1,
samples: vk::SampleCountFlags::TYPE_1,
tiling: vk::ImageTiling::LINEAR,
usage: vk::ImageUsageFlags::TRANSFER_DST,
sharing_mode: vk::SharingMode::EXCLUSIVE,
queue_family_index_count: 0,
p_queue_family_indices: std::ptr::null(),
initial_layout: vk::ImageLayout::UNDEFINED,
_marker: std::marker::PhantomData,
};
let export_ai = vk::ExportMemoryAllocateInfo {
s_type: vk::StructureType::EXPORT_MEMORY_ALLOCATE_INFO,
p_next: std::ptr::null_mut(),
handle_types: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
if let Some(r) = unsafe { alloc_image(ds, &ci, Some(&export_ai), label) } {
return Some(r);
}
log::warn!(
"DMA-BUF alloc failed for '{}' — using plain host image. \
Zero-copy GPU path will be unavailable; CPU readback fallback active.",
label
);
unsafe { allocate_host_image(ds, w, h, fmt, label) }
}
unsafe fn alloc_image(
ds: &crate::state::DeviceState,
ci: &vk::ImageCreateInfo,
export: Option<&vk::ExportMemoryAllocateInfo>,
label: &str,
) -> Option<(vk::Image, vk::DeviceMemory)> {
let mut image = vk::Image::null();
if unsafe { (ds.fp.create_image)(ds.raw, ci, std::ptr::null(), &mut image) }
!= vk::Result::SUCCESS
{
return None;
}
let mut mr = vk::MemoryRequirements {
size: 0,
alignment: 0,
memory_type_bits: 0,
};
unsafe { (ds.fp.get_image_memory_requirements)(ds.raw, image, &mut mr) };
let mt = unsafe { find_host_coherent_mt(ds, mr.memory_type_bits) };
let p_next: *const _ = match export {
Some(e) => e as *const _ as *const _,
None => std::ptr::null(),
};
let ai = vk::MemoryAllocateInfo {
s_type: vk::StructureType::MEMORY_ALLOCATE_INFO,
p_next,
allocation_size: mr.size,
memory_type_index: mt,
_marker: std::marker::PhantomData,
};
let mut mem = vk::DeviceMemory::null();
if unsafe { (ds.fp.allocate_memory)(ds.raw, &ai, std::ptr::null(), &mut mem) }
!= vk::Result::SUCCESS
{
unsafe { (ds.fp.destroy_image)(ds.raw, image, std::ptr::null()) };
return None;
}
if unsafe { (ds.fp.bind_image_memory)(ds.raw, image, mem, 0) } != vk::Result::SUCCESS {
unsafe { (ds.fp.free_memory)(ds.raw, mem, std::ptr::null()) };
unsafe { (ds.fp.destroy_image)(ds.raw, image, std::ptr::null()) };
return None;
}
log::info!(
"alloc {} {}x{} fmt={} ({} bytes)",
label,
ci.extent.width,
ci.extent.height,
ci.format.as_raw(),
mr.size
);
Some((image, mem))
}
// ── Stride query ──────────────────────────────────────────────────────────────
/// Query and cache the row stride of final_image.
/// Returns stride in bytes; 0 on failure.
pub unsafe fn query_and_cache_final_stride(
ds: &crate::state::DeviceState,
image: vk::Image,
) -> u32 {
let subresource = vk::ImageSubresource {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
array_layer: 0,
};
let mut layout = vk::SubresourceLayout {
offset: 0,
size: 0,
row_pitch: 0,
array_pitch: 0,
depth_pitch: 0,
};
unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &subresource, &mut layout) };
let stride = layout.row_pitch as u32;
ds.final_stride.store(stride, Ordering::Relaxed);
stride
}
// ── DMA-BUF fd export ─────────────────────────────────────────────────────────
/// Export `memory` as a DMA-BUF fd via vkGetMemoryFdKHR.
/// Callers own the fd and must close it when done.
/// Returns `None` if VK_KHR_external_memory_fd is unavailable.
pub unsafe fn get_dmabuf_fd(
ds: &crate::state::DeviceState,
memory: vk::DeviceMemory,
) -> Option<c_int> {
let f = match ds.fp.get_memory_fd_khr {
Some(f) => f,
None => {
log::warn!("get_dmabuf_fd: vkGetMemoryFdKHR not available");
return None;
}
};
let fi = vk::MemoryGetFdInfoKHR {
s_type: vk::StructureType::MEMORY_GET_FD_INFO_KHR,
p_next: std::ptr::null(),
memory,
handle_type: vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
_marker: std::marker::PhantomData,
};
let mut fd: c_int = -1;
let result = unsafe { f(ds.raw, &fi, &mut fd) };
if result == vk::Result::SUCCESS && fd >= 0 {
Some(fd)
} else {
log::warn!("get_dmabuf_fd failed: result={:?} fd={}", result, fd);
None
}
}
// ── ensure helpers ────────────────────────────────────────────────────────────
pub unsafe fn ensure_hudless_image(ds: &crate::state::DeviceState, w: u32, h: u32, f: vk::Format) {
let mut ig = ds.hudless_image.lock().unwrap();
let mut mg = ds.hudless_memory.lock().unwrap();
let mut sg = ds.hudless_size.lock().unwrap();
if let (Some(i), Some(m)) = (*ig, *mg) {
let (ew, eh, ef) = *sg;
if ew >= w && eh >= h && ef == f {
return;
}
unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) };
unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) };
*ig = None;
*mg = None;
}
if let Some((i, m)) = unsafe { allocate_host_image(ds, w, h, f, "nescapture") } {
*ig = Some(i);
*mg = Some(m);
*sg = (w, h, f);
}
}
pub unsafe fn ensure_final_image(ds: &crate::state::DeviceState, w: u32, h: u32, f: vk::Format) {
let mut ig = ds.final_image.lock().unwrap();
let mut mg = ds.final_memory.lock().unwrap();
let mut sg = ds.final_size.lock().unwrap();
if let (Some(i), Some(m)) = (*ig, *mg) {
let (ew, eh, ef) = *sg;
if ew >= w && eh >= h && ef == f {
return;
}
unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) };
unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) };
*ig = None;
*mg = None;
ds.final_stride.store(0, Ordering::Relaxed);
}
if let Some((i, m)) = unsafe { allocate_dmabuf_image(ds, w, h, f, "final") } {
// Query stride immediately after allocation so it's available on first frame.
unsafe { query_and_cache_final_stride(ds, i) };
*ig = Some(i);
*mg = Some(m);
*sg = (w, h, f);
}
}
// ── HUDless command injection ─────────────────────────────────────────────────
pub unsafe fn inject_hudless_copy(cb: vk::CommandBuffer, dk: usize) {
let ds = match DEVICE_STATE.get(&dk) {
Some(s) => s.clone(),
None => return,
};
let cbk = cb.as_raw();
let cs = match CB_STATE.get(&cbk) {
Some(e) => e.value().clone(),
None => return,
};
let ci = match cs.current_color_image {
Some(i) => i,
None => return,
};
let fmt = match cs.current_image_format {
Some(f) => f,
None => return,
};
let ext = match cs.current_image_extent {
Some(e) => e,
None => return,
};
let sc = *ds.swapchain_extent.lock().unwrap();
if sc.width > 0 && sc.height > 0 && (ext.width != sc.width || ext.height != sc.height) {
return;
}
unsafe { ensure_hudless_image(&ds, ext.width, ext.height, fmt) };
let (hi, _) = {
let a = ds.hudless_image.lock().unwrap();
let b = ds.hudless_memory.lock().unwrap();
match (*a, *b) {
(Some(i), Some(m)) => (i, m),
_ => return,
}
};
// src → TRANSFER_SRC
let b1 = image_barrier!(
vk::AccessFlags::COLOR_ATTACHMENT_WRITE,
vk::AccessFlags::TRANSFER_READ,
vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
ci
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT,
vk::PipelineStageFlags::TRANSFER,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b1,
);
}
// dst → TRANSFER_DST
let b2 = image_barrier!(
vk::AccessFlags::empty(),
vk::AccessFlags::TRANSFER_WRITE,
vk::ImageLayout::UNDEFINED,
vk::ImageLayout::TRANSFER_DST_OPTIMAL,
hi
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::TOP_OF_PIPE,
vk::PipelineStageFlags::TRANSFER,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b2,
);
}
let cr = vk::ImageCopy {
src_subresource: make_subresource_layers(),
src_offset: vk::Offset3D { x: 0, y: 0, z: 0 },
dst_subresource: make_subresource_layers(),
dst_offset: vk::Offset3D { x: 0, y: 0, z: 0 },
extent: vk::Extent3D {
width: ext.width,
height: ext.height,
depth: 1,
},
};
unsafe {
(ds.fp.cmd_copy_image)(
cb,
ci,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
hi,
vk::ImageLayout::TRANSFER_DST_OPTIMAL,
1,
&cr,
);
}
// restore src
let b3 = image_barrier!(
vk::AccessFlags::TRANSFER_READ,
vk::AccessFlags::COLOR_ATTACHMENT_WRITE,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
ci
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::TRANSFER,
vk::PipelineStageFlags::COLOR_ATTACHMENT_OUTPUT,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b3,
);
}
if let Some(mut s) = CB_STATE.get_mut(&cb.as_raw()) {
s.pending_capture = false;
s.capture_injected = true;
}
}
// ── Final frame GPU blit (swapchain → final_image) ────────────────────────────
pub unsafe fn capture_final_frame(
ds: &crate::state::DeviceState,
queue: vk::Queue,
si: vk::Image,
fmt: vk::Format,
ext: vk::Extent2D,
_frame: u64,
) {
if ext.width == 0 || ext.height == 0 {
return;
}
unsafe { ensure_final_image(ds, ext.width, ext.height, fmt) };
let fi = match *ds.final_image.lock().unwrap() {
Some(i) => i,
None => return,
};
// ── Lazy-init reusable capture resources ──────────────────────
let mut res_guard = ds.capture_resources.lock().unwrap();
let res = match res_guard.as_mut() {
Some(r) => r,
None => match unsafe { create_capture_resources(ds) } {
Some(r) => {
*res_guard = Some(r);
res_guard.as_mut().unwrap()
}
None => return,
},
};
let idx = res.current;
let cb = res.command_buffers[idx];
let fence = res.fences[idx];
// Wait for THIS slot's previous use to finish (not the other slot).
// Use a short timeout — if the GPU is busy with game rendering, skip
// this capture instead of stalling the game's render loop.
unsafe {
let result = (ds.fp.wait_for_fences)(ds.raw, 1, &fence, vk::TRUE, 1_000_000); // 1ms timeout
if result != vk::Result::SUCCESS {
// GPU not ready — skip this capture, try next slot
res.current = (idx + 1) % 4;
return;
}
let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence);
}
// Reset and re-record
unsafe {
let _ = (ds.fp.reset_command_buffer)(cb, vk::CommandBufferResetFlags::empty());
}
let bi = vk::CommandBufferBeginInfo {
s_type: vk::StructureType::COMMAND_BUFFER_BEGIN_INFO,
p_next: std::ptr::null(),
flags: vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT,
p_inheritance_info: std::ptr::null(),
_marker: std::marker::PhantomData,
};
if unsafe { (ds.fp.begin_command_buffer)(cb, &bi) } != vk::Result::SUCCESS {
return;
}
let b1 = image_barrier!(
vk::AccessFlags::MEMORY_READ,
vk::AccessFlags::TRANSFER_READ,
vk::ImageLayout::PRESENT_SRC_KHR,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
si
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::BOTTOM_OF_PIPE,
vk::PipelineStageFlags::TRANSFER,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b1,
);
}
let b2 = image_barrier!(
vk::AccessFlags::empty(),
vk::AccessFlags::TRANSFER_WRITE,
vk::ImageLayout::UNDEFINED,
vk::ImageLayout::TRANSFER_DST_OPTIMAL,
fi
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::TOP_OF_PIPE,
vk::PipelineStageFlags::TRANSFER,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b2,
);
}
let cr = vk::ImageCopy {
src_subresource: make_subresource_layers(),
src_offset: vk::Offset3D { x: 0, y: 0, z: 0 },
dst_subresource: make_subresource_layers(),
dst_offset: vk::Offset3D { x: 0, y: 0, z: 0 },
extent: vk::Extent3D {
width: ext.width,
height: ext.height,
depth: 1,
},
};
unsafe {
(ds.fp.cmd_copy_image)(
cb,
si,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
fi,
vk::ImageLayout::TRANSFER_DST_OPTIMAL,
1,
&cr,
);
}
let b3 = image_barrier!(
vk::AccessFlags::TRANSFER_READ,
vk::AccessFlags::MEMORY_READ,
vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
vk::ImageLayout::PRESENT_SRC_KHR,
si
);
unsafe {
(ds.fp.cmd_pipeline_barrier)(
cb,
vk::PipelineStageFlags::TRANSFER,
vk::PipelineStageFlags::TOP_OF_PIPE,
vk::DependencyFlags::empty(),
0,
std::ptr::null(),
0,
std::ptr::null(),
1,
&b3,
);
}
if unsafe { (ds.fp.end_command_buffer)(cb) } != vk::Result::SUCCESS {
return;
}
let subi = vk::SubmitInfo {
s_type: vk::StructureType::SUBMIT_INFO,
p_next: std::ptr::null(),
wait_semaphore_count: 0,
p_wait_semaphores: std::ptr::null(),
p_wait_dst_stage_mask: std::ptr::null(),
command_buffer_count: 1,
p_command_buffers: &cb,
signal_semaphore_count: 0,
p_signal_semaphores: std::ptr::null(),
_marker: std::marker::PhantomData,
};
unsafe {
if (ds.fp.queue_submit)(queue, 1, &subi, fence) != vk::Result::SUCCESS {
log::warn!("capture queue_submit failed — frame skipped");
let _ = (ds.fp.reset_fences)(ds.raw, 1, &fence);
return;
}
}
// Toggle to the next slot
res.current = (idx + 1) % 4;
}
unsafe fn create_capture_resources(ds: &crate::state::DeviceState) -> Option<CaptureResources> {
let pci = vk::CommandPoolCreateInfo {
s_type: vk::StructureType::COMMAND_POOL_CREATE_INFO,
p_next: std::ptr::null(),
flags: vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER, // allow per-cb reset
queue_family_index: 0,
_marker: std::marker::PhantomData,
};
let mut cp = vk::CommandPool::null();
if unsafe { (ds.fp.create_command_pool)(ds.raw, &pci, std::ptr::null(), &mut cp) }
!= vk::Result::SUCCESS
{
return None;
}
let ai = vk::CommandBufferAllocateInfo {
s_type: vk::StructureType::COMMAND_BUFFER_ALLOCATE_INFO,
p_next: std::ptr::null(),
command_pool: cp,
level: vk::CommandBufferLevel::PRIMARY,
command_buffer_count: 4,
_marker: std::marker::PhantomData,
};
let mut cbs = [vk::CommandBuffer::null(); 4];
if unsafe { (ds.fp.allocate_command_buffers)(ds.raw, &ai, cbs.as_mut_ptr()) }
!= vk::Result::SUCCESS
{
unsafe { (ds.fp.destroy_command_pool)(ds.raw, cp, std::ptr::null()) };
return None;
}
// Create fences PRE-SIGNALED so the first wait_for_fences returns immediately
let fci = vk::FenceCreateInfo {
s_type: vk::StructureType::FENCE_CREATE_INFO,
p_next: std::ptr::null(),
flags: vk::FenceCreateFlags::SIGNALED,
_marker: std::marker::PhantomData,
};
let mut fences = [vk::Fence::null(); 4];
for f in &mut fences {
if unsafe { (ds.fp.create_fence)(ds.raw, &fci, std::ptr::null(), f) } != vk::Result::SUCCESS
{
unsafe { (ds.fp.destroy_command_pool)(ds.raw, cp, std::ptr::null()) };
return None;
}
}
Some(CaptureResources {
command_pool: cp,
command_buffers: cbs,
fences,
current: 0,
})
}
// ── CPU pixel readback (fallback when DMA-BUF unavailable) ───────────────────
pub unsafe fn read_frame_pixels(
ds: &crate::state::DeviceState,
image: vk::Image,
mem: vk::DeviceMemory,
w: u32,
h: u32,
) -> Option<Vec<u8>> {
if w == 0 || h == 0 {
return None;
}
let subresource = vk::ImageSubresource {
aspect_mask: vk::ImageAspectFlags::COLOR,
mip_level: 0,
array_layer: 0,
};
let mut layout = vk::SubresourceLayout {
offset: 0,
size: 0,
row_pitch: 0,
array_pitch: 0,
depth_pitch: 0,
};
unsafe { (ds.fp.get_image_subresource_layout)(ds.raw, image, &subresource, &mut layout) };
let row_pitch = layout.row_pitch as usize;
let bpr = w as usize * 4;
let mut mp: *mut std::os::raw::c_void = std::ptr::null_mut();
if unsafe {
(ds.fp.map_memory)(
ds.raw,
mem,
0,
vk::WHOLE_SIZE,
vk::MemoryMapFlags::empty(),
&mut mp,
) != vk::Result::SUCCESS
} {
return None;
}
let mut pixels = vec![0u8; bpr * h as usize];
let base = mp as *const u8;
for row in 0..h as usize {
let src = unsafe { std::slice::from_raw_parts(base.add(row * row_pitch), bpr) };
pixels[row * bpr..row * bpr + bpr].copy_from_slice(src);
}
unsafe { (ds.fp.unmap_memory)(ds.raw, mem) };
Some(pixels)
}

731
apps/nescapture/src/commands.rs Normal file
View File

@@ -0,0 +1,731 @@
// ─────────────────────────────────────────────────────────────────────────────
// commands.rs — Phase 2: command buffer hooks
//
// vkCmdBindPipeline → update CbState.active_vert_hash/active_frag_hash
// vkCmdBeginRenderPass → resolve framebuffer → views → image, populate CbState
// vkCmdEndRenderPass → clear CbState attachment fields
// vkCmdBeginRenderingKHR → resolve views → image from VkRenderingInfoKHR
// vkCmdEndRenderingKHR → clear CbState attachment fields
// ─────────────────────────────────────────────────────────────────────────────
use crate::capture;
use crate::discovery;
use crate::state::{CB_STATE, CMD_BUF_TO_DEVICE_KEY, DEVICE_STATE};
use ash::vk::{self, Handle};
// ─────────────────────────────────────────────────────────────────────────────
// vkCmdBindPipeline — track which pipeline (and thus which shader hashes)
// is currently bound for draw calls.
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdBindPipeline(
command_buffer: vk::CommandBuffer,
pipeline_bind_point: vk::PipelineBindPoint,
pipeline: vk::Pipeline,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
(ds.fp.cmd_bind_pipeline)(command_buffer, pipeline_bind_point, pipeline);
}
if pipeline_bind_point == vk::PipelineBindPoint::GRAPHICS {
let hashes = ds
.pipeline_registry
.get(&pipeline.as_raw())
.map(|r| r.clone());
if let Some(hashes) = hashes {
if let Some(mut state) = CB_STATE.get_mut(&cb_key) {
state.active_vert_hash = hashes.vert_hash;
state.active_frag_hash = hashes.frag_hash;
} else {
CB_STATE.insert(
cb_key,
crate::state::CbState {
device_key,
active_vert_hash: hashes.vert_hash,
active_frag_hash: hashes.frag_hash,
..Default::default()
},
);
}
// Phase 3: Check against loaded shader hash config
if let Some(ref shader_set) = ds.shader_hashes {
let is_hud = shader_set.is_hud_shader(hashes.vert_hash, hashes.frag_hash);
let is_skip = shader_set.is_skip_shader(hashes.frag_hash);
if is_hud {
log::debug!(
"HUD pipeline detected → vert={} frag={}",
hashes
.vert_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
hashes
.frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
);
// Device-level HUD detection (shared across all command buffers)
if !ds
.hud_detected_frame
.load(std::sync::atomic::Ordering::Relaxed)
{
ds.hud_detected_frame
.store(true, std::sync::atomic::Ordering::Relaxed);
ds.pending_capture_frame
.store(true, std::sync::atomic::Ordering::Relaxed);
}
}
if is_skip {
log::debug!(
"Skip pipeline detected → frag={}",
hashes
.frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
);
}
}
/*log::trace!(
"pipeline bound → vert={} frag={}",
hashes
.vert_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
hashes
.frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
);*/
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// vkCmdBeginRenderPass — resolve the current color attachment image.
//
// The framebuffer contains VkImageViews. We look up each view in the
// view_to_image map to get the VkImage, and use view_format for the format.
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdBeginRenderPass(
command_buffer: vk::CommandBuffer,
p_render_pass_begin: *const vk::RenderPassBeginInfo,
contents: vk::SubpassContents,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
let rpb = unsafe { &*p_render_pass_begin };
let framebuffer = rpb.framebuffer;
let (color_image, format, extent) = if let Some(views) =
ds.framebuffer_to_views.get(&framebuffer.as_raw())
{
let view_keys: Vec<u64> = views.iter().copied().collect();
let extent = ds
.framebuffer_extent
.get(&framebuffer.as_raw())
.map(|r| *r)
.unwrap_or(vk::Extent2D {
width: rpb.render_area.extent.width,
height: rpb.render_area.extent.height,
});
if !view_keys.is_empty() {
let first_view = view_keys[0];
let image = ds
.view_to_image
.get(&first_view)
.map(|r| vk::Image::from_raw(*r));
let fmt = ds.view_format.get(&first_view).map(|r| *r);
log::debug!(
"vkCmdBeginRenderPass → fb={:#010x} view={:#010x} image={:?} format={} extent={}x{}",
framebuffer.as_raw(),
first_view,
image.map(|i| i.as_raw()),
fmt.map(|f| f.as_raw()).unwrap_or(0),
extent.width,
extent.height,
);
(image, fmt, Some(extent))
} else {
log::debug!(
"vkCmdBeginRenderPass → fb={:#010x} no views",
framebuffer.as_raw(),
);
(None, None, Some(extent))
}
} else {
log::debug!(
"vkCmdBeginRenderPass → fb={:#010x} NOT FOUND in framebuffer_to_views",
framebuffer.as_raw(),
);
(None, None, None)
};
unsafe {
(ds.fp.cmd_begin_render_pass)(command_buffer, p_render_pass_begin, contents);
}
if let Some(mut state) = CB_STATE.get_mut(&cb_key) {
state.current_color_image = color_image;
state.current_image_format = format;
state.current_image_extent = extent;
} else {
CB_STATE.insert(
cb_key,
crate::state::CbState {
device_key,
current_color_image: color_image,
current_image_format: format,
current_image_extent: extent,
..Default::default()
},
);
}
if let (Some(img), Some(ext)) = (color_image, extent) {
log::debug!(
"render pass begin → color attachment image {:#010x} extent {}x{}",
img.as_raw(),
ext.width,
ext.height,
);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdEndRenderPass(command_buffer: vk::CommandBuffer) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
// Phase 4: track the largest extent we've seen (main framebuffer)
let current_extent = CB_STATE
.get(&cb_key)
.map(|r| r.current_image_extent)
.unwrap_or(None);
if let Some(ext) = current_extent {
let mut largest = ds.largest_extent.lock().unwrap();
if ext.width * ext.height > largest.width * largest.height {
*largest = ext;
}
}
// Phase 4: check if we need to inject a HUDless capture copy (device-level)
let needs_hudless_capture = ds
.pending_capture_frame
.load(std::sync::atomic::Ordering::Relaxed)
&& !ds
.capture_injected_frame
.load(std::sync::atomic::Ordering::Relaxed);
unsafe {
(ds.fp.cmd_end_render_pass)(command_buffer);
}
// Inject HUDless capture if HUD was detected this frame
if needs_hudless_capture {
log::info!("injecting HUDless capture copy for cb {:#010x}", cb_key);
unsafe {
capture::inject_hudless_copy(command_buffer, device_key);
}
ds.pending_capture_frame
.store(false, std::sync::atomic::Ordering::Relaxed);
ds.capture_injected_frame
.store(true, std::sync::atomic::Ordering::Relaxed);
}
let skipped = ds
.skipped_draws_frame
.swap(0, std::sync::atomic::Ordering::Relaxed);
if skipped > 0 {
log::info!("render pass end → {} draws skipped", skipped);
}
if let Some(mut state) = CB_STATE.get_mut(&cb_key) {
state.current_color_image = None;
state.current_image_format = None;
state.current_image_extent = None;
}
}
// ─────────────────────────────────────────────────────────────────────────────
// vkCmdBeginRenderingKHR — dynamic rendering path (DXVK 1.10+).
//
// Unlike the framebuffer path, VkRenderingInfoKHR gives VkImageViews directly
// in VkRenderingAttachmentInfoKHR. No framebuffer object is involved.
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdBeginRenderingKHR(
command_buffer: vk::CommandBuffer,
p_rendering_info: *const vk::RenderingInfo,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
let ri = unsafe { &*p_rendering_info };
let extent = ri.render_area.extent;
let (color_image, format) =
if ri.color_attachment_count > 0 && !ri.p_color_attachments.is_null() {
let first = unsafe { &*ri.p_color_attachments };
if first.image_view != vk::ImageView::null() {
let image = ds
.view_to_image
.get(&first.image_view.as_raw())
.map(|r| vk::Image::from_raw(*r));
let fmt = ds.view_format.get(&first.image_view.as_raw()).map(|r| *r);
(image, fmt)
} else {
(None, None)
}
} else {
(None, None)
};
unsafe {
(ds.fp.cmd_begin_rendering_khr.unwrap())(command_buffer, p_rendering_info);
}
if let Some(mut state) = CB_STATE.get_mut(&cb_key) {
state.current_color_image = color_image;
state.current_image_format = format;
state.current_image_extent = Some(extent);
} else {
CB_STATE.insert(
cb_key,
crate::state::CbState {
device_key,
current_color_image: color_image,
current_image_format: format,
current_image_extent: Some(extent),
..Default::default()
},
);
}
log::info!(
"vkCmdBeginRenderingKHR → color_image={:?} format={:?} extent={}x{}",
color_image.map(|i| i.as_raw()),
format.map(|f| f.as_raw()),
extent.width,
extent.height,
);
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdEndRenderingKHR(command_buffer: vk::CommandBuffer) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
// Phase 4: check if we need to inject a HUDless capture copy (device-level)
let needs_hudless_capture = ds
.pending_capture_frame
.load(std::sync::atomic::Ordering::Relaxed)
&& !ds
.capture_injected_frame
.load(std::sync::atomic::Ordering::Relaxed);
unsafe {
(ds.fp.cmd_end_rendering_khr.unwrap())(command_buffer);
}
// Inject HUDless capture if HUD was detected this frame
if needs_hudless_capture {
log::info!("injecting HUDless capture copy for cb {:#010x}", cb_key);
unsafe {
capture::inject_hudless_copy(command_buffer, device_key);
}
ds.pending_capture_frame
.store(false, std::sync::atomic::Ordering::Relaxed);
ds.capture_injected_frame
.store(true, std::sync::atomic::Ordering::Relaxed);
}
let skipped = ds
.skipped_draws_frame
.swap(0, std::sync::atomic::Ordering::Relaxed);
if skipped > 0 {
log::info!("rendering end → {} draws skipped", skipped);
}
if let Some(mut state) = CB_STATE.get_mut(&cb_key) {
state.current_color_image = None;
state.current_image_format = None;
state.current_image_extent = None;
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Phase 6: vkCmdDraw* hooks — skip HUD/skip shaders, log for discovery
//
// When a config is loaded and the active pipeline matches a HUD or skip shader,
// the draw call is NOT forwarded to the driver. This prevents the HUD from
// being rendered to the color attachment. The nescapture capture at
// vkCmdEndRenderPass then copies the clean (HUD-free) attachment.
//
// In discovery mode (HUDLESS_DISCOVER=1), draws are never skipped.
// ─────────────────────────────────────────────────────────────────────────────
/// Check if the currently bound pipeline should be suppressed.
/// Returns (should_skip, vert_hash, frag_hash) so the caller doesn't need
/// to re-acquire the CB_STATE lock.
unsafe fn should_skip_draw(
cb_key: u64,
ds: &std::sync::Arc<crate::state::DeviceState>,
) -> (bool, Option<u64>, Option<u64>) {
if discovery::is_discovery_mode() {
return (false, None, None);
}
let shader_set = match &ds.shader_hashes {
Some(s) => s,
None => return (false, None, None),
};
let state = match CB_STATE.get(&cb_key) {
Some(s) => s,
None => return (false, None, None),
};
let vh = state.active_vert_hash;
let fh = state.active_frag_hash;
let skip = shader_set.is_hud_shader(vh, fh) || shader_set.is_skip_shader(fh);
(skip, vh, fh)
}
/// Record that a draw was skipped (for diagnostics).
unsafe fn record_skipped_draw(
ds: &crate::state::DeviceState,
vert_hash: Option<u64>,
frag_hash: Option<u64>,
) {
ds.skipped_draws_frame
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
log::info!(
"SKIPPING draw → vert={} frag={}",
vert_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
);
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDraw(
command_buffer: vk::CommandBuffer,
vertex_count: u32,
instance_count: u32,
first_vertex: u32,
first_instance: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, vertex_count);
return;
}
(ds.fp.cmd_draw)(
command_buffer,
vertex_count,
instance_count,
first_vertex,
first_instance,
);
discovery::record_draw(command_buffer, vertex_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndexed(
command_buffer: vk::CommandBuffer,
index_count: u32,
instance_count: u32,
first_index: u32,
vertex_offset: i32,
first_instance: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, index_count);
return;
}
(ds.fp.cmd_draw_indexed)(
command_buffer,
index_count,
instance_count,
first_index,
vertex_offset,
first_instance,
);
discovery::record_draw(command_buffer, index_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndirect(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
draw_count: u32,
stride: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, draw_count);
return;
}
(ds.fp.cmd_draw_indirect)(command_buffer, buffer, offset, draw_count, stride);
discovery::record_draw(command_buffer, draw_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndexedIndirect(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
draw_count: u32,
stride: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, draw_count);
return;
}
(ds.fp.cmd_draw_indexed_indirect)(command_buffer, buffer, offset, draw_count, stride);
discovery::record_draw(command_buffer, draw_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndirectCount(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
count_buffer: vk::Buffer,
count_buffer_offset: vk::DeviceSize,
max_draw_count: u32,
stride: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, max_draw_count);
return;
}
if let Some(f) = ds.fp.cmd_draw_indirect_count {
f(
command_buffer,
buffer,
offset,
count_buffer,
count_buffer_offset,
max_draw_count,
stride,
);
}
discovery::record_draw(command_buffer, max_draw_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndexedIndirectCount(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
count_buffer: vk::Buffer,
count_buffer_offset: vk::DeviceSize,
max_draw_count: u32,
stride: u32,
) {
let cb_key = command_buffer.as_raw();
let device_key = match CMD_BUF_TO_DEVICE_KEY.get(&cb_key) {
Some(r) => *r,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s.clone(),
None => return,
};
unsafe {
let (skip, vh, fh) = should_skip_draw(cb_key, &ds);
if skip {
record_skipped_draw(&ds, vh, fh);
discovery::record_draw(command_buffer, max_draw_count);
return;
}
if let Some(f) = ds.fp.cmd_draw_indexed_indirect_count {
f(
command_buffer,
buffer,
offset,
count_buffer,
count_buffer_offset,
max_draw_count,
stride,
);
}
discovery::record_draw(command_buffer, max_draw_count);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndirectCountKHR(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
count_buffer: vk::Buffer,
count_buffer_offset: vk::DeviceSize,
max_draw_count: u32,
stride: u32,
) {
unsafe {
vkCmdDrawIndirectCount(
command_buffer,
buffer,
offset,
count_buffer,
count_buffer_offset,
max_draw_count,
stride,
);
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCmdDrawIndexedIndirectCountKHR(
command_buffer: vk::CommandBuffer,
buffer: vk::Buffer,
offset: vk::DeviceSize,
count_buffer: vk::Buffer,
count_buffer_offset: vk::DeviceSize,
max_draw_count: u32,
stride: u32,
) {
unsafe {
vkCmdDrawIndexedIndirectCount(
command_buffer,
buffer,
offset,
count_buffer,
count_buffer_offset,
max_draw_count,
stride,
);
}
}

141
apps/nescapture/src/config.rs Normal file
View File

@@ -0,0 +1,141 @@
// ─────────────────────────────────────────────────────────────────────────────
// config.rs — Phase 3: per-game shader hash configuration
//
// Loads a TOML config file that maps game executables to sets of shader hashes:
// hud_fragment_shaders — fragment shaders used for HUD/UI elements
// hud_vertex_shaders — vertex shaders used for HUD/UI elements
// skip_fragment_shaders — fragment shaders to silently drop (motion blur, etc.)
//
// Environment variables:
// HUDLESS_CONFIG=/path/to/config.toml — explicit config path
// HUDLESS_GAME_NAME=Game.exe — override game name detection
//
// Config format:
// [game."Control_DX11.exe"]
// hud_fragment_shaders = ["0xaabbccddeeff0011"]
// hud_vertex_shaders = ["0x1a2b3c4d5e6f7890"]
// skip_fragment_shaders = ["0x1122334455667788"]
// ─────────────────────────────────────────────────────────────────────────────
use serde::Deserialize;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
/// Parsed shader hash sets for a single game.
#[derive(Clone, Debug, Default)]
pub struct ShaderHashSet {
pub hud_fragment_shaders: HashSet<u64>,
pub hud_vertex_shaders: HashSet<u64>,
pub skip_fragment_shaders: HashSet<u64>,
}
impl ShaderHashSet {
/// Check if the given vert/frag hash pair matches a HUD shader.
///
/// A HUD match occurs when:
/// - frag_hash is in hud_fragment_shaders, OR
/// - vert_hash is in hud_vertex_shaders (if frag not set)
pub fn is_hud_shader(&self, vert_hash: Option<u64>, frag_hash: Option<u64>) -> bool {
match frag_hash {
Some(fh) => self.hud_fragment_shaders.contains(&fh),
None => vert_hash.is_some_and(|vh| self.hud_vertex_shaders.contains(&vh)),
}
}
/// Check if the given frag_hash matches a skip shader.
pub fn is_skip_shader(&self, frag_hash: Option<u64>) -> bool {
if let Some(fh) = frag_hash {
return self.skip_fragment_shaders.contains(&fh);
}
false
}
}
/// TOML deserialization structure.
#[derive(Deserialize)]
struct ConfigFile {
game: Option<std::collections::HashMap<String, GameConfig>>,
}
#[derive(Deserialize)]
struct GameConfig {
hud_fragment_shaders: Option<Vec<String>>,
hud_vertex_shaders: Option<Vec<String>>,
skip_fragment_shaders: Option<Vec<String>>,
}
/// Parse a hex string like "0xaabbccddeeff0011" into a u64.
fn parse_hex(s: &str) -> Option<u64> {
let trimmed = s.trim().trim_start_matches("0x").trim_start_matches("0X");
u64::from_str_radix(trimmed, 16).ok()
}
/// Load the config from the given path and return the ShaderHashSet for the
/// current game.
pub fn load_config(path: &Path) -> Option<ShaderHashSet> {
let content = match fs::read_to_string(path) {
Ok(c) => c,
Err(e) => {
log::warn!("failed to read config {:?}: {}", path, e);
return None;
}
};
let config: ConfigFile = match toml::from_str(&content) {
Ok(c) => c,
Err(e) => {
log::warn!("failed to parse config {:?}: {}", path, e);
return None;
}
};
let game_name = std::env::var("NESCAPTURE_GAME_NAME").ok()?;
let games = config.game.as_ref()?;
let game_config = games.get(&game_name)?;
let mut set = ShaderHashSet::default();
if let Some(hashes) = &game_config.hud_fragment_shaders {
for h in hashes {
if let Some(v) = parse_hex(h) {
set.hud_fragment_shaders.insert(v);
}
}
}
if let Some(hashes) = &game_config.hud_vertex_shaders {
for h in hashes {
if let Some(v) = parse_hex(h) {
set.hud_vertex_shaders.insert(v);
}
}
}
if let Some(hashes) = &game_config.skip_fragment_shaders {
for h in hashes {
if let Some(v) = parse_hex(h) {
set.skip_fragment_shaders.insert(v);
}
}
}
log::info!(
"loaded config for '{}' — {} hud_frag, {} hud_vert, {} skip_frag",
game_name,
set.hud_fragment_shaders.len(),
set.hud_vertex_shaders.len(),
set.skip_fragment_shaders.len(),
);
Some(set)
}
/// Resolve the config file path from HUDLESS_CONFIG env var, or fall back to
/// a default location relative to the game executable.
pub fn resolve_config_path() -> Option<std::path::PathBuf> {
if let Ok(path) = std::env::var("NESCAPTURE_CONFIG") {
return Some(std::path::PathBuf::from(path));
}
None
}

528
apps/nescapture/src/device.rs Normal file
View File

@@ -0,0 +1,528 @@
// ─────────────────────────────────────────────────────────────────────────────
// device.rs — vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue
// ─────────────────────────────────────────────────────────────────────────────
use crate::config;
use crate::dispatch::{NextDeviceFn, PFN_vkCreateDevice, PFN_vkGetInstanceProcAddr};
use crate::state::{DEVICE_STATE, DeviceState, INSTANCE_STATE, QUEUE_TO_DEVICE_KEY};
use crate::{
VkLayerDeviceCreateInfo, dispatch_key, find_layer_link, load_device_fn, try_load_device_fn,
};
use ash::vk::{self, Handle};
use dashmap::DashMap;
use std::os::raw::c_void;
use std::sync::Arc;
use std::sync::atomic::Ordering;
const VK_LAYER_LINK_INFO: u32 = 0;
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateDevice(
physical_device: vk::PhysicalDevice,
p_create_info: *const vk::DeviceCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_device: *mut vk::Device,
) -> vk::Result {
let layer_info: *mut VkLayerDeviceCreateInfo = match unsafe {
find_layer_link((*p_create_info).p_next as *const c_void, VK_LAYER_LINK_INFO)
} {
Some(p) => p,
None => return vk::Result::ERROR_INITIALIZATION_FAILED,
};
let dev_link = unsafe { (*layer_info).u.pDeviceLayerInfo };
let next_gipa: PFN_vkGetInstanceProcAddr =
match unsafe { (*dev_link).pfnNextGetInstanceProcAddr } {
Some(f) => f,
None => return vk::Result::ERROR_INITIALIZATION_FAILED,
};
let next_gdpa = unsafe {
match (*dev_link).pfnNextGetDeviceProcAddr {
Some(f) => f,
None => return vk::Result::ERROR_INITIALIZATION_FAILED,
}
};
unsafe { (*layer_info).u.pDeviceLayerInfo = (*dev_link).pNext };
let inst_key = unsafe { dispatch_key(physical_device.as_raw() as *const c_void) };
let istate = match INSTANCE_STATE.get(&inst_key) {
Some(s) => s.clone(),
None => {
let next_create: PFN_vkCreateDevice = unsafe {
crate::load_instance_fn(next_gipa, vk::Instance::null(), b"vkCreateDevice\0")
};
return unsafe { next_create(physical_device, p_create_info, p_allocator, p_device) };
}
};
// ── Inject DMA-BUF extensions for zero-copy capture ──────────────────
let ci = unsafe { &*p_create_info };
// Collect the game's original extensions.
let original_extensions: Vec<*const libc::c_char> =
if ci.enabled_extension_count > 0 && !ci.pp_enabled_extension_names.is_null() {
unsafe {
std::slice::from_raw_parts(
ci.pp_enabled_extension_names,
ci.enabled_extension_count as usize,
)
}
.to_vec()
} else {
Vec::new()
};
// Extensions we need — static byte strings so pointers stay valid.
const EXT_EXTERNAL_MEMORY: &[u8] = b"VK_KHR_external_memory\0";
const EXT_EXTERNAL_MEMORY_FD: &[u8] = b"VK_KHR_external_memory_fd\0";
const EXT_EXTERNAL_MEMORY_DMABUF: &[u8] = b"VK_EXT_external_memory_dma_buf\0";
let needed: &[&[u8]] = &[
EXT_EXTERNAL_MEMORY,
EXT_EXTERNAL_MEMORY_FD,
EXT_EXTERNAL_MEMORY_DMABUF,
];
// Build extended list: original + any of ours not already present.
let mut extended = original_extensions.clone();
for &ext in needed {
let name_cstr = unsafe { std::ffi::CStr::from_bytes_with_nul_unchecked(ext) };
let already = extended
.iter()
.any(|&ptr| unsafe { std::ffi::CStr::from_ptr(ptr) == name_cstr });
if !already {
extended.push(ext.as_ptr() as *const libc::c_char);
}
}
// ── Bump queue count for dedicated capture queue ──────────────────
// Add one extra queue to the first queue family so capture
// submissions don't compete with game rendering.
let mut capture_queue_index = 0u32;
let queue_infos: Vec<vk::DeviceQueueCreateInfo> = if ci.queue_create_info_count > 0
&& !ci.p_queue_create_infos.is_null()
{
let slice = unsafe {
std::slice::from_raw_parts(ci.p_queue_create_infos, ci.queue_create_info_count as usize)
};
let mut qis = slice.to_vec();
if let Some(first) = qis.first_mut() {
capture_queue_index = first.queue_count; // use the NEXT index
first.queue_count += 1;
}
qis
} else {
Vec::new()
};
// Try with injected extensions first.
let mut modified_ci = *ci;
modified_ci.enabled_extension_count = extended.len() as u32;
modified_ci.pp_enabled_extension_names = extended.as_ptr();
modified_ci.queue_create_info_count = queue_infos.len() as u32;
modified_ci.p_queue_create_infos = queue_infos.as_ptr();
let mut dmabuf_available = true;
let result =
unsafe { (istate.create_device)(physical_device, &modified_ci, p_allocator, p_device) };
let result = if result != vk::Result::SUCCESS {
// Driver rejected our extensions — retry with original create info.
log::warn!(
"vkCreateDevice with DMA-BUF extensions failed ({:?}), \
retrying without — CPU readback fallback will be used",
result
);
dmabuf_available = false;
unsafe { (istate.create_device)(physical_device, p_create_info, p_allocator, p_device) }
} else {
log::info!("DMA-BUF extensions injected successfully");
result
};
if result != vk::Result::SUCCESS {
return result;
}
if !dmabuf_available {
log::warn!("DMA-BUF extensions missing — will use CPU readback fallback (expensive!)");
}
let device = unsafe { *p_device };
// Cache physical device memory properties
let mut mem_props = vk::PhysicalDeviceMemoryProperties::default();
unsafe { (istate.get_physical_device_memory_properties)(physical_device, &mut mem_props) };
macro_rules! load {
($name:literal) => {
unsafe { load_device_fn(next_gdpa, device, $name) }
};
}
macro_rules! try_load {
($name:literal) => {
unsafe { try_load_device_fn(next_gdpa, device, $name) }
};
}
let fp = NextDeviceFn {
// Infrastructure
get_device_proc_addr: next_gdpa,
destroy_device: load!(b"vkDestroyDevice\0"),
get_device_queue: load!(b"vkGetDeviceQueue\0"),
queue_present_khr: try_load!(b"vkQueuePresentKHR\0"),
// Phase 1
create_shader_module: load!(b"vkCreateShaderModule\0"),
destroy_shader_module: load!(b"vkDestroyShaderModule\0"),
create_graphics_pipelines: load!(b"vkCreateGraphicsPipelines\0"),
destroy_pipeline: load!(b"vkDestroyPipeline\0"),
// Phase 2
create_image_view: load!(b"vkCreateImageView\0"),
destroy_image_view: load!(b"vkDestroyImageView\0"),
create_framebuffer: load!(b"vkCreateFramebuffer\0"),
destroy_framebuffer: load!(b"vkDestroyFramebuffer\0"),
allocate_command_buffers: load!(b"vkAllocateCommandBuffers\0"),
free_command_buffers: load!(b"vkFreeCommandBuffers\0"),
cmd_bind_pipeline: load!(b"vkCmdBindPipeline\0"),
cmd_begin_render_pass: load!(b"vkCmdBeginRenderPass\0"),
cmd_end_render_pass: load!(b"vkCmdEndRenderPass\0"),
cmd_begin_rendering_khr: try_load!(b"vkCmdBeginRenderingKHR\0"),
cmd_end_rendering_khr: try_load!(b"vkCmdEndRenderingKHR\0"),
// Phase 4 — capture images
create_image: load!(b"vkCreateImage\0"),
destroy_image: load!(b"vkDestroyImage\0"),
allocate_memory: load!(b"vkAllocateMemory\0"),
free_memory: load!(b"vkFreeMemory\0"),
bind_image_memory: load!(b"vkBindImageMemory\0"),
get_image_memory_requirements: load!(b"vkGetImageMemoryRequirements\0"),
map_memory: load!(b"vkMapMemory\0"),
unmap_memory: load!(b"vkUnmapMemory\0"),
cmd_pipeline_barrier: load!(b"vkCmdPipelineBarrier\0"),
cmd_copy_image: load!(b"vkCmdCopyImage\0"),
get_image_subresource_layout: load!(b"vkGetImageSubresourceLayout\0"),
get_memory_fd_khr: try_load!(b"vkGetMemoryFdKHR\0"),
// Phase 4 — synchronisation
create_fence: load!(b"vkCreateFence\0"),
destroy_fence: load!(b"vkDestroyFence\0"),
create_command_pool: load!(b"vkCreateCommandPool\0"),
destroy_command_pool: load!(b"vkDestroyCommandPool\0"),
reset_command_pool: load!(b"vkResetCommandPool\0"),
begin_command_buffer: load!(b"vkBeginCommandBuffer\0"),
end_command_buffer: load!(b"vkEndCommandBuffer\0"),
reset_command_buffer: load!(b"vkResetCommandBuffer\0"), // needed for double-buffered capture
queue_submit: load!(b"vkQueueSubmit\0"),
wait_for_fences: load!(b"vkWaitForFences\0"),
reset_fences: load!(b"vkResetFences\0"),
// Phase 4 — swapchain
create_swapchain_khr: try_load!(b"vkCreateSwapchainKHR\0"),
destroy_swapchain_khr: try_load!(b"vkDestroySwapchainKHR\0"),
get_swapchain_images_khr: try_load!(b"vkGetSwapchainImagesKHR\0"),
// Phase 6 — draw commands
cmd_draw: load!(b"vkCmdDraw\0"),
cmd_draw_indexed: load!(b"vkCmdDrawIndexed\0"),
cmd_draw_indirect: load!(b"vkCmdDrawIndirect\0"),
cmd_draw_indexed_indirect: load!(b"vkCmdDrawIndexedIndirect\0"),
cmd_draw_indirect_count: try_load!(b"vkCmdDrawIndirectCount\0"),
cmd_draw_indexed_indirect_count: try_load!(b"vkCmdDrawIndexedIndirectCount\0"),
};
// ── Retrieve capture queue from the bumped slot ───────────────────
let mut capture_queue = vk::Queue::null();
if queue_infos
.first()
.map(|q| q.queue_count > 1)
.unwrap_or(false)
{
let qi = &queue_infos[0];
unsafe {
(fp.get_device_queue)(
device,
qi.queue_family_index,
capture_queue_index,
&mut capture_queue,
);
}
log::info!(
"capture queue: family={} index={capture_queue_index}",
qi.queue_family_index
);
}
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
// Phase 3: load shader hash config
let shader_hashes = config::resolve_config_path()
.as_ref()
.and_then(|p| config::load_config(p));
if let Some(ref set) = shader_hashes {
log::info!(
"config loaded — {} hud_frag, {} hud_vert, {} skip_frag",
set.hud_fragment_shaders.len(),
set.hud_vertex_shaders.len(),
set.skip_fragment_shaders.len(),
);
} else {
log::warn!("no config — draw suppression disabled");
}
let dev_state = Arc::new(DeviceState {
raw: device,
physical_device,
fp,
shader_registry: DashMap::new(),
pipeline_registry: DashMap::new(),
pipeline_state: DashMap::new(),
view_to_image: DashMap::new(),
view_format: DashMap::new(),
framebuffer_to_views: DashMap::new(),
framebuffer_extent: DashMap::new(),
shader_hashes,
hudless_image: std::sync::Mutex::new(None),
hudless_memory: std::sync::Mutex::new(None),
hudless_size: std::sync::Mutex::new((0, 0, vk::Format::UNDEFINED)),
final_image: std::sync::Mutex::new(None),
final_memory: std::sync::Mutex::new(None),
final_size: std::sync::Mutex::new((0, 0, vk::Format::UNDEFINED)),
final_stride: std::sync::atomic::AtomicU32::new(0),
swapchain: std::sync::Mutex::new(None),
swapchain_images: std::sync::Mutex::new(Vec::new()),
swapchain_format: std::sync::Mutex::new(vk::Format::UNDEFINED),
swapchain_extent: std::sync::Mutex::new(vk::Extent2D {
width: 0,
height: 0,
}),
swapchain_colorspace: std::sync::atomic::AtomicU32::new(0),
frame_counter: std::sync::atomic::AtomicU64::new(0),
largest_extent: std::sync::Mutex::new(vk::Extent2D {
width: 0,
height: 0,
}),
hud_detected_frame: std::sync::atomic::AtomicBool::new(false),
pending_capture_frame: std::sync::atomic::AtomicBool::new(false),
capture_injected_frame: std::sync::atomic::AtomicBool::new(false),
skipped_draws_frame: std::sync::atomic::AtomicU32::new(0),
encoder: std::sync::Mutex::new(None),
capture_resources: std::sync::Mutex::new(None),
capture_queue: std::sync::Mutex::new(capture_queue),
fake_images: std::sync::Mutex::new(Vec::new()),
fake_memories: std::sync::Mutex::new(Vec::new()),
fake_fds: std::sync::Mutex::new(Vec::new()),
fake_strides: std::sync::Mutex::new(Vec::new()),
fake_available: std::sync::Mutex::new(Vec::new()),
fake_image_count: std::sync::atomic::AtomicU32::new(0),
fake_swapchain: std::sync::Mutex::new(None),
signal_queue: std::sync::Mutex::new(vk::Queue::null()),
next_acquire: std::sync::atomic::AtomicU32::new(0),
memory_properties: std::sync::Mutex::new(mem_props),
acquire_dummy_pool: std::sync::Mutex::new(vk::CommandPool::null()),
acquire_dummy_cb: std::sync::Mutex::new(vk::CommandBuffer::null()),
cached_dmabuf_fd: std::sync::atomic::AtomicI32::new(-1),
target_fps: std::sync::atomic::AtomicU32::new(0),
last_capture_time: std::sync::Mutex::new(None),
capture_tx: std::sync::Mutex::new(None),
});
DEVICE_STATE.insert(key, dev_state);
log::info!("vkCreateDevice OK — key {:#x}", key);
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyDevice(
device: vk::Device,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.remove(&key) {
Some((_, ds)) => ds,
None => return,
};
log::info!(
"vkDestroyDevice — {} shaders, {} pipelines evicted",
ds.shader_registry.len(),
ds.pipeline_registry.len(),
);
// ── 1. Shut down encoder pipeline (unblocks encoder + RTP threads) ───
{
let mut enc_guard = ds.encoder.lock().unwrap();
if let Some(handle) = enc_guard.take() {
handle.shutdown();
// `handle` is dropped here → drops `frame_tx` → encoder thread's
// recv_timeout returns Disconnected → encoder thread drops
// `encoded_tx` → RTP thread exits too.
//
// Give threads a moment to drain. In production you'd join the
// JoinHandles, but since we don't store them, a short sleep +
// the AtomicBool shutdown flag is sufficient.
log::info!("encoder pipeline shutdown signaled");
}
}
// Brief yield to let threads notice the disconnect.
std::thread::sleep(std::time::Duration::from_millis(50));
// ── 2. Clean up capture resources (double-buffered cmd pool + fences) ─
{
let mut res_guard = ds.capture_resources.lock().unwrap();
if let Some(res) = res_guard.take() {
unsafe {
// Wait for any in-flight capture commands to finish before
// destroying the fences / command pool.
let _ = (ds.fp.wait_for_fences)(
ds.raw,
res.fences.len() as u32,
res.fences.as_ptr(),
vk::TRUE,
5_000_000_000, // 5 seconds — should be instant
);
for &f in &res.fences {
(ds.fp.destroy_fence)(ds.raw, f, std::ptr::null());
}
(ds.fp.destroy_command_pool)(ds.raw, res.command_pool, std::ptr::null());
}
log::debug!("capture resources destroyed");
}
}
// ── 3. Free final_image / final_memory ────────────────────────────────
{
let img = ds.final_image.lock().unwrap().take();
let mem = ds.final_memory.lock().unwrap().take();
if let Some(i) = img {
unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) };
}
if let Some(m) = mem {
unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) };
}
}
// ── 4. Free hudless_image / hudless_memory ────────────────────────────
{
let img = ds.hudless_image.lock().unwrap().take();
let mem = ds.hudless_memory.lock().unwrap().take();
if let Some(i) = img {
unsafe { (ds.fp.destroy_image)(ds.raw, i, std::ptr::null()) };
}
if let Some(m) = mem {
unsafe { (ds.fp.free_memory)(ds.raw, m, std::ptr::null()) };
}
}
// ── 5. Close cached DMA-BUF fd ────────────────────────────────────────
{
let fd = ds.cached_dmabuf_fd.load(Ordering::Relaxed);
if fd >= 0 {
unsafe { libc::close(fd) };
log::debug!("cached DMA-BUF fd {} closed", fd);
}
}
// ── 6. Clean up queue → device key mappings for this device ───────────
QUEUE_TO_DEVICE_KEY.retain(|_, dk| *dk != key);
// ── 7. Call the real vkDestroyDevice ──────────────────────────────────
unsafe { (ds.fp.destroy_device)(device, p_allocator) };
log::info!("vkDestroyDevice complete");
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkGetDeviceQueue(
device: vk::Device,
queue_family_index: u32,
queue_index: u32,
p_queue: *mut vk::Queue,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
unsafe { (ds.fp.get_device_queue)(device, queue_family_index, queue_index, p_queue) };
let queue = unsafe { *p_queue };
QUEUE_TO_DEVICE_KEY.insert(queue.as_raw(), key);
// Store first queue for acquire semaphore signaling
let mut sq = ds.signal_queue.lock().unwrap();
if *sq == vk::Queue::null() {
*sq = queue;
}
}
}
/// Enumerate device extensions supported by the physical device.
unsafe fn enumerate_device_extensions(
istate: &crate::dispatch::NextInstanceFn,
physical_device: vk::PhysicalDevice,
) -> Vec<std::ffi::CString> {
// We need vkEnumerateDeviceExtensionProperties. Load it from the
// instance dispatch since it's a physical-device-level function.
// For simplicity, use ash's raw function signature.
type PFN_vkEnumerateDeviceExtensionProperties = unsafe extern "system" fn(
vk::PhysicalDevice,
*const libc::c_char,
*mut u32,
*mut vk::ExtensionProperties,
) -> vk::Result;
let func: Option<PFN_vkEnumerateDeviceExtensionProperties> = {
let raw = unsafe {
(istate.get_instance_proc_addr)(
vk::Instance::null(),
b"vkEnumerateDeviceExtensionProperties\0".as_ptr() as *const libc::c_char,
)
};
raw.map(|f| unsafe { std::mem::transmute(f) })
};
let Some(enumerate) = func else {
log::warn!("could not load vkEnumerateDeviceExtensionProperties");
return Vec::new();
};
let mut count = 0u32;
if unsafe {
enumerate(
physical_device,
std::ptr::null(),
&mut count,
std::ptr::null_mut(),
)
} != vk::Result::SUCCESS
{
return Vec::new();
}
let mut props = vec![vk::ExtensionProperties::default(); count as usize];
if unsafe {
enumerate(
physical_device,
std::ptr::null(),
&mut count,
props.as_mut_ptr(),
)
} != vk::Result::SUCCESS
{
return Vec::new();
}
props
.iter()
.filter_map(|p| {
let cstr = unsafe { std::ffi::CStr::from_ptr(p.extension_name.as_ptr()) };
Some(cstr.to_owned())
})
.collect()
}

165
apps/nescapture/src/discovery.rs Normal file
View File

@@ -0,0 +1,165 @@
// ─────────────────────────────────────────────────────────────────────────────
// discovery.rs — Phase 6: per-draw logging for shader hash discovery
//
// When HUDLESS_DISCOVER=1 is set, every vkCmdDraw* call is logged to:
// /tmp/hudless_discover_$EXE.log
//
// Log format:
// frame=0001 draw=00042 vert=0x1a2b3c4d5e6f7890 frag=0xaabbccddeeff0011 verts=6 blend=true depth=false
//
// HUD suspect heuristics:
// - blend=true && depth_test=false → strong suspect (UI quads)
// - verts <= 6 → two triangles = one quad
// - Same pipeline reused many times per frame at low vertex count
// ─────────────────────────────────────────────────────────────────────────────
use crate::state::{CB_STATE, DEVICE_STATE};
use ash::vk::{self, Handle};
use std::fs::OpenOptions;
use std::io::Write;
use std::sync::Mutex;
static LOG_FILE: Mutex<Option<std::fs::File>> = Mutex::new(None);
pub fn is_discovery_mode() -> bool {
std::env::var("NESCAPTURE_DISCOVER")
.map(|v| v == "1")
.unwrap_or(false)
}
fn ensure_log_file() -> bool {
let mut guard = LOG_FILE.lock().unwrap();
if guard.is_some() {
return true;
}
let exe_name = std::env::var("NESCAPTURE_GAME_NAME").unwrap_or_else(|_| {
std::fs::read_link("/proc/self/exe")
.ok()
.and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
.unwrap_or_else(|| "unknown".to_string())
});
let path = format!("/tmp/hudless_discover_{}.log", exe_name);
match OpenOptions::new().create(true).append(true).open(&path) {
Ok(f) => {
log::info!("discovery logging to {}", path);
*guard = Some(f);
true
}
Err(e) => {
log::warn!("failed to open discovery log {}: {}", path, e);
false
}
}
}
pub fn log_draw(
frame: u64,
draw: u32,
vert_hash: Option<u64>,
frag_hash: Option<u64>,
vert_count: u32,
blend_enabled: bool,
depth_test_enabled: bool,
) {
if !ensure_log_file() {
return;
}
let mut guard = match LOG_FILE.lock() {
Ok(g) => g,
Err(_) => return,
};
let file = match guard.as_mut() {
Some(f) => f,
None => return,
};
let vert_str = vert_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string());
let frag_str = frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string());
let is_hud_suspect = blend_enabled && !depth_test_enabled && vert_count <= 6;
let suspect_marker = if is_hud_suspect { " [SUSPECT]" } else { "" };
let line = format!(
"frame={:04} draw={:05} vert={} frag={} verts={:<5} blend={:<5} depth={}{}\n",
frame,
draw,
vert_str,
frag_str,
vert_count,
blend_enabled,
depth_test_enabled,
suspect_marker
);
let _ = file.write_all(line.as_bytes());
let _ = file.flush();
}
pub unsafe fn record_draw(cb: vk::CommandBuffer, vert_count: u32) {
if !is_discovery_mode() {
return;
}
let cb_key = cb.as_raw();
let device_key = match CB_STATE.get(&cb_key) {
Some(r) => r.device_key,
None => return,
};
let ds = match DEVICE_STATE.get(&device_key) {
Some(s) => s,
None => return,
};
let frame = ds.frame_counter.load(std::sync::atomic::Ordering::Relaxed);
let (vert_hash, frag_hash, blend_enabled, depth_test_enabled) = {
let state = match CB_STATE.get_mut(&cb_key) {
Some(s) => s,
None => return,
};
let draw = state.draw_counter + 1;
let vert_hash = state.active_vert_hash;
let frag_hash = state.active_frag_hash;
let (blend_enabled, depth_test_enabled) = if let Some(active_frag) = frag_hash {
let mut blend = false;
let mut depth = false;
for entry in ds.pipeline_state.iter() {
if let Some(ph) = ds.pipeline_registry.get(entry.key()) {
if ph.frag_hash == Some(active_frag) {
blend = entry.value().blend_enabled;
depth = entry.value().depth_test_enabled;
break;
}
}
}
(blend, depth)
} else {
(false, false)
};
drop(state);
log_draw(
frame,
draw,
vert_hash,
frag_hash,
vert_count,
blend_enabled,
depth_test_enabled,
);
(vert_hash, frag_hash, blend_enabled, depth_test_enabled)
};
}

363
apps/nescapture/src/dispatch.rs Normal file
View File

@@ -0,0 +1,363 @@
// ─────────────────────────────────────────────────────────────────────────────
// dispatch.rs — raw Vulkan function-pointer type aliases and dispatch tables
// ─────────────────────────────────────────────────────────────────────────────
use ash::vk;
use std::os::raw::c_char;
pub type RawFn = unsafe extern "system" fn();
// ── Instance-level ────────────────────────────────────────────────────────────
pub type PFN_vkGetInstanceProcAddr =
unsafe extern "system" fn(vk::Instance, *const c_char) -> Option<RawFn>;
pub type PFN_vkGetDeviceProcAddr =
unsafe extern "system" fn(vk::Device, *const c_char) -> Option<RawFn>;
pub type PFN_vkCreateInstance = unsafe extern "system" fn(
*const vk::InstanceCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Instance,
) -> vk::Result;
pub type PFN_vkDestroyInstance =
unsafe extern "system" fn(vk::Instance, *const vk::AllocationCallbacks);
pub type PFN_vkGetPhysicalDeviceMemoryProperties =
unsafe extern "system" fn(vk::PhysicalDevice, *mut vk::PhysicalDeviceMemoryProperties);
pub type PFN_vkCreateDevice = unsafe extern "system" fn(
vk::PhysicalDevice,
*const vk::DeviceCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Device,
) -> vk::Result;
// ── Device infrastructure ─────────────────────────────────────────────────────
pub type PFN_vkDestroyDevice =
unsafe extern "system" fn(vk::Device, *const vk::AllocationCallbacks);
pub type PFN_vkGetDeviceQueue = unsafe extern "system" fn(vk::Device, u32, u32, *mut vk::Queue);
pub type PFN_vkQueuePresentKHR =
unsafe extern "system" fn(vk::Queue, *const vk::PresentInfoKHR) -> vk::Result;
// ── Phase 1: Shader modules ───────────────────────────────────────────────────
pub type PFN_vkCreateShaderModule = unsafe extern "system" fn(
vk::Device,
*const vk::ShaderModuleCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::ShaderModule,
) -> vk::Result;
pub type PFN_vkDestroyShaderModule =
unsafe extern "system" fn(vk::Device, vk::ShaderModule, *const vk::AllocationCallbacks);
// ── Phase 1: Graphics pipelines ──────────────────────────────────────────────
pub type PFN_vkCreateGraphicsPipelines = unsafe extern "system" fn(
vk::Device,
vk::PipelineCache,
u32,
*const vk::GraphicsPipelineCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Pipeline,
) -> vk::Result;
pub type PFN_vkDestroyPipeline =
unsafe extern "system" fn(vk::Device, vk::Pipeline, *const vk::AllocationCallbacks);
// ── Phase 2: Image views and framebuffers ─────────────────────────────────────
pub type PFN_vkCreateImageView = unsafe extern "system" fn(
vk::Device,
*const vk::ImageViewCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::ImageView,
) -> vk::Result;
pub type PFN_vkDestroyImageView =
unsafe extern "system" fn(vk::Device, vk::ImageView, *const vk::AllocationCallbacks);
pub type PFN_vkCreateFramebuffer = unsafe extern "system" fn(
vk::Device,
*const vk::FramebufferCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Framebuffer,
) -> vk::Result;
pub type PFN_vkDestroyFramebuffer =
unsafe extern "system" fn(vk::Device, vk::Framebuffer, *const vk::AllocationCallbacks);
pub type PFN_vkAllocateCommandBuffers = unsafe extern "system" fn(
vk::Device,
*const vk::CommandBufferAllocateInfo,
*mut vk::CommandBuffer,
) -> vk::Result;
pub type PFN_vkFreeCommandBuffers =
unsafe extern "system" fn(vk::Device, vk::CommandPool, u32, *const vk::CommandBuffer);
// ── Phase 2: Render pass and rendering ───────────────────────────────────────
pub type PFN_vkCmdBindPipeline =
unsafe extern "system" fn(vk::CommandBuffer, vk::PipelineBindPoint, vk::Pipeline);
pub type PFN_vkCmdBeginRenderPass = unsafe extern "system" fn(
vk::CommandBuffer,
*const vk::RenderPassBeginInfo,
vk::SubpassContents,
);
pub type PFN_vkCmdEndRenderPass = unsafe extern "system" fn(vk::CommandBuffer);
pub type PFN_vkCmdBeginRenderingKHR =
unsafe extern "system" fn(vk::CommandBuffer, *const vk::RenderingInfo);
pub type PFN_vkCmdEndRenderingKHR = unsafe extern "system" fn(vk::CommandBuffer);
// ── Phase 4: Capture images and memory ───────────────────────────────────────
pub type PFN_vkCreateImage = unsafe extern "system" fn(
vk::Device,
*const vk::ImageCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Image,
) -> vk::Result;
pub type PFN_vkDestroyImage =
unsafe extern "system" fn(vk::Device, vk::Image, *const vk::AllocationCallbacks);
pub type PFN_vkAllocateMemory = unsafe extern "system" fn(
vk::Device,
*const vk::MemoryAllocateInfo,
*const vk::AllocationCallbacks,
*mut vk::DeviceMemory,
) -> vk::Result;
pub type PFN_vkFreeMemory =
unsafe extern "system" fn(vk::Device, vk::DeviceMemory, *const vk::AllocationCallbacks);
pub type PFN_vkBindImageMemory = unsafe extern "system" fn(
vk::Device,
vk::Image,
vk::DeviceMemory,
vk::DeviceSize,
) -> vk::Result;
pub type PFN_vkGetImageMemoryRequirements =
unsafe extern "system" fn(vk::Device, vk::Image, *mut vk::MemoryRequirements);
pub type PFN_vkMapMemory = unsafe extern "system" fn(
vk::Device,
vk::DeviceMemory,
vk::DeviceSize,
vk::DeviceSize,
vk::MemoryMapFlags,
*mut *mut std::os::raw::c_void,
) -> vk::Result;
pub type PFN_vkUnmapMemory = unsafe extern "system" fn(vk::Device, vk::DeviceMemory);
pub type PFN_vkCmdPipelineBarrier = unsafe extern "system" fn(
vk::CommandBuffer,
vk::PipelineStageFlags,
vk::PipelineStageFlags,
vk::DependencyFlags,
u32,
*const vk::MemoryBarrier,
u32,
*const vk::BufferMemoryBarrier,
u32,
*const vk::ImageMemoryBarrier,
);
pub type PFN_vkCmdCopyImage = unsafe extern "system" fn(
vk::CommandBuffer,
vk::Image,
vk::ImageLayout,
vk::Image,
vk::ImageLayout,
u32,
*const vk::ImageCopy,
);
pub type PFN_vkGetImageSubresourceLayout = unsafe extern "system" fn(
vk::Device,
vk::Image,
*const vk::ImageSubresource,
*mut vk::SubresourceLayout,
);
// DMA-BUF fd export (used to share final_image with pixelforge zero-copy)
pub type PFN_vkGetMemoryFdKHR = unsafe extern "system" fn(
vk::Device,
*const vk::MemoryGetFdInfoKHR,
*mut std::os::raw::c_int,
) -> vk::Result;
// ── Phase 4: Synchronisation ─────────────────────────────────────────────────
pub type PFN_vkCreateFence = unsafe extern "system" fn(
vk::Device,
*const vk::FenceCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::Fence,
) -> vk::Result;
pub type PFN_vkDestroyFence =
unsafe extern "system" fn(vk::Device, vk::Fence, *const vk::AllocationCallbacks);
pub type PFN_vkCreateCommandPool = unsafe extern "system" fn(
vk::Device,
*const vk::CommandPoolCreateInfo,
*const vk::AllocationCallbacks,
*mut vk::CommandPool,
) -> vk::Result;
pub type PFN_vkDestroyCommandPool =
unsafe extern "system" fn(vk::Device, vk::CommandPool, *const vk::AllocationCallbacks);
pub type PFN_vkResetCommandPool =
unsafe extern "system" fn(vk::Device, vk::CommandPool, vk::CommandPoolResetFlags) -> vk::Result;
pub type PFN_vkBeginCommandBuffer =
unsafe extern "system" fn(vk::CommandBuffer, *const vk::CommandBufferBeginInfo) -> vk::Result;
pub type PFN_vkEndCommandBuffer = unsafe extern "system" fn(vk::CommandBuffer) -> vk::Result;
pub type PFN_vkQueueSubmit =
unsafe extern "system" fn(vk::Queue, u32, *const vk::SubmitInfo, vk::Fence) -> vk::Result;
pub type PFN_vkWaitForFences =
unsafe extern "system" fn(vk::Device, u32, *const vk::Fence, vk::Bool32, u64) -> vk::Result;
pub type PFN_vkResetFences =
unsafe extern "system" fn(vk::Device, u32, *const vk::Fence) -> vk::Result;
// ── Phase 4: Swapchain tracking ──────────────────────────────────────────────
pub type PFN_vkCreateSwapchainKHR = unsafe extern "system" fn(
vk::Device,
*const vk::SwapchainCreateInfoKHR,
*const vk::AllocationCallbacks,
*mut vk::SwapchainKHR,
) -> vk::Result;
pub type PFN_vkDestroySwapchainKHR =
unsafe extern "system" fn(vk::Device, vk::SwapchainKHR, *const vk::AllocationCallbacks);
pub type PFN_vkGetSwapchainImagesKHR =
unsafe extern "system" fn(vk::Device, vk::SwapchainKHR, *mut u32, *mut vk::Image) -> vk::Result;
// ── Phase 6: Draw commands ───────────────────────────────────────────────────
pub type PFN_vkCmdDraw = unsafe extern "system" fn(vk::CommandBuffer, u32, u32, u32, u32);
pub type PFN_vkCmdDrawIndexed =
unsafe extern "system" fn(vk::CommandBuffer, u32, u32, u32, i32, u32);
pub type PFN_vkCmdDrawIndirect =
unsafe extern "system" fn(vk::CommandBuffer, vk::Buffer, vk::DeviceSize, u32, u32);
pub type PFN_vkCmdDrawIndexedIndirect =
unsafe extern "system" fn(vk::CommandBuffer, vk::Buffer, vk::DeviceSize, u32, u32);
pub type PFN_vkCmdDrawIndirectCount = unsafe extern "system" fn(
vk::CommandBuffer,
vk::Buffer,
vk::DeviceSize,
vk::Buffer,
vk::DeviceSize,
u32,
u32,
);
pub type PFN_vkCmdDrawIndexedIndirectCount = unsafe extern "system" fn(
vk::CommandBuffer,
vk::Buffer,
vk::DeviceSize,
vk::Buffer,
vk::DeviceSize,
u32,
u32,
);
pub type PFN_vkResetCommandBuffer =
unsafe extern "system" fn(vk::CommandBuffer, vk::CommandBufferResetFlags) -> vk::Result;
// ── Dispatch table structs ────────────────────────────────────────────────────
pub struct NextInstanceFn {
pub get_instance_proc_addr: PFN_vkGetInstanceProcAddr,
pub destroy_instance: PFN_vkDestroyInstance,
pub get_physical_device_memory_properties: PFN_vkGetPhysicalDeviceMemoryProperties,
pub create_device: PFN_vkCreateDevice,
}
#[derive(Clone, Copy)]
pub struct NextDeviceFn {
// Infrastructure
pub get_device_proc_addr: PFN_vkGetDeviceProcAddr,
pub destroy_device: PFN_vkDestroyDevice,
pub get_device_queue: PFN_vkGetDeviceQueue,
pub queue_present_khr: Option<PFN_vkQueuePresentKHR>,
// Phase 1
pub create_shader_module: PFN_vkCreateShaderModule,
pub destroy_shader_module: PFN_vkDestroyShaderModule,
pub create_graphics_pipelines: PFN_vkCreateGraphicsPipelines,
pub destroy_pipeline: PFN_vkDestroyPipeline,
// Phase 2
pub create_image_view: PFN_vkCreateImageView,
pub destroy_image_view: PFN_vkDestroyImageView,
pub create_framebuffer: PFN_vkCreateFramebuffer,
pub destroy_framebuffer: PFN_vkDestroyFramebuffer,
pub allocate_command_buffers: PFN_vkAllocateCommandBuffers,
pub free_command_buffers: PFN_vkFreeCommandBuffers,
pub cmd_bind_pipeline: PFN_vkCmdBindPipeline,
pub cmd_begin_render_pass: PFN_vkCmdBeginRenderPass,
pub cmd_end_render_pass: PFN_vkCmdEndRenderPass,
pub cmd_begin_rendering_khr: Option<PFN_vkCmdBeginRenderingKHR>,
pub cmd_end_rendering_khr: Option<PFN_vkCmdEndRenderingKHR>,
// Phase 4 — capture images
pub create_image: PFN_vkCreateImage,
pub destroy_image: PFN_vkDestroyImage,
pub allocate_memory: PFN_vkAllocateMemory,
pub free_memory: PFN_vkFreeMemory,
pub bind_image_memory: PFN_vkBindImageMemory,
pub get_image_memory_requirements: PFN_vkGetImageMemoryRequirements,
pub map_memory: PFN_vkMapMemory,
pub unmap_memory: PFN_vkUnmapMemory,
pub cmd_pipeline_barrier: PFN_vkCmdPipelineBarrier,
pub cmd_copy_image: PFN_vkCmdCopyImage,
pub get_image_subresource_layout: PFN_vkGetImageSubresourceLayout,
/// `None` when `VK_KHR_external_memory_fd` is unavailable.
/// Required for DMA-BUF export to pixelforge's VkDevice.
pub get_memory_fd_khr: Option<PFN_vkGetMemoryFdKHR>,
// Phase 4 — synchronisation
pub create_fence: PFN_vkCreateFence,
pub destroy_fence: PFN_vkDestroyFence,
pub create_command_pool: PFN_vkCreateCommandPool,
pub destroy_command_pool: PFN_vkDestroyCommandPool,
pub reset_command_pool: PFN_vkResetCommandPool,
pub begin_command_buffer: PFN_vkBeginCommandBuffer,
pub end_command_buffer: PFN_vkEndCommandBuffer,
pub queue_submit: PFN_vkQueueSubmit,
pub wait_for_fences: PFN_vkWaitForFences,
pub reset_fences: PFN_vkResetFences,
// Phase 4 — swapchain
pub create_swapchain_khr: Option<PFN_vkCreateSwapchainKHR>,
pub destroy_swapchain_khr: Option<PFN_vkDestroySwapchainKHR>,
pub get_swapchain_images_khr: Option<PFN_vkGetSwapchainImagesKHR>,
// Phase 6 — draw commands
pub cmd_draw: PFN_vkCmdDraw,
pub cmd_draw_indexed: PFN_vkCmdDrawIndexed,
pub cmd_draw_indirect: PFN_vkCmdDrawIndirect,
pub cmd_draw_indexed_indirect: PFN_vkCmdDrawIndexedIndirect,
pub cmd_draw_indirect_count: Option<PFN_vkCmdDrawIndirectCount>,
pub cmd_draw_indexed_indirect_count: Option<PFN_vkCmdDrawIndexedIndirectCount>,
pub reset_command_buffer: PFN_vkResetCommandBuffer,
}

242
apps/nescapture/src/dmabuf_import.rs Normal file
View File

@@ -0,0 +1,242 @@
//! DMA-BUF import support for zero-copy video encoding.
//!
//! This module provides the ability to import Linux DMA-BUF file descriptors as
//! Vulkan images for direct video encoding without CPU-side copies.
//!
//! `DmaBufImporter` caches imported Vulkan resources per compositor buffer index
//! so that pre-allocated GBM buffers are imported only once. Subsequent frames
//! from the same buffer reuse the cached `VkImage` and `VkDeviceMemory`,
//! eliminating per-frame Vulkan object creation and layout transitions.
use anyhow::Result;
use ash::vk;
use log::debug;
use pixelforge::VideoContext;
use std::os::fd::RawFd;
use std::os::unix::io::{BorrowedFd, IntoRawFd};
/// Information about a single DMA-BUF plane.
#[derive(Debug, Clone, Copy)]
pub struct DmaBufPlane {
/// File descriptor for the DMA-BUF.
pub fd: RawFd,
/// Offset within the DMA-BUF to the start of this plane.
pub offset: u32,
/// Row stride in bytes.
pub stride: u32,
/// DRM format modifier.
pub modifier: u64,
}
/// Cached Vulkan resources for a single compositor buffer slot.
struct CachedImport {
image: vk::Image,
memory: vk::DeviceMemory,
}
/// Importer for DMA-BUF file descriptors into Vulkan images.
///
/// Owns a per-buffer-index cache of `VkImage` + `VkDeviceMemory`.
/// Layout transitions are deferred to the consumer (e.g. `ColorConverter`)
/// to avoid a separate GPU submission per first-time import.
pub struct DmaBufImporter {
context: VideoContext,
external_memory_fd: ash::khr::external_memory_fd::Device,
/// Per-buffer-index cache. Index corresponds to `ExportedFrame::buffer_index`.
cached_imports: Vec<Option<CachedImport>>,
}
impl DmaBufImporter {
/// Create a new DMA-BUF importer.
pub fn new(context: VideoContext) -> Result<Self> {
let external_memory_fd =
ash::khr::external_memory_fd::Device::load(context.instance(), context.device());
Ok(Self {
context,
external_memory_fd,
cached_imports: Vec::new(),
})
}
/// Import a DMA-BUF as a Vulkan image, reusing a cached import when
/// the same `buffer_index` has been seen before.
///
/// The `format` parameter specifies the Vulkan format matching the DMA-BUF
/// pixel format (e.g. `B8G8R8A8_UNORM` for SDR, `A2B10G10R10_UNORM_PACK32`
/// for 10-bit HDR, `R16G16B16A16_SFLOAT` for FP16 HDR).
///
/// Returns `(image, needs_transition)` where `needs_transition` is `true`
/// for first-time imports whose image is still in `UNDEFINED` layout.
/// The caller is responsible for transitioning the image (e.g. by passing
/// the appropriate `src_layout` to `ColorConverter::convert`).
pub fn import_or_reuse(
&mut self,
buffer_index: usize,
width: u32,
height: u32,
format: vk::Format,
planes: &[DmaBufPlane],
) -> Result<(vk::Image, bool)> {
// Grow the cache vector if needed.
if self.cached_imports.len() <= buffer_index {
self.cached_imports.resize_with(buffer_index + 1, || None);
}
if let Some(cached) = &self.cached_imports[buffer_index] {
return Ok((cached.image, false));
}
// First time seeing this buffer — full import.
debug!(
"First import for buffer {buffer_index}: {}x{}, format={:?}, fd={}, stride={}, modifier={:#x}",
width, height, format, planes[0].fd, planes[0].stride, planes[0].modifier
);
let (image, memory) = self.import_internal(width, height, format, planes)?;
self.cached_imports[buffer_index] = Some(CachedImport { image, memory });
Ok((image, true))
}
/// Perform the raw Vulkan import of a DMA-BUF with the specified format.
///
/// Returns the `(VkImage, VkDeviceMemory)` pair. The image is in
/// `UNDEFINED` layout; the caller must transition it.
fn import_internal(
&self,
width: u32,
height: u32,
format: vk::Format,
planes: &[DmaBufPlane],
) -> Result<(vk::Image, vk::DeviceMemory)> {
if planes.is_empty() {
return Err(anyhow::anyhow!("At least one DMA-BUF plane is required"));
}
let device = self.context.device();
// Build DRM format modifier plane layouts for all planes.
// AMD modifiers (e.g. tiled/DCC) may require multiple planes;
// the layout count must match the modifier's expected plane count.
let plane_layouts: Vec<vk::SubresourceLayout> = planes
.iter()
.map(|p| {
vk::SubresourceLayout::default()
.offset(p.offset as u64)
.row_pitch(p.stride as u64)
})
.collect();
let modifier = planes[0].modifier;
let mut drm_format_modifier_info =
vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
.drm_format_modifier(modifier)
.plane_layouts(&plane_layouts);
let mut external_memory_info = vk::ExternalMemoryImageCreateInfo::default()
.handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
external_memory_info.p_next = &mut drm_format_modifier_info
as *mut vk::ImageDrmFormatModifierExplicitCreateInfoEXT
as *mut _;
let mut image_create_info = vk::ImageCreateInfo::default()
.image_type(vk::ImageType::TYPE_2D)
.format(format)
.extent(vk::Extent3D {
width,
height,
depth: 1,
})
.mip_levels(1)
.array_layers(1)
.samples(vk::SampleCountFlags::TYPE_1)
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
.usage(vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::SAMPLED)
.sharing_mode(vk::SharingMode::EXCLUSIVE)
.initial_layout(vk::ImageLayout::UNDEFINED);
image_create_info.p_next =
&mut external_memory_info as *mut vk::ExternalMemoryImageCreateInfo as *mut _;
let image = unsafe { device.create_image(&image_create_info, None) }
.map_err(|e| anyhow::anyhow!("DMA-BUF image creation: {e}"))?;
// Memory requirements.
let mem_requirements = unsafe { device.get_image_memory_requirements(image) };
// FD memory properties.
let mut memory_fd_properties = vk::MemoryFdPropertiesKHR::default();
unsafe {
self.external_memory_fd.get_memory_fd_properties(
vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT,
planes[0].fd,
&mut memory_fd_properties,
)
}
.map_err(|e| anyhow::anyhow!("Failed to get memory FD properties: {e}"))?;
// Duplicate the FD — vkAllocateMemory consumes it.
let fd = unsafe { BorrowedFd::borrow_raw(planes[0].fd) }
.try_clone_to_owned()
.map_err(|e| anyhow::anyhow!("Failed to duplicate DMA-BUF FD: {e}"))?
.into_raw_fd();
let mut import_memory_fd_info = vk::ImportMemoryFdInfoKHR::default()
.handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
.fd(fd);
let memory_type_bits =
mem_requirements.memory_type_bits & memory_fd_properties.memory_type_bits;
debug!(
"Memory allocation: size={}, image_type_bits={:#x}, fd_type_bits={:#x}, combined={:#x}",
mem_requirements.size,
mem_requirements.memory_type_bits,
memory_fd_properties.memory_type_bits,
memory_type_bits
);
let memory_type_index = self
.context
.find_memory_type(memory_type_bits, vk::MemoryPropertyFlags::empty())
.ok_or_else(|| anyhow::anyhow!("No suitable memory type for DMA-BUF import"))?;
// Dedicated allocation (required by many drivers for external memory).
let mut dedicated_alloc_info = vk::MemoryDedicatedAllocateInfo::default().image(image);
import_memory_fd_info.p_next =
&mut dedicated_alloc_info as *mut vk::MemoryDedicatedAllocateInfo as *mut _;
let mut alloc_info = vk::MemoryAllocateInfo::default()
.allocation_size(mem_requirements.size)
.memory_type_index(memory_type_index);
alloc_info.p_next = &mut import_memory_fd_info as *mut vk::ImportMemoryFdInfoKHR as *mut _;
let memory = unsafe { device.allocate_memory(&alloc_info, None) }.map_err(|e| {
unsafe { device.destroy_image(image, None) };
anyhow::anyhow!("DMA-BUF memory import: {e}")
})?;
if let Err(e) = unsafe { device.bind_image_memory(image, memory, 0) } {
unsafe {
device.free_memory(memory, None);
device.destroy_image(image, None);
}
return Err(anyhow::anyhow!("DMA-BUF memory bind: {e}"));
}
Ok((image, memory))
}
}
impl Drop for DmaBufImporter {
fn drop(&mut self) {
let device = self.context.device();
unsafe {
// Clean up cached imports.
for cached in self.cached_imports.drain(..).flatten() {
device.destroy_image(cached.image, None);
device.free_memory(cached.memory, None);
}
}
}
}

1105
apps/nescapture/src/encode.rs Normal file
View File

File diff suppressed because it is too large Load Diff

193
apps/nescapture/src/framebuffer.rs Normal file
View File

@@ -0,0 +1,193 @@
// ─────────────────────────────────────────────────────────────────────────────
// framebuffer.rs — Phase 2: image view and framebuffer tracking
//
// We need to know which VkImage is bound as a color attachment at draw time.
// The chain is: VkFramebuffer → VkImageView → VkImage + VkFormat.
//
// vkCreateImageView → view_to_image[view] = image, view_format[view] = fmt
// vkCreateFramebuffer → framebuffer_to_views[fb] = views, framebuffer_extent[fb] = extent
// vkAllocateCommandBuffers → cmd_buf_to_device_key[cmd_buf] = device_key
// vkFreeCommandBuffers → remove from CB_STATE and cmd_buf_to_device_key
// ─────────────────────────────────────────────────────────────────────────────
use crate::dispatch_key;
use crate::state::{CB_STATE, CMD_BUF_TO_DEVICE_KEY, DEVICE_STATE};
use ash::vk::{self, Handle};
use std::os::raw::c_void;
// ─────────────────────────────────────────────────────────────────────────────
// vkCreateImageView — map view → image and view → format
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateImageView(
device: vk::Device,
p_create_info: *const vk::ImageViewCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_view: *mut vk::ImageView,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
let result = unsafe { (ds.fp.create_image_view)(device, p_create_info, p_allocator, p_view) };
if result != vk::Result::SUCCESS {
return result;
}
let ci = unsafe { &*p_create_info };
let view = unsafe { *p_view };
let image = ci.image;
ds.view_to_image.insert(view.as_raw(), image.as_raw());
ds.view_format.insert(view.as_raw(), ci.format);
/*log::trace!(
"image view {:#010x} → image {:#010x} format={}",
view.as_raw(),
image.as_raw(),
ci.format.as_raw(),
);*/
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyImageView(
device: vk::Device,
image_view: vk::ImageView,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
ds.view_to_image.remove(&image_view.as_raw());
ds.view_format.remove(&image_view.as_raw());
unsafe { (ds.fp.destroy_image_view)(device, image_view, p_allocator) };
}
}
// ─────────────────────────────────────────────────────────────────────────────
// vkCreateFramebuffer — map framebuffer → views and framebuffer → extent
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateFramebuffer(
device: vk::Device,
p_create_info: *const vk::FramebufferCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_framebuffer: *mut vk::Framebuffer,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
let result =
unsafe { (ds.fp.create_framebuffer)(device, p_create_info, p_allocator, p_framebuffer) };
if result != vk::Result::SUCCESS {
return result;
}
let ci = unsafe { &*p_create_info };
let fb = unsafe { *p_framebuffer };
let views: Vec<u64> =
unsafe { std::slice::from_raw_parts(ci.p_attachments, ci.attachment_count as usize) }
.iter()
.map(|v| v.as_raw())
.collect();
ds.framebuffer_to_views.insert(fb.as_raw(), views.clone());
ds.framebuffer_extent.insert(
fb.as_raw(),
vk::Extent2D {
width: ci.width,
height: ci.height,
},
);
log::trace!(
"framebuffer {:#010x} → {} views, extent {}x{}",
fb.as_raw(),
ci.attachment_count,
ci.width,
ci.height,
);
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyFramebuffer(
device: vk::Device,
framebuffer: vk::Framebuffer,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
ds.framebuffer_to_views.remove(&framebuffer.as_raw());
ds.framebuffer_extent.remove(&framebuffer.as_raw());
unsafe { (ds.fp.destroy_framebuffer)(device, framebuffer, p_allocator) };
}
}
// ─────────────────────────────────────────────────────────────────────────────
// vkAllocateCommandBuffers — register command buffer → device key mapping
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkAllocateCommandBuffers(
device: vk::Device,
p_allocate_info: *const vk::CommandBufferAllocateInfo,
p_command_buffers: *mut vk::CommandBuffer,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
let result =
unsafe { (ds.fp.allocate_command_buffers)(device, p_allocate_info, p_command_buffers) };
if result != vk::Result::SUCCESS {
return result;
}
let ai = unsafe { &*p_allocate_info };
let count = ai.command_buffer_count;
let buffers = unsafe { std::slice::from_raw_parts(p_command_buffers, count as usize) };
for &cb in buffers {
CMD_BUF_TO_DEVICE_KEY.insert(cb.as_raw(), key);
}
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkFreeCommandBuffers(
device: vk::Device,
command_pool: vk::CommandPool,
command_buffer_count: u32,
p_command_buffers: *const vk::CommandBuffer,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
let buffers =
unsafe { std::slice::from_raw_parts(p_command_buffers, command_buffer_count as usize) };
for &cb in buffers {
CMD_BUF_TO_DEVICE_KEY.remove(&cb.as_raw());
CB_STATE.remove(&cb.as_raw());
}
unsafe {
(ds.fp.free_command_buffers)(
device,
command_pool,
command_buffer_count,
p_command_buffers,
)
};
}
}

80
apps/nescapture/src/instance.rs Normal file
View File

@@ -0,0 +1,80 @@
// ─────────────────────────────────────────────────────────────────────────────
// instance.rs — vkCreateInstance and vkDestroyInstance
// ─────────────────────────────────────────────────────────────────────────────
use crate::dispatch::NextInstanceFn;
use crate::state::INSTANCE_STATE;
use crate::{VkLayerInstanceCreateInfo, dispatch_key, find_layer_link, load_instance_fn};
use ash::vk::{self, Handle};
use std::os::raw::c_void;
use std::sync::Arc;
use crate::dispatch::PFN_vkCreateInstance;
const VK_LAYER_LINK_INFO: u32 = 0;
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateInstance(
p_create_info: *const vk::InstanceCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_instance: *mut vk::Instance,
) -> vk::Result {
crate::init_logger();
// ── Walk the pNext chain to find the loader's layer link ─────────────────
let layer_info: *mut VkLayerInstanceCreateInfo = match unsafe {
find_layer_link((*p_create_info).p_next as *const c_void, VK_LAYER_LINK_INFO)
} {
Some(p) => p,
None => return vk::Result::ERROR_INITIALIZATION_FAILED,
};
let layer_link = unsafe { (*layer_info).u.pLayerInfo };
let next_gipa = match unsafe { (*layer_link).pfnNextGetInstanceProcAddr } {
Some(f) => f,
None => return vk::Result::ERROR_INITIALIZATION_FAILED,
};
// Advance the chain before calling through so the next layer gets its link.
unsafe { (*layer_info).u.pLayerInfo = (*layer_link).pNext };
// ── Call the next layer / loader's vkCreateInstance ──────────────────────
let next_create: PFN_vkCreateInstance =
unsafe { load_instance_fn(next_gipa, vk::Instance::null(), b"vkCreateInstance\0") };
let result = unsafe { next_create(p_create_info, p_allocator, p_instance) };
if result != vk::Result::SUCCESS {
return result;
}
let instance = unsafe { *p_instance };
let key = unsafe { dispatch_key(instance.as_raw() as *const c_void) };
// ── Build and store per-instance dispatch table ───────────────────────────
let istate = Arc::new(NextInstanceFn {
get_instance_proc_addr: next_gipa,
destroy_instance: unsafe { load_instance_fn(next_gipa, instance, b"vkDestroyInstance\0") },
get_physical_device_memory_properties: unsafe {
load_instance_fn(
next_gipa,
instance,
b"vkGetPhysicalDeviceMemoryProperties\0",
)
},
create_device: unsafe { load_instance_fn(next_gipa, instance, b"vkCreateDevice\0") },
});
INSTANCE_STATE.insert(key, istate);
log::debug!("vkCreateInstance OK (enabled={})", crate::enabled());
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyInstance(
instance: vk::Instance,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(instance.as_raw() as *const c_void) };
if let Some((_, state)) = INSTANCE_STATE.remove(&key) {
unsafe { (state.destroy_instance)(instance, p_allocator) };
}
}

353
apps/nescapture/src/lib.rs Normal file
View File

@@ -0,0 +1,353 @@
// ─────────────────────────────────────────────────────────────────────────────
// lib.rs — nescapture Vulkan implicit layer
//
// Entry points:
// vkNegotiateLoaderLayerInterfaceVersion
// vkGetInstanceProcAddr
// vkGetDeviceProcAddr
//
// To add a new hook:
// 1. Implement it in the relevant module.
// 2. Add it to `match_device_fn` below.
// 3. Export it with #[unsafe(no_mangle)] in the module.
// ─────────────────────────────────────────────────────────────────────────────
#![allow(
non_snake_case,
non_camel_case_types,
dead_code,
unused_variables,
clippy::missing_safety_doc,
clippy::too_many_arguments
)]
mod capture;
mod commands;
mod config;
mod device;
mod discovery;
mod dispatch;
mod dmabuf_import;
mod encode;
mod framebuffer;
mod instance;
mod pipeline;
mod present;
mod shader;
mod state;
mod swapchain;
use commands::{
vkCmdBeginRenderPass, vkCmdBeginRenderingKHR, vkCmdBindPipeline, vkCmdDraw, vkCmdDrawIndexed,
vkCmdDrawIndexedIndirect, vkCmdDrawIndexedIndirectCount, vkCmdDrawIndexedIndirectCountKHR,
vkCmdDrawIndirect, vkCmdDrawIndirectCount, vkCmdDrawIndirectCountKHR, vkCmdEndRenderPass,
vkCmdEndRenderingKHR,
};
use device::{vkCreateDevice, vkDestroyDevice, vkGetDeviceQueue};
use framebuffer::{
vkAllocateCommandBuffers, vkCreateFramebuffer, vkCreateImageView, vkDestroyFramebuffer,
vkDestroyImageView, vkFreeCommandBuffers,
};
use instance::{vkCreateInstance, vkDestroyInstance};
use pipeline::{vkCreateGraphicsPipelines, vkDestroyPipeline};
use present::vkQueuePresentKHR;
use shader::{vkCreateShaderModule, vkDestroyShaderModule};
use swapchain::{vkCreateSwapchainKHR, vkDestroySwapchainKHR, vkGetSwapchainImagesKHR};
use dispatch::{PFN_vkGetDeviceProcAddr, PFN_vkGetInstanceProcAddr, RawFn};
use state::{DEVICE_STATE, INSTANCE_STATE};
use ash::vk::{self, Handle};
use once_cell::sync::OnceCell;
use std::ffi::CStr;
use std::os::raw::{c_char, c_void};
const ENABLE_ENV: &str = "NESCAPTURE_ENABLE";
pub(crate) fn enabled() -> bool {
std::env::var(ENABLE_ENV).map(|v| v == "1").unwrap_or(false)
}
static LOGGER: OnceCell<()> = OnceCell::new();
pub(crate) fn init_logger() {
LOGGER.get_or_init(|| {
env_logger::Builder::from_default_env().init();
});
}
// ── Vulkan loader structs (not in ash) ────────────────────────────────────────
#[repr(C)]
pub(crate) struct VkLayerInstanceLink {
pNext: *mut VkLayerInstanceLink,
pfnNextGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
pfnNextGetPhysicalDeviceProcAddr: Option<RawFn>,
}
#[repr(C)]
pub(crate) struct VkLayerDeviceLink {
pNext: *mut VkLayerDeviceLink,
pfnNextGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
pfnNextGetDeviceProcAddr: Option<PFN_vkGetDeviceProcAddr>,
}
#[repr(C)]
pub(crate) union VkLayerCreateInfoU {
pub pLayerInfo: *mut VkLayerInstanceLink,
pub pDeviceLayerInfo: *mut VkLayerDeviceLink,
}
const VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO: i32 = 47;
const VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO: i32 = 48;
#[repr(C)]
pub(crate) struct VkLayerInstanceCreateInfo {
pub sType: i32,
pub pNext: *const c_void,
pub function: u32,
pub u: VkLayerCreateInfoU,
}
pub(crate) type VkLayerDeviceCreateInfo = VkLayerInstanceCreateInfo;
pub(crate) unsafe fn dispatch_key(handle: *const c_void) -> usize {
unsafe { *(handle as *const usize) }
}
pub(crate) unsafe fn find_layer_link<T>(p_next: *const c_void, function: u32) -> Option<*mut T> {
let mut current = p_next;
while !current.is_null() {
let header = current as *const VkLayerInstanceCreateInfo;
let s = unsafe { (*header).sType };
if (s == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO
|| s == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO)
&& unsafe { (*header).function } == function
{
return Some(header as *mut T);
}
current = unsafe { (*header).pNext };
}
None
}
pub(crate) unsafe fn load_device_fn<T>(
get: PFN_vkGetDeviceProcAddr,
device: vk::Device,
name: &[u8],
) -> T {
let raw = unsafe {
get(device, name.as_ptr() as *const c_char)
.unwrap_or_else(|| panic!("missing device fn: {}", core::str::from_utf8(name).unwrap()))
};
unsafe { std::mem::transmute_copy(&raw) }
}
pub(crate) unsafe fn try_load_device_fn<T>(
get: PFN_vkGetDeviceProcAddr,
device: vk::Device,
name: &[u8],
) -> Option<T> {
let raw = unsafe { get(device, name.as_ptr() as *const c_char) }?;
Some(unsafe { std::mem::transmute_copy(&raw) })
}
pub(crate) unsafe fn load_instance_fn<T>(
get: PFN_vkGetInstanceProcAddr,
instance: vk::Instance,
name: &[u8],
) -> T {
let raw = unsafe {
get(instance, name.as_ptr() as *const c_char).unwrap_or_else(|| {
panic!(
"missing instance fn: {}",
core::str::from_utf8(name).unwrap()
)
})
};
unsafe { std::mem::transmute_copy(&raw) }
}
#[inline]
pub(crate) unsafe fn to_raw(addr: usize) -> RawFn {
unsafe { std::mem::transmute(addr) }
}
macro_rules! layer_fn {
($f:ident) => {
return Some(unsafe { to_raw($f as *const () as usize) })
};
}
// ─────────────────────────────────────────────────────────────────────────────
// vkGetInstanceProcAddr
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkGetInstanceProcAddr(
instance: vk::Instance,
p_name: *const c_char,
) -> Option<RawFn> {
let name = unsafe { CStr::from_ptr(p_name).to_bytes() };
match name {
b"vkGetInstanceProcAddr" => layer_fn!(vkGetInstanceProcAddr),
b"vkCreateInstance" => layer_fn!(vkCreateInstance),
b"vkDestroyInstance" => layer_fn!(vkDestroyInstance),
b"vkCreateDevice" => layer_fn!(vkCreateDevice),
_ => {}
}
if let Some(f) = unsafe { match_device_fn(name) } {
return Some(f);
}
if instance.as_raw() == 0 {
return None;
}
let ikey = unsafe { dispatch_key(instance.as_raw() as *const c_void) };
if let Some(istate) = INSTANCE_STATE.get(&ikey) {
return unsafe { (istate.get_instance_proc_addr)(instance, p_name) };
}
None
}
// ─────────────────────────────────────────────────────────────────────────────
// vkGetDeviceProcAddr
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkGetDeviceProcAddr(
device: vk::Device,
p_name: *const c_char,
) -> Option<RawFn> {
let name = unsafe { CStr::from_ptr(p_name).to_bytes() };
if let Some(f) = unsafe { match_device_fn(name) } {
return Some(f);
}
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
return unsafe { (ds.fp.get_device_proc_addr)(device, p_name) };
}
None
}
unsafe fn match_device_fn(name: &[u8]) -> Option<RawFn> {
let name_str = std::str::from_utf8(name).unwrap_or("<non-utf8>");
// Only log swapchain/acquire/present functions to keep noise down
let log_this = name_str.contains("Swapchain")
|| name_str.contains("AcquireNextImage")
|| name_str.contains("QueuePresent")
|| name_str.contains("CreateDevice")
|| name_str.contains("GetSwapchain");
if log_this {
log::debug!("gdpa: {}", name_str);
}
unsafe {
match name {
b"vkGetDeviceProcAddr" => Some(to_raw(vkGetDeviceProcAddr as *const () as usize)),
b"vkCreateDevice" => Some(to_raw(vkCreateDevice as *const () as usize)),
b"vkDestroyDevice" => Some(to_raw(vkDestroyDevice as *const () as usize)),
b"vkGetDeviceQueue" => Some(to_raw(vkGetDeviceQueue as *const () as usize)),
b"vkQueuePresentKHR" => Some(to_raw(vkQueuePresentKHR as *const () as usize)),
b"vkCreateShaderModule" => Some(to_raw(vkCreateShaderModule as *const () as usize)),
b"vkDestroyShaderModule" => Some(to_raw(vkDestroyShaderModule as *const () as usize)),
b"vkCreateGraphicsPipelines" => {
Some(to_raw(vkCreateGraphicsPipelines as *const () as usize))
}
b"vkDestroyPipeline" => Some(to_raw(vkDestroyPipeline as *const () as usize)),
b"vkCreateImageView" => Some(to_raw(vkCreateImageView as *const () as usize)),
b"vkDestroyImageView" => Some(to_raw(vkDestroyImageView as *const () as usize)),
b"vkCreateFramebuffer" => Some(to_raw(vkCreateFramebuffer as *const () as usize)),
b"vkDestroyFramebuffer" => Some(to_raw(vkDestroyFramebuffer as *const () as usize)),
b"vkAllocateCommandBuffers" => {
Some(to_raw(vkAllocateCommandBuffers as *const () as usize))
}
b"vkFreeCommandBuffers" => Some(to_raw(vkFreeCommandBuffers as *const () as usize)),
b"vkCmdBindPipeline" => Some(to_raw(vkCmdBindPipeline as *const () as usize)),
b"vkCmdBeginRenderPass" => Some(to_raw(vkCmdBeginRenderPass as *const () as usize)),
b"vkCmdEndRenderPass" => Some(to_raw(vkCmdEndRenderPass as *const () as usize)),
b"vkCmdBeginRenderingKHR" => Some(to_raw(vkCmdBeginRenderingKHR as *const () as usize)),
b"vkCmdEndRenderingKHR" => Some(to_raw(vkCmdEndRenderingKHR as *const () as usize)),
b"vkCmdDraw" => Some(to_raw(vkCmdDraw as *const () as usize)),
b"vkCmdDrawIndexed" => Some(to_raw(vkCmdDrawIndexed as *const () as usize)),
b"vkCmdDrawIndirect" => Some(to_raw(vkCmdDrawIndirect as *const () as usize)),
b"vkCmdDrawIndexedIndirect" => {
Some(to_raw(vkCmdDrawIndexedIndirect as *const () as usize))
}
b"vkCmdDrawIndirectCount" => Some(to_raw(vkCmdDrawIndirectCount as *const () as usize)),
b"vkCmdDrawIndexedIndirectCount" => {
Some(to_raw(vkCmdDrawIndexedIndirectCount as *const () as usize))
}
b"vkCmdDrawIndirectCountKHR" => {
Some(to_raw(vkCmdDrawIndirectCountKHR as *const () as usize))
}
b"vkCmdDrawIndexedIndirectCountKHR" => Some(to_raw(
vkCmdDrawIndexedIndirectCountKHR as *const () as usize,
)),
b"vkCreateSwapchainKHR" => {
if log_this {
log::debug!("gdpa: → our vkCreateSwapchainKHR");
}
Some(to_raw(vkCreateSwapchainKHR as *const () as usize))
}
b"vkDestroySwapchainKHR" => {
if log_this {
log::debug!("gdpa: → our vkDestroySwapchainKHR");
}
Some(to_raw(vkDestroySwapchainKHR as *const () as usize))
}
b"vkGetSwapchainImagesKHR" => {
if log_this {
log::debug!("gdpa: → our vkGetSwapchainImagesKHR");
}
Some(to_raw(vkGetSwapchainImagesKHR as *const () as usize))
}
_ => {
if log_this {
log::debug!("gdpa: → passthrough (not intercepted)");
}
None
}
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// vkNegotiateLoaderLayerInterfaceVersion
// ─────────────────────────────────────────────────────────────────────────────
#[repr(C)]
pub struct VkNegotiateLayerInterface {
pub sType: u32,
pub pNext: *mut c_void,
pub loaderLayerInterfaceVersion: u32,
pub pfnGetInstanceProcAddr: Option<PFN_vkGetInstanceProcAddr>,
pub pfnGetDeviceProcAddr: Option<PFN_vkGetDeviceProcAddr>,
pub pfnGetPhysicalDeviceProcAddr:
Option<unsafe extern "system" fn(vk::Instance, *const c_char) -> Option<RawFn>>,
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkNegotiateLoaderLayerInterfaceVersion(
p_version_struct: *mut VkNegotiateLayerInterface,
) -> vk::Result {
init_logger();
let s = unsafe { &mut *p_version_struct };
if s.loaderLayerInterfaceVersion > 2 {
s.loaderLayerInterfaceVersion = 2;
}
s.pfnGetInstanceProcAddr = Some(vkGetInstanceProcAddr);
s.pfnGetDeviceProcAddr = Some(vkGetDeviceProcAddr);
s.pfnGetPhysicalDeviceProcAddr = None;
vk::Result::SUCCESS
}

144
apps/nescapture/src/pipeline.rs Normal file
View File

@@ -0,0 +1,144 @@
// ─────────────────────────────────────────────────────────────────────────────
// pipeline.rs — Phase 1: graphics pipeline tracking
//
// When the game calls vkCreateGraphicsPipelines, we:
// 1. Let the call through to the next layer / driver.
// 2. For each created pipeline, walk its shader stages.
// 3. Look up each stage's VkShaderModule in shader_registry to get its hash.
// 4. Store (VkPipeline → PipelineHashes{vert_hash, frag_hash}) in
// DeviceState::pipeline_registry.
//
// vkDestroyPipeline removes the entry to keep the map bounded.
//
// Phases 3+ use pipeline_registry in vkCmdBindPipeline to know which
// shader hashes are active when draw calls are issued.
// ─────────────────────────────────────────────────────────────────────────────
use crate::dispatch_key;
use crate::state::{DEVICE_STATE, PipelineHashes, PipelineState};
use ash::vk::{self, Handle};
use std::os::raw::c_void;
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateGraphicsPipelines(
device: vk::Device,
pipeline_cache: vk::PipelineCache,
create_info_count: u32,
p_create_infos: *const vk::GraphicsPipelineCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_pipelines: *mut vk::Pipeline,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
// Call through first. On success, p_pipelines is populated.
let result = unsafe {
(ds.fp.create_graphics_pipelines)(
device,
pipeline_cache,
create_info_count,
p_create_infos,
p_allocator,
p_pipelines,
)
};
if result != vk::Result::SUCCESS {
return result;
}
let infos = unsafe { std::slice::from_raw_parts(p_create_infos, create_info_count as usize) };
let pipelines = unsafe { std::slice::from_raw_parts(p_pipelines, create_info_count as usize) };
for (ci, &pipeline) in infos.iter().zip(pipelines.iter()) {
if pipeline == vk::Pipeline::null() {
// Can happen when VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT is set.
continue;
}
let stages = unsafe { std::slice::from_raw_parts(ci.p_stages, ci.stage_count as usize) };
let mut vert_hash: Option<u64> = None;
let mut frag_hash: Option<u64> = None;
for stage in stages {
let module_hash = ds.shader_registry.get(&stage.module.as_raw()).map(|r| *r);
match stage.stage {
vk::ShaderStageFlags::VERTEX => vert_hash = module_hash,
vk::ShaderStageFlags::FRAGMENT => frag_hash = module_hash,
_ => {}
}
}
// Phase 6: extract blend and depth state for discovery mode
let mut blend_enabled = false;
if !ci.p_color_blend_state.is_null() {
let blend = unsafe { &*ci.p_color_blend_state };
for i in 0..blend.attachment_count as usize {
let attachment = unsafe { &*blend.p_attachments.add(i) };
if attachment.blend_enable != vk::FALSE {
blend_enabled = true;
break;
}
}
}
let mut depth_test_enabled = false;
let mut depth_write_enabled = false;
if !ci.p_depth_stencil_state.is_null() {
let ds_state = unsafe { &*ci.p_depth_stencil_state };
depth_test_enabled = ds_state.depth_test_enable != vk::FALSE;
depth_write_enabled = ds_state.depth_write_enable != vk::FALSE;
}
/*log::trace!(
"pipeline {:#010x} → vert={} frag={} blend={} depth_test={} depth_write={}",
pipeline.as_raw(),
vert_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
frag_hash
.map(|h| format!("{:#018x}", h))
.unwrap_or_else(|| "none".to_string()),
blend_enabled,
depth_test_enabled,
depth_write_enabled,
);*/
ds.pipeline_registry.insert(
pipeline.as_raw(),
PipelineHashes {
vert_hash,
frag_hash,
},
);
ds.pipeline_state.insert(
pipeline.as_raw(),
PipelineState {
blend_enabled,
depth_test_enabled,
depth_write_enabled,
},
);
}
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyPipeline(
device: vk::Device,
pipeline: vk::Pipeline,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
ds.pipeline_registry.remove(&pipeline.as_raw());
ds.pipeline_state.remove(&pipeline.as_raw());
unsafe { (ds.fp.destroy_pipeline)(device, pipeline, p_allocator) };
}
}

214
apps/nescapture/src/present.rs Normal file
View File

@@ -0,0 +1,214 @@
use crate::capture;
use crate::encode::{CapturedFrame, FrameSource, PipelineConfig, PipelineHandle};
use crate::state::{DEVICE_STATE, QUEUE_TO_DEVICE_KEY};
use ash::vk::{self, Handle};
use std::os::raw::c_void;
use std::sync::atomic::Ordering;
use std::sync::mpsc;
pub struct CaptureJob {
pub queue: vk::Queue,
pub sc_image: vk::Image,
pub sc_fmt: vk::Format,
pub sc_ext: vk::Extent2D,
pub frame: u64,
pub ds_key: usize,
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkQueuePresentKHR(
queue: vk::Queue,
p_present_info: *const vk::PresentInfoKHR,
) -> vk::Result {
let ds = {
let dk = unsafe { crate::dispatch_key(queue.as_raw() as *const c_void) };
DEVICE_STATE
.get(&dk)
.map(|r| r.clone())
.or_else(|| {
QUEUE_TO_DEVICE_KEY
.get(&queue.as_raw())
.and_then(|dk| DEVICE_STATE.get(dk.value()).map(|r| r.clone()))
})
.or_else(|| DEVICE_STATE.iter().next().map(|e| e.value().clone()))
};
let ds = match ds {
Some(d) => d,
None => return vk::Result::ERROR_DEVICE_LOST,
};
let frame = ds.frame_counter.fetch_add(1, Ordering::Relaxed);
ds.hud_detected_frame.store(false, Ordering::Relaxed);
ds.pending_capture_frame.store(false, Ordering::Relaxed);
ds.capture_injected_frame.store(false, Ordering::Relaxed);
ds.skipped_draws_frame.store(0, Ordering::Relaxed);
if let Ok(enc) = ds.encoder.lock() {
if let Some(ref h) = *enc {
h.present_attempts.fetch_add(1, Ordering::Relaxed);
}
}
let pi = unsafe { &*p_present_info };
if pi.swapchain_count > 0 && !pi.p_swapchains.is_null() && !pi.p_image_indices.is_null() {
let idx = unsafe { *pi.p_image_indices as usize };
let (sc_image, sc_fmt, sc_ext) = {
let images = ds.swapchain_images.lock().unwrap();
let fmt = *ds.swapchain_format.lock().unwrap();
let ext = *ds.swapchain_extent.lock().unwrap();
if idx < images.len() && ext.width > 0 && ext.height > 0 {
(Some(images[idx]), fmt, ext)
} else {
(None, fmt, ext)
}
};
if let Some(sc_image) = sc_image {
// No time-based throttle — let the encoder channel provide natural backpressure
let should = true;
if should {
if let Ok(enc) = ds.encoder.lock() {
if let Some(ref h) = *enc {
h.capture_attempts.fetch_add(1, Ordering::Relaxed);
}
}
// Ensure capture worker is running
{
let mut ctx = ds.capture_tx.lock().unwrap();
if ctx.is_none() {
let (tx, rx) = mpsc::channel();
let key = unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) };
start_capture_worker(key, rx);
*ctx = Some(tx);
}
}
// Queue job to worker thread — don't block present
let job = CaptureJob {
queue,
sc_image,
sc_fmt,
sc_ext,
frame,
ds_key: unsafe { crate::dispatch_key(ds.raw.as_raw() as *const c_void) },
};
if let Ok(capture_tx) = ds.capture_tx.lock() {
let _ = capture_tx.as_ref().unwrap().send(job);
}
}
}
}
match ds.fp.queue_present_khr {
Some(f) => unsafe { f(queue, p_present_info) },
None => vk::Result::ERROR_EXTENSION_NOT_PRESENT,
}
}
pub fn start_capture_worker(ds_key: usize, capture_rx: mpsc::Receiver<CaptureJob>) {
std::thread::Builder::new()
.name("nescapture-capture".into())
.spawn(move || {
while let Ok(job) = capture_rx.recv() {
let t0 = std::time::Instant::now();
let ds = match DEVICE_STATE.get(&job.ds_key) {
Some(s) => s.clone(),
None => {
log::error!("capture worker: device state gone");
break;
}
};
// Do the blit on the worker's own time
unsafe {
capture::capture_final_frame(
&ds,
job.queue,
job.sc_image,
job.sc_fmt,
job.sc_ext,
job.frame,
)
};
// Export DMA-BUF and push to encoder
let source = {
let mem_guard = ds.final_memory.lock().unwrap();
let stride = ds.final_stride.load(Ordering::Relaxed);
if let Some(mem) = *mem_guard {
unsafe { try_make_dmabuf_source(&ds, mem, stride) }.unwrap_or_else(|| {
let (w, h, _) = *ds.final_size.lock().unwrap();
let img = ds.final_image.lock().unwrap().unwrap();
unsafe { capture::read_frame_pixels(&ds, img, mem, w, h) }
.map(FrameSource::Pixels)
.unwrap_or(FrameSource::Pixels(Vec::new()))
})
} else {
continue;
}
};
let (w, h, _) = *ds.final_size.lock().unwrap();
if !matches!(&source, FrameSource::Pixels(p) if p.is_empty()) {
// Lazy-init encoder
{
let mut enc = ds.encoder.lock().unwrap();
if enc.is_none() {
if let Some(cfg) = PipelineConfig::from_env(w, h) {
ds.target_fps.store(cfg.fps, Ordering::Relaxed);
match PipelineHandle::new(cfg) {
Ok(h) => *enc = Some(h),
Err(e) => panic!("{e}"),
}
}
}
}
let enc_guard = ds.encoder.lock().unwrap();
if let Some(ref encoder) = *enc_guard {
let capture_elapsed = t0.elapsed().as_secs_f32() * 1000.0;
encoder
.capture_ms
.store(capture_elapsed.to_bits(), Ordering::Relaxed);
encoder.push_frame(CapturedFrame {
source,
width: w,
height: h,
vk_format: job.sc_fmt.as_raw() as u32,
vk_colorspace: ds.swapchain_colorspace.load(Ordering::Relaxed),
});
}
}
}
log::info!("capture worker exiting");
})
.ok();
}
unsafe fn try_make_dmabuf_source(
ds: &crate::state::DeviceState,
mem: vk::DeviceMemory,
stride: u32,
) -> Option<FrameSource> {
let cached = ds.cached_dmabuf_fd.load(Ordering::Relaxed);
let fd = if cached >= 0 {
let duped = unsafe { libc::dup(cached) };
if duped >= 0 {
duped
} else {
let fresh = unsafe { capture::get_dmabuf_fd(ds, mem)? };
ds.cached_dmabuf_fd.store(fresh, Ordering::Relaxed);
unsafe { libc::dup(fresh) }
}
} else {
let fresh = unsafe { capture::get_dmabuf_fd(ds, mem)? };
ds.cached_dmabuf_fd.store(fresh, Ordering::Relaxed);
unsafe { libc::dup(fresh) }
};
if fd < 0 {
return None;
}
Some(FrameSource::DmaBuf {
fd,
stride,
modifier: 0,
})
}

92
apps/nescapture/src/shader.rs Normal file
View File

@@ -0,0 +1,92 @@
// ─────────────────────────────────────────────────────────────────────────────
// shader.rs — Phase 1: shader module interception and SPIR-V fingerprinting
//
// When the game (or DXVK/VKD3D) calls vkCreateShaderModule, we:
// 1. Let the call through to the next layer / driver.
// 2. Hash the raw SPIR-V bytecode with SHA-256, truncated to 64 bits.
// 3. Store (VkShaderModule → hash) in DeviceState::shader_registry.
//
// vkDestroyShaderModule removes the entry to keep the map bounded.
//
// The hash is stable for a fixed (game version, DXVK version) pair.
// It is looked up in vkCreateGraphicsPipelines (pipeline.rs) to tag each
// pipeline with the hashes of its vertex and fragment shaders.
// ─────────────────────────────────────────────────────────────────────────────
use crate::dispatch_key;
use crate::state::DEVICE_STATE;
use ash::vk::{self, Handle};
use sha2::{Digest, Sha256};
use std::os::raw::c_void;
/// Compute a stable 64-bit fingerprint of a SPIR-V module.
///
/// SPIR-V is a sequence of u32 words. We hash the raw bytes in native endian —
/// endianness consistency is all that matters since the hash is only compared
/// on the same machine within the same session.
///
/// SHA-256 is collision-resistant. Truncating to 64 bits is safe because any
/// single game has hundreds of shaders at most, making collisions astronomically
/// unlikely.
fn hash_spirv(words: &[u32]) -> u64 {
let bytes: &[u8] = bytemuck::cast_slice(words);
let digest = Sha256::digest(bytes);
// Take the first 8 bytes as a little-endian u64.
u64::from_le_bytes(digest[..8].try_into().unwrap())
}
// ─────────────────────────────────────────────────────────────────────────────
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateShaderModule(
device: vk::Device,
p_create_info: *const vk::ShaderModuleCreateInfo,
p_allocator: *const vk::AllocationCallbacks,
p_shader_module: *mut vk::ShaderModule,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
// Call through first so the driver creates the real object.
let result = unsafe {
(ds.fp.create_shader_module)(device, p_create_info, p_allocator, p_shader_module)
};
if result != vk::Result::SUCCESS {
return result;
}
// Hash the SPIR-V bytecode.
let ci = unsafe { &*p_create_info };
// code_size is in bytes; p_code points to u32 words.
let word_count = ci.code_size / std::mem::size_of::<u32>();
let words = unsafe { std::slice::from_raw_parts(ci.p_code, word_count) };
let hash = hash_spirv(words);
let module = unsafe { *p_shader_module };
ds.shader_registry.insert(module.as_raw(), hash);
/*log::trace!(
"shader module {:#010x} → spir-v hash {:#018x} ({} words)",
module.as_raw(),
hash,
word_count,
);*/
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroyShaderModule(
device: vk::Device,
shader_module: vk::ShaderModule,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
ds.shader_registry.remove(&shader_module.as_raw());
unsafe { (ds.fp.destroy_shader_module)(device, shader_module, p_allocator) };
}
}

155
apps/nescapture/src/state.rs Normal file
View File

@@ -0,0 +1,155 @@
// ─────────────────────────────────────────────────────────────────────────────
// state.rs — global state shared across all hooks
// ─────────────────────────────────────────────────────────────────────────────
use crate::config::ShaderHashSet;
use crate::dispatch::NextDeviceFn;
use crate::encode::PipelineHandle;
use ash::vk;
use dashmap::DashMap;
use once_cell::sync::Lazy;
use std::sync::Arc;
// Holds capture-specific resources instead of re-creating them *each frame*
pub struct CaptureResources {
pub command_pool: vk::CommandPool,
pub command_buffers: [vk::CommandBuffer; 4],
pub fences: [vk::Fence; 4],
pub current: usize,
}
// ── Per-pipeline records ──────────────────────────────────────────────────────
#[derive(Clone, Debug, Default)]
pub struct PipelineHashes {
pub vert_hash: Option<u64>,
pub frag_hash: Option<u64>,
}
#[derive(Clone, Debug, Default)]
pub struct PipelineState {
pub blend_enabled: bool,
pub depth_test_enabled: bool,
pub depth_write_enabled: bool,
}
// ── Per-device state ──────────────────────────────────────────────────────────
pub struct DeviceState {
pub raw: vk::Device,
pub physical_device: vk::PhysicalDevice,
pub fp: NextDeviceFn,
// Phase 1: shader / pipeline
pub shader_registry: DashMap<u64, u64>,
pub pipeline_registry: DashMap<u64, PipelineHashes>,
pub pipeline_state: DashMap<u64, PipelineState>,
// Phase 2: view / framebuffer tracking
pub view_to_image: DashMap<u64, u64>,
pub view_format: DashMap<u64, vk::Format>,
pub framebuffer_to_views: DashMap<u64, Vec<u64>>,
pub framebuffer_extent: DashMap<u64, vk::Extent2D>,
// Phase 3: shader hash config
pub shader_hashes: Option<ShaderHashSet>,
// Phase 4: HUDless capture (optional — HUDLESS_CAPTURE_HUDLESS=1)
pub hudless_image: std::sync::Mutex<Option<vk::Image>>,
pub hudless_memory: std::sync::Mutex<Option<vk::DeviceMemory>>,
pub hudless_size: std::sync::Mutex<(u32, u32, vk::Format)>,
// Phase 4: final-frame capture (DMA-BUF exportable)
pub final_image: std::sync::Mutex<Option<vk::Image>>,
pub final_memory: std::sync::Mutex<Option<vk::DeviceMemory>>,
pub final_size: std::sync::Mutex<(u32, u32, vk::Format)>,
/// Row stride in bytes of `final_image`, queried once after allocation.
/// Zero until the first frame is captured.
pub final_stride: std::sync::atomic::AtomicU32,
// Phase 4: swapchain tracking
pub swapchain: std::sync::Mutex<Option<vk::SwapchainKHR>>,
pub swapchain_images: std::sync::Mutex<Vec<vk::Image>>,
pub swapchain_format: std::sync::Mutex<vk::Format>,
/// Raw `VkColorSpaceKHR` value, captured from vkCreateSwapchainKHR.
/// Used to derive color space for the encoder (SDR vs HDR10 etc.).
pub swapchain_colorspace: std::sync::atomic::AtomicU32,
pub swapchain_extent: std::sync::Mutex<vk::Extent2D>,
pub frame_counter: std::sync::atomic::AtomicU64,
pub largest_extent: std::sync::Mutex<vk::Extent2D>,
// Phase 3/4: per-frame HUD detection flags
pub hud_detected_frame: std::sync::atomic::AtomicBool,
pub pending_capture_frame: std::sync::atomic::AtomicBool,
pub capture_injected_frame: std::sync::atomic::AtomicBool,
pub skipped_draws_frame: std::sync::atomic::AtomicU32,
// Phase 7: encode + IPC pipeline (lazy-init on first frame)
pub encoder: std::sync::Mutex<Option<PipelineHandle>>,
// Re-usable capture resources (double-buffered)
pub capture_resources: std::sync::Mutex<Option<CaptureResources>>,
/// Dedicated queue for capture submissions (separate from game rendering).
pub capture_queue: std::sync::Mutex<vk::Queue>,
// Fake swapchain pool (headless — no real present)
pub fake_images: std::sync::Mutex<Vec<vk::Image>>,
pub fake_memories: std::sync::Mutex<Vec<vk::DeviceMemory>>,
pub fake_fds: std::sync::Mutex<Vec<std::os::raw::c_int>>,
pub fake_strides: std::sync::Mutex<Vec<u32>>,
pub fake_available: std::sync::Mutex<Vec<bool>>,
pub fake_image_count: std::sync::atomic::AtomicU32,
pub fake_swapchain: std::sync::Mutex<Option<vk::SwapchainKHR>>,
pub signal_queue: std::sync::Mutex<vk::Queue>,
pub next_acquire: std::sync::atomic::AtomicU32,
pub memory_properties: std::sync::Mutex<vk::PhysicalDeviceMemoryProperties>,
pub acquire_dummy_pool: std::sync::Mutex<vk::CommandPool>,
pub acquire_dummy_cb: std::sync::Mutex<vk::CommandBuffer>,
/// Cached DMA-BUF fd for final_memory (-1 = not cached).
/// Avoids a kernel ioctl per frame.
pub cached_dmabuf_fd: std::sync::atomic::AtomicI32,
// ── Frame-rate throttle ───────────────────────────────────────────
/// Target FPS for capture throttling (set from HUDLESS_FPS on pipeline init).
pub target_fps: std::sync::atomic::AtomicU32,
/// Timestamp of last captured frame (for rate limiting).
pub last_capture_time: std::sync::Mutex<Option<std::time::Instant>>,
/// Channel for threaded capture worker (present → worker).
pub capture_tx: std::sync::Mutex<Option<std::sync::mpsc::Sender<crate::present::CaptureJob>>>,
}
// ── Per-command-buffer state ──────────────────────────────────────────────────
#[derive(Default, Clone)]
pub struct CbState {
pub device_key: usize,
pub current_color_image: Option<vk::Image>,
pub current_image_format: Option<vk::Format>,
pub current_image_extent: Option<vk::Extent2D>,
pub active_vert_hash: Option<u64>,
pub active_frag_hash: Option<u64>,
pub hud_captured: bool,
pub pending_capture: bool,
pub capture_injected: bool,
pub hud_detected: bool,
pub draw_counter: u32,
}
// ── Global state ──────────────────────────────────────────────────────────────
pub static INSTANCE_STATE: Lazy<DashMap<usize, Arc<crate::dispatch::NextInstanceFn>>> =
Lazy::new(DashMap::new);
pub static DEVICE_STATE: Lazy<DashMap<usize, Arc<DeviceState>>> = Lazy::new(DashMap::new);
pub static CB_STATE: Lazy<DashMap<u64, CbState>> = Lazy::new(DashMap::new);
/// VkQueue → device dispatch key
pub static QUEUE_TO_DEVICE_KEY: Lazy<DashMap<u64, usize>> = Lazy::new(DashMap::new);
/// VkCommandBuffer → device dispatch key
pub static CMD_BUF_TO_DEVICE_KEY: Lazy<DashMap<u64, usize>> = Lazy::new(DashMap::new);

121
apps/nescapture/src/swapchain.rs Normal file
View File

@@ -0,0 +1,121 @@
// ─────────────────────────────────────────────────────────────────────────────
// swapchain.rs — Phase 4: swapchain tracking
//
// Captures format, extent AND color space from vkCreateSwapchainKHR so the
// encoder can automatically determine the correct color space pipeline
// (SDR/BT.709 vs HDR10/BT.2020-PQ vs FP16) without any manual configuration.
// ─────────────────────────────────────────────────────────────────────────────
use crate::dispatch_key;
use crate::state::DEVICE_STATE;
use ash::vk::{self, Handle};
use std::os::raw::c_void;
use std::sync::atomic::Ordering;
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkCreateSwapchainKHR(
device: vk::Device,
p_create_info: *const vk::SwapchainCreateInfoKHR,
p_allocator: *const vk::AllocationCallbacks,
p_swapchain: *mut vk::SwapchainKHR,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
let create_fn = match ds.fp.create_swapchain_khr {
Some(f) => f,
None => return vk::Result::ERROR_EXTENSION_NOT_PRESENT,
};
let ci = unsafe { &*p_create_info };
// Add TRANSFER_SRC so we can blit from swapchain images.
let mut modified_ci = *ci;
modified_ci.image_usage = ci.image_usage | vk::ImageUsageFlags::TRANSFER_SRC;
let result = unsafe { create_fn(device, &modified_ci, p_allocator, p_swapchain) };
// If the driver rejects TRANSFER_SRC (e.g. composited window), try without.
let result = if result != vk::Result::SUCCESS {
unsafe { create_fn(device, ci, p_allocator, p_swapchain) }
} else {
result
};
if result != vk::Result::SUCCESS {
return result;
}
*ds.swapchain.lock().unwrap() = Some(unsafe { *p_swapchain });
*ds.swapchain_format.lock().unwrap() = ci.image_format;
*ds.swapchain_extent.lock().unwrap() = ci.image_extent;
ds.swapchain_colorspace
.store(ci.image_color_space.as_raw() as u32, Ordering::Relaxed);
log::debug!(
"swapchain created — format={:?} colorspace={:?} extent={}x{}",
ci.image_format,
ci.image_color_space,
ci.image_extent.width,
ci.image_extent.height,
);
vk::Result::SUCCESS
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkDestroySwapchainKHR(
device: vk::Device,
swapchain: vk::SwapchainKHR,
p_allocator: *const vk::AllocationCallbacks,
) {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
if let Some(ds) = DEVICE_STATE.get(&key) {
*ds.swapchain.lock().unwrap() = None;
*ds.swapchain_images.lock().unwrap() = Vec::new();
if let Some(destroy_fn) = ds.fp.destroy_swapchain_khr {
unsafe { destroy_fn(device, swapchain, p_allocator) };
}
}
}
#[unsafe(no_mangle)]
pub unsafe extern "system" fn vkGetSwapchainImagesKHR(
device: vk::Device,
swapchain: vk::SwapchainKHR,
p_swapchain_image_count: *mut u32,
p_swapchain_images: *mut vk::Image,
) -> vk::Result {
let key = unsafe { dispatch_key(device.as_raw() as *const c_void) };
let ds = match DEVICE_STATE.get(&key) {
Some(s) => s.clone(),
None => return vk::Result::ERROR_DEVICE_LOST,
};
let get_fn = match ds.fp.get_swapchain_images_khr {
Some(f) => f,
None => return vk::Result::ERROR_EXTENSION_NOT_PRESENT,
};
let result = unsafe {
get_fn(
device,
swapchain,
p_swapchain_image_count,
p_swapchain_images,
)
};
if result != vk::Result::SUCCESS {
return result;
}
if !p_swapchain_images.is_null() {
let count = unsafe { *p_swapchain_image_count as usize };
let images = unsafe { std::slice::from_raw_parts(p_swapchain_images, count) };
*ds.swapchain_images.lock().unwrap() = images.to_vec();
log::debug!("swapchain images — {} images", count);
}
vk::Result::SUCCESS
}