diff --git a/Cargo.lock b/Cargo.lock index 67f8d545..e9207348 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,6 +628,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1022,11 +1028,17 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "2a7a45518d2863d18aa47f4a0cf9faec2aa4304cc09df5e41299f276b3ad135e" dependencies = [ "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -1243,28 +1255,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gbm" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" -dependencies = [ - "bitflags 2.13.1", - "drm", - "drm-fourcc", - "gbm-sys", - "libc", -] - -[[package]] -name = "gbm-sys" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f" -dependencies = [ - "libc", -] - [[package]] name = "generator" version = "0.8.9" @@ -1349,17 +1339,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gl_generator" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" -dependencies = [ - "khronos_api", - "log", - "xml-rs", -] - [[package]] name = "glob" version = "0.3.4" @@ -2156,12 +2135,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "khronos_api" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" - [[package]] name = "lazy_static" version = "1.5.0" @@ -2371,6 +2344,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "multiversion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "target-features", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "n0-error" version = "1.0.1" @@ -3102,24 +3103,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "pixman" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea217d496c19ac0a8e502b37078e1f683d16344adee9eb247a5d57c165e1edf" -dependencies = [ - "drm-fourcc", - "paste", - "pixman-sys", - "thiserror 1.0.69", -] - -[[package]] -name = "pixman-sys" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0483e89e81d7915defe83c51f23f6800594d64f6f4a21253ce87fd8444ada" - [[package]] name = "pkcs8" version = "0.11.0" @@ -3905,7 +3888,6 @@ dependencies = [ "atomic_float", "bitflags 2.13.1", "calloop", - "cc", "cgmath", "cursor-icon", "downcast-rs", @@ -3914,14 +3896,9 @@ dependencies = [ "drm-fourcc", "encoding_rs", "errno", - "gbm", - "gl_generator", "indexmap", "input", "libc", - "libloading", - "pixman", - "pkg-config", "profiling", "rand 0.9.5", "rustix 1.1.4", @@ -4103,6 +4080,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "target-lexicon" version = "0.13.5" diff --git a/apps/nescapture/scripts/verify-chain.sh b/apps/nescapture/scripts/verify-chain.sh index 9b184484..b68245fb 100755 --- a/apps/nescapture/scripts/verify-chain.sh +++ b/apps/nescapture/scripts/verify-chain.sh @@ -36,6 +36,21 @@ done echo "building…" cargo build --release -p nescope -p nescapture --manifest-path "$ROOT/Cargo.toml" >/dev/null +# The whole method here is two independent instruments on the same frames, and +# the second one is the compositor's own readback. While nescope's screenshot +# path is commented out there is no second instrument, so this script cannot +# make the comparison it exists for. Said here rather than fifty lines later as +# "compositor readback produced no frames", which reads like a capture bug. +# +# Asked of the binary rather than hard-coded, so this comes back by itself on +# the commit that brings the path back. +if ! "$ROOT/target/release/nescope" --help 2>&1 | grep -q -- --screenshot-ipc; then + echo "this nescope has no --screenshot-ipc, so there is no readback to compare" >&2 + echo "the encoded frames against; the GPU readback path in nescope is" >&2 + echo "commented out. See apps/nescope/src/main.rs." >&2 + exit 1 +fi + LAYER="$ROOT/target/release/libnescapture_layer.so" MANIFEST_DIR="$WORK/implicit_layer.d" mkdir -p "$MANIFEST_DIR" diff --git a/apps/nescope/Cargo.toml b/apps/nescope/Cargo.toml index dfa79a7b..9afca051 100644 --- a/apps/nescope/Cargo.toml +++ b/apps/nescope/Cargo.toml @@ -18,13 +18,11 @@ smithay = { version = "0.7", default-features = false, features = [ "backend_drm", "desktop", "backend_libinput", - "renderer_pixman", # needed for on_commit_buffer_handler + #"renderer_pixman", # needed for on_commit_buffer_handler # Reading a dmabuf back to the CPU. nescope still does not composite or # present anything -- this is import-and-copy only, so a client that # renders on the GPU can be screenshotted like any other. - "renderer_gl", - "backend_egl", - "backend_gbm", + #"backend_gbm", ] } # Wayland client – connects to the host compositor to forward buffers. diff --git a/apps/nescope/src/main.rs b/apps/nescope/src/main.rs index 7bd8316f..444136a3 100644 --- a/apps/nescope/src/main.rs +++ b/apps/nescope/src/main.rs @@ -60,15 +60,15 @@ use smithay::reexports::wayland_server::Display; use smithay::wayland::socket::ListeningSocketSource; mod focus; -mod gpu_readback; +//mod gpu_readback; mod handlers; mod hdr; mod input; mod input_ipc; mod libinput_backend; mod protocols; -mod screenshot_ipc; -mod screenshot_wire; +//mod screenshot_ipc; +//mod screenshot_wire; mod state; mod xwm; @@ -125,14 +125,11 @@ struct Args { )] input_ipc: String, - /// Path to the hub's screenshot IPC socket (nescope connects as client). - /// - /// Optional, and absent means the feature is simply off: it exists for - /// clients that are not games — a Steam login screen has no Vulkan frames - /// for `nescapture` to take, so its pixels can only come from here. - #[arg(long, env = "NESCOPE_SCREENSHOT_IPC")] - screenshot_ipc: Option, - + // There is no `--screenshot-ipc`. The path it named is commented out below, + // and an option that is accepted and does nothing is worse than one that is + // refused: a caller passing it gets no error, no capture, and nothing to + // read that says which. It comes back with the code, or not at all. + // /// GPU render device (e.g. /dev/dri/renderD128). Sets VK_DRIVER_FILES /// for the game so it uses the same GPU. #[arg(long, env = "NESCOPE_RENDER_DEVICE")] @@ -317,13 +314,13 @@ fn main() { // The GPU to import dmabufs on for screenshots. Same device the game is // pointed at, because a buffer the game produced can only be imported on // the device that made it. - gpu_readback::set_render_device(args.render_device.clone()); + //gpu_readback::set_render_device(args.render_device.clone()); // ── Screenshot IPC source ──────────────────────────────────────────── // Same dial-out shape as the input socket below, so the hub is the // listener and there is no race against a socket that does not exist yet. // Absent means the feature is off, which is the normal case for a game. - if let Some(path) = args.screenshot_ipc.clone() { + /*if let Some(path) = args.screenshot_ipc.clone() { match screenshot_ipc::ScreenshotIpcSource::connect(&path) { Ok(source) => match source.try_clone_writer() { Ok(mut writer) => { @@ -357,7 +354,7 @@ fn main() { }, Err(e) => tracing::warn!("Failed to connect to screenshot IPC socket {path}: {e}"), } - } + }*/ // ── Input IPC source ───────────────────────────────────────────────── // Connect to the neshub input socket and feed events into the diff --git a/apps/nesinit/README.md b/apps/nesinit/README.md index 0aec304e..ecae58fe 100644 --- a/apps/nesinit/README.md +++ b/apps/nesinit/README.md @@ -2,8 +2,9 @@ PID 1 inside a box. -A microVM has no init unless something is it. Three of the jobs are nobody -else's, and this is all of them: +A microVM has no init unless something is it, and in a box nothing else is: +there is no service manager in the image and no init scripts. Four jobs, and +this is all of them: - **Reaping.** A process whose parent dies is reparented to PID 1. Without a reaper, every orphan the workload leaves behind holds a pid and a slot in the @@ -11,25 +12,46 @@ else's, and this is all of them: - **Ordered shutdown.** The workload stops first and alone, then everything else, then the disks are flushed and the machine is powered off. An init that returns leaves a guest running with nothing in it. +- **The box's own services.** The bus, audio, and the transport that carries a + session out, started in order from a table compiled into this binary. There + is no unit format and no directory of files to read: the services in a box + are fixed, and running on any distribution comes from depending on none of + their init scripts rather than from being configurable. - **The guest end of the control channel.** One vsock connection out, carrying what to run in and what happened back. -It does not know what it is running. It is handed a command line, a set of -shares and what an exit means; there is no code path here that branches on -which workload it started, and there is not meant to be. +It does not know what it is running. It is handed a set of shares, and then +commands naming what to run and what an exit means; there is no code path here +that branches on which workload it started, and there is not meant to be. + +**A box outlives what runs in it.** Init mounts, brings the services up, says +it is ready, and then takes commands for as long as the box lives — so this +image on its own runs nothing at all, and a box may be launched into more than +once. ### The channel The guest dials out on a fixed vsock port and speaks first: ``` -guest → { "type": "ready", "protocol_version": 2 } -guest ← { "type": "boot", "exec": {...}, "mounts": [...], "geometry": {...}, "on_exit": {...} } +guest → { "type": "ready", "protocol_version": 3 } +guest ← { "type": "boot", "mounts": [...] } guest → { "type": "mounted" } -guest → { "type": "started" } -guest → { "type": "workload_exited", "exit_code": 0 } +guest → { "type": "initialized", "services": ["dbus-system", ...] } +guest ← { "type": "launch", "id": "…", "exec": {...}, "on_exit": {...} } +guest → { "type": "started", "id": "…" } +guest → { "type": "workload_exited", "id": "…", "exit_code": 0 } ``` +`ready` is the handshake and `initialized` is the box working. They are two +facts and must not be treated as one: a caller that waits on the first has a +wait that succeeds before anything in the guest has started. + +Every launch carries an id and every message about a launch carries it back. +Without one, a second launch's exit is indistinguishable from the first's — +which reads at the far end as a finished session still running, or a running +one reported as stopped. + Newline-delimited JSON. Dialling out rather than being connected to is worth keeping for two reasons: the listener is up before the VM starts, so nothing races a booting kernel and nothing has to retry, and the connection @@ -109,19 +131,40 @@ later for no visible reason. ### It reports; it does not supervise -When the workload ends, the exit goes up the channel and the session is over. -`on_exit` says what that exit *means* — whether it ends the session — and -nothing here restarts anything. Starting something again is a decision for the -end that can see whether restarting is repair or a loop. +When a launch ends, the exit goes up the channel. `on_exit` says what that exit +*means* — whether it ends the session or leaves the box up to be launched into +again — and nothing here restarts anything of its own accord. Starting +something again is a decision for the end that can see whether restarting is +repair or a loop. + +`restart` exists as one message and is defined as exactly that: a kill followed +by a launch of the same command, keeping the id, with no retry and no backoff. +It is one message rather than two only because a caller sending two has the +same effect with a worse race in it. + +The same rule covers the box's own services. One that dies is **reported and +left dead** — nothing else in the guest is watching them, so a death that is +not said here is a box that looks healthy and cannot work. + +**One launch at a time.** A launch arriving while one is running is refused, +carrying the id it was asked for, rather than queued or silently replacing it: +a box has one screen, so a second concurrent launch has nowhere to draw. A signalled workload is reported as signalled, with no exit code. Reporting `0` for a killed process would make a kill look like a clean run. ### What is not here yet -`geometry` is carried and parsed but nothing consumes it: nesinit does not -start the guest's own services yet. `ticket` exists as a message with no -producer wired to it. +**It has never been PID 1 of anything.** Every line of this is written for a +box and all of it has been tested outside one. It runs perfectly well as an +ordinary process — it warns rather than fails when it is not PID 1 — which is +how most of it is exercised, and is also how a guest that will not boot gets +debugged: `init=/bin/bash` on the kernel command line, then run this by hand +and watch it fail. + +Output geometry is deliberately absent from this layer. The compositor is +started by a launch, with that launch's geometry in its own arguments, so the +numbers appear in one place rather than two that can disagree. ### Testing diff --git a/apps/nesinit/src/filesystems.rs b/apps/nesinit/src/filesystems.rs index 04d03d5e..05cb5fd5 100644 --- a/apps/nesinit/src/filesystems.rs +++ b/apps/nesinit/src/filesystems.rs @@ -31,6 +31,12 @@ struct Early { /// so a device node or a setuid bit appearing in one did not come from us. const NOSUID_NODEV: libc::c_ulong = libc::MS_NOSUID | libc::MS_NODEV; +// Every tmpfs below is capped, and the caps are load-bearing rather than +// tidiness. A tmpfs with no `size=` may grow to half of RAM, and the RAM in +// question is the box's whole allowance — so an uncapped `/tmp` is a workload +// that can OOM the box it runs in by writing files. The numbers are carried +// over from the mount table this replaced, where they were already considered. + const EARLY: &[Early] = &[ Early { source: "proc", @@ -41,6 +47,38 @@ const EARLY: &[Early] = &[ cost: "this process cannot make itself ineligible for the OOM killer, \ and nothing in the guest can read its own state", }, + // Usually already there: a kernel built with `CONFIG_DEVTMPFS_MOUNT` mounts + // this before init runs. Listed anyway because the check below skips what + // is already mounted, so the entry costs nothing when the kernel did it and + // is the difference between a working box and one with no device nodes when + // it did not. Without `nodev`, obviously — device nodes are the point. + Early { + source: "devtmpfs", + target: "/dev", + fstype: "devtmpfs", + flags: libc::MS_NOSUID, + data: "mode=755", + cost: "there are no device nodes at all, so nothing can open the GPU", + }, + Early { + source: "devpts", + target: "/dev/pts", + fstype: "devpts", + flags: NOSUID_NODEV | libc::MS_NOEXEC, + data: "mode=620,gid=5,ptmxmode=666", + cost: "nothing that wants a terminal can allocate one", + }, + // The image creates this directory, and the mode is the load-bearing part: + // a workload and the box's own services are different users, and shared + // memory between them is how a Vulkan client hands buffers around. + Early { + source: "tmpfs", + target: "/dev/shm", + fstype: "tmpfs", + flags: NOSUID_NODEV, + data: "mode=1777,size=256m", + cost: "anything using shared memory fails, which includes most graphics", + }, Early { source: "sysfs", target: "/sys", @@ -59,7 +97,7 @@ const EARLY: &[Early] = &[ flags: NOSUID_NODEV, // The sticky bit, because the workload does not run as this process // does and what it binds here is its own. - data: "mode=1777", + data: "mode=1777,size=64m", cost: "whatever serves this session's address cannot bind its socket, \ so the session never gets one", }, @@ -74,20 +112,51 @@ const EARLY: &[Early] = &[ // Octal, and without a leading zero on purpose: the kernel parses a // tmpfs mode as octal either way, and this is the spelling `mount` // itself documents. - data: "mode=755", + data: "mode=755,size=32m", cost: "there is nowhere for a runtime socket to live, so neither the \ payload relay nor this session's address can be served", }, + // The tree a session's shares are mounted into. + // + // A share's target is named by the descriptor and may be any path under + // here, so something has to create directories on a root that is read-only + // by design. That is what this is: `workload::mount` calls `create_dir_all` + // on each target, which fails with `EROFS` unless the tree it is creating + // in is writable. + // + // **This entry used to be forbidden, and the reason it was forbidden is + // gone.** A test here asserted that `/nestri` must never be mounted over, + // because a fresh tmpfs would hide the install, the user state and the work + // directory that the image had prepared underneath. That was true of the + // image that shipped those directories and an `fstab` that mounted into + // them. The image prepares nothing here now — the host names every share + // and every target — so there is nothing left to hide, and the rule had + // become a guard on a hazard that was deleted with the image that had it. + // ref(d-0064) + // + // Small on purpose. Everything real is mounted *over* this, so what remains + // is a handful of empty directories; the cap matters for the case where a + // share fails to mount and a workload writes to the bare mount point + // instead, which would otherwise be RAM the box cannot get back. + Early { + source: "tmpfs", + target: "/nestri", + fstype: "tmpfs", + flags: NOSUID_NODEV, + data: "mode=755,size=4m", + cost: "no share can be mounted, because its target cannot be created on a read-only root", + }, // The relay's own directory, and it is deliberately **not** in the tree the // session's shares live in. // - // It was, and that was wrong in a way no test here would have caught: a - // fresh tmpfs over the share tree hides every directory the image prepared - // underneath it — the install, the user state, the work directory, and the - // mount point the log share is attached to from `fstab`. The box then has a - // socket and none of the places its workload expects to find its files, and - // the exact-path check below cannot notice, because what `fstab` mounts is - // a directory *inside* that tree rather than the tree itself. + // It was, and moving it out stays right for a reason that outlived the one + // originally given. The first reason was that a tmpfs over the share tree + // would hide what the image had prepared there; that image is gone and the + // entry above now mounts that tree deliberately. The reason that remains is + // ownership: this directory is written by this process and by nothing else, + // which is what makes the socket in it unreplaceable. The share tree is + // mounted into by the host's own shares, so a relay socket living there + // would sit in a tree a workload's own share can be attached over. // // Owned by this process and writable by nothing else, which is what makes // the socket in it unreplaceable. The workload reaches it because the @@ -98,10 +167,22 @@ const EARLY: &[Early] = &[ target: crate::payload::DIRECTORY, fstype: "tmpfs", flags: NOSUID_NODEV | libc::MS_NOEXEC, - data: "mode=755", + // One socket lives here, so this is as small as a tmpfs usefully gets. + data: "mode=755,size=1m", cost: "the payload relay cannot bind, so nothing reaches the workload \ over the channel", }, + // The root is read-only and some things write here whether or not anything + // reads it back. A box's real logs leave over the control channel; this is + // so that a library writing a file does not fail on `EROFS` instead. + Early { + source: "tmpfs", + target: "/var/log", + fstype: "tmpfs", + flags: NOSUID_NODEV | libc::MS_NOEXEC, + data: "mode=755,size=16m", + cost: "anything that writes a log file fails on a read-only root", + }, ]; /// Mount what the rest of this component assumes is already there. @@ -220,6 +301,25 @@ mod tests { } } + /// **Every tmpfs is capped.** One without a `size=` may grow to half of RAM, + /// and the RAM in question is the whole box's — so an uncapped `/tmp` hands + /// a workload a way to OOM the box it is running in by writing files. The + /// failure looks like a box that died under load rather than like a missing + /// mount option, which is why this is a test. + #[test] + fn no_tmpfs_is_unbounded() { + for early in EARLY { + if early.fstype != "tmpfs" { + continue; + } + assert!( + early.data.contains("size="), + "{} is an uncapped tmpfs", + early.target + ); + } + } + /// `/proc` is mounted before `mountinfo` is read, so it has to be first. #[test] fn proc_is_the_first_entry() { @@ -243,20 +343,32 @@ mod tests { } /// **Nothing here may be mounted over the tree the session's shares live - /// in.** A fresh tmpfs there hides every directory the image prepared - /// underneath — the install, the user state, the work directory, and the - /// mount point the log share attaches to — and the exact-path check cannot - /// notice, because what is mounted from `fstab` is a directory inside that - /// tree rather than the tree itself. So a box would come up with a socket - /// and without any of the places its workload looks for its files. + /// in**, and the share tree itself is mounted so that targets under it can + /// be created at all. + /// + /// This replaces a test that asserted the exact opposite — that `/nestri` + /// must never be mounted over — on the grounds that a tmpfs there would + /// hide the install, the user state and the work directory the image had + /// prepared. The image that prepared them no longer exists; the host names + /// every share and every target now, and a read-only root cannot have a + /// directory created on it. Measured 2026-09-11: without this entry the + /// first real boot refused its own descriptor with + /// `/nestri/payload: Read-only file system`. #[test] - fn the_share_tree_is_never_mounted_over() { - for early in EARLY { - assert_ne!( - early.target, "/nestri", - "this hides the directories the image prepared for a session" - ); - } + fn the_share_tree_is_writable_and_the_relay_is_not_inside_it() { + let tree = EARLY + .iter() + .find(|e| e.target == "/nestri") + .expect("a share's target cannot be created without this"); + assert_eq!(tree.fstype, "tmpfs"); + assert!( + tree.data.contains("size="), + "an uncapped tmpfs here is RAM a box cannot get back" + ); + assert!( + !crate::payload::DIRECTORY.starts_with("/nestri/"), + "the relay's socket would sit in a tree a share can be mounted over" + ); } /// The relay's directory is the one this cannot hardcode: it belongs to diff --git a/apps/nesinit/src/lib.rs b/apps/nesinit/src/lib.rs index 32df7edd..85730fe0 100644 --- a/apps/nesinit/src/lib.rs +++ b/apps/nesinit/src/lib.rs @@ -4,14 +4,20 @@ // nobody else's: reaping whatever the workload orphans, turning a signal into // an ordered shutdown, and being the guest end of the one channel out. // -// It does not know what it is running. It is handed a command, a set of shares -// and what an exit means, and it carries that out; a field that only makes -// sense for one kind of workload cannot reach it. ref(d-0033) +// It does not know what it is running. It is handed a set of shares, and then +// commands naming what to run and what an exit means, and it carries those out; +// a field that only makes sense for one kind of workload cannot reach it. +// ref(d-0033) +// +// It is also the box's only init: there is no service manager in the image, so +// the box's own services come up from a table in this binary. ref(d-0064) pub mod filesystems; pub mod payload; pub mod reap; +pub mod services; pub mod session; pub mod shutdown; +pub mod system; pub mod ticket; pub mod workload; diff --git a/apps/nesinit/src/main.rs b/apps/nesinit/src/main.rs index cb44cbed..78c2728c 100644 --- a/apps/nesinit/src/main.rs +++ b/apps/nesinit/src/main.rs @@ -9,6 +9,7 @@ use std::time::Duration; use nesinit::payload::{self, Ports}; use nesinit::reap::{self, Waiters}; +use nesinit::services::Stack; use nesinit::session::{self, Outcome}; use nesinit::shutdown::{self, Machine}; use nesinit::ticket; @@ -45,6 +46,13 @@ fn main() -> anyhow::Result<()> { // is no `/proc` to score this process in and nowhere to put a socket. nesinit::filesystems::establish(); + // Everything a distribution's init scripts used to do, and nothing else is + // going to: a hostname, the box's address, the directories a session's + // sockets live in, and device nodes something is allowed to open. Before + // the runtime, so the few processes it starts are waited for directly + // rather than racing the reaper into existence. ref(d-0064) + nesinit::system::prepare(); + // Both before anything is started, so nothing can be orphaned or scored // in the window where neither is true yet. if let Err(error) = reap::become_subreaper() { @@ -80,7 +88,18 @@ fn main() -> anyhow::Result<()> { // Reached however the session ended, including an error: an init that // returns leaves the guest running with nothing in it. - let mut machine = Guest { workload }; + // + // `is_init` is what makes that safe to say. The three machine-wide steps + // below — signal everything, kill everything, power off — are correct for + // PID 1 of a box and catastrophic anywhere else, and this program is meant + // to be runnable by hand: that is how most of it is tested and it is the + // documented way to debug a guest that will not boot. Run as root outside a + // box, the old path reached `kill(-1)` and `reboot` the moment the control + // channel could not be dialled. + let mut machine = Guest { + workload, + is_init: pid == 1, + }; shutdown::ordered(&mut machine, GRACE); unreachable!("power_off does not return"); } @@ -124,8 +143,13 @@ async fn guest(waiters: &Waiters, workload: &mut Process) -> anyhow::Result outcome?, + outcome = session::run(channel, workload, &mut services, &mut ports, &mut found_rx, &untrusted) => outcome?, signal = asked_to_stop() => { signal?; tracing::info!("asked to stop"); @@ -170,6 +194,9 @@ async fn asked_to_stop() -> std::io::Result<()> { /// The machine, for real. struct Guest { workload: Process, + /// Whether this process is PID 1, and therefore whether the steps that act + /// on *the machine* rather than on our own children may be taken at all. + is_init: bool, } impl Machine for Guest { @@ -194,6 +221,13 @@ impl Machine for Guest { } fn signal_rest(&mut self, grace: Duration) { + if !self.is_init { + tracing::warn!( + "not PID 1, so not signalling every process: outside a box that \ + is this machine's processes, not this box's" + ); + return; + } // -1 is every process this one may signal, which as PID 1 is all of // them but itself. The workload has already stopped by here. unsafe { libc::kill(-1, libc::SIGTERM) }; @@ -201,15 +235,31 @@ impl Machine for Guest { } fn kill_rest(&mut self) { + if !self.is_init { + return; + } unsafe { libc::kill(-1, libc::SIGKILL) }; wait_for_quiet(Duration::from_secs(1)); } fn flush_disks(&mut self) { + // Harmless anywhere, so it is not guarded: the worst it does outside a + // box is flush somebody's page cache. unsafe { libc::sync() }; } fn power_off(&mut self) { + if !self.is_init { + // Everything this process started has been stopped by here, which + // is the whole of what it may take responsibility for when it is + // not the machine's init. What it prepared — the mounts, the + // runtime directories — is deliberately left behind, because that + // is exactly what makes a hand-run useful: run it, watch it fail to + // reach a control channel that is not there, and then poke at a + // guest that is otherwise set up. + tracing::warn!("not PID 1, so not powering the machine off"); + std::process::exit(1); + } // SAFETY: reboot is the only way out of a guest whose init is done. unsafe { libc::reboot(libc::RB_POWER_OFF) }; // Reached only if the guest refused to power off, which no caller can diff --git a/apps/nesinit/src/services.rs b/apps/nesinit/src/services.rs new file mode 100644 index 00000000..856679a6 --- /dev/null +++ b/apps/nesinit/src/services.rs @@ -0,0 +1,959 @@ +// The box's own services, and the table that is the whole of what a box runs +// before anything is launched into it. +// +// There is no service manager in a box and no init scripts, so this is what +// replaces them. ref(d-0064) +// +// # Why the table is in the binary +// +// A unit format would make this configurable, and nothing wants to configure +// it: the services in a box are ours, they are the same in every box, and a +// table in a file is a table two images can disagree about. Being able to run +// on any distribution comes from depending on no distribution's init scripts, +// which this does — not from being told what to start. +// +// # What is deliberately not here +// +// **The compositor.** It wraps the workload and is started by a launch, with +// that launch's geometry, and dies with it. A compositor in this table would be +// a compositor with no geometry to come up with. +// +// **Restarting.** A service that dies is reported up the channel and left dead. +// Whether restarting it is repair or a loop is not visible from inside the box. +// +// **Readiness beyond "its socket exists".** A service that names a socket is +// waited for until that socket is there; nothing here asks it a question or +// waits for a bus name. That much was in the init scripts this replaced and +// leaving it out was a regression: `spawn` returns at fork, so without it the +// bus's client is started before the bus is listening and audio comes up +// against nothing. It races rather than failing — which is the shape this +// component is least able to see — and the cost of losing the race is a box +// that boots, reports itself ready, and has no sound. +// +// That is as far as it goes. Health checks, restarts and readiness that is not +// a file on a path would make this a supervisor; see the decision's own +// falsification list. + +use std::os::unix::process::CommandExt; + +use nesprotocol::lifecycle::Exit; +use tokio::sync::mpsc::{Receiver, Sender}; + +use crate::reap::{Waiters, Watched}; +use crate::workload::Failure; + +/// A service that died, and how. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Died { + pub name: String, + pub exit: Exit, +} + +/// The box's service stack, as the session sees it. +/// +/// A trait because the interesting behaviour is the session's — that a death is +/// reported and not repaired, that a stack which will not come up refuses the +/// box — and none of that needs a process to assert. +pub trait Services { + /// Bring the stack up in order, and name what came up. + /// + /// Called once, after the shares are mounted and before anything may be + /// launched. An empty stack is legitimate: a box with no services still + /// boots, and a caller can still launch something that needs none. + fn bring_up(&mut self) -> Result, Failure>; + + /// Deaths, as they happen. + /// + /// A channel rather than a future so the session can wait on it beside the + /// control channel, the relay and the address carrier without any of them + /// being able to starve the others. + fn deaths(&mut self) -> &mut Receiver; +} + +// There is deliberately no ordered `stop_all`, and reverse-order stopping buys +// nothing on a machine that is about to be powered off. What there is instead +// is a `Drop` that signals the children this stack started — because the +// argument for having nothing at all was "this process is PID 1 and the ordered +// shutdown signals every process", and that is true of a box and false of the +// way this program is run by hand to debug one. Outside PID 1 the old path left +// a bus, an audio server and a hub running with sockets nobody was serving. + +/// One service, and everything about starting it. +/// +/// `env` is per-entry rather than inherited: init's own environment is the +/// kernel's command line and says nothing a service should read. +pub struct Service { + /// What appears in a log line and in `initialized`. + pub name: &'static str, + pub argv: &'static [&'static str], + pub env: &'static [(&'static str, &'static str)], + /// Who it runs as. `None` means init's own user, which is root. + pub user: Option<(u32, u32)>, + /// Said when it will not start, in terms of what stops working. The same + /// discipline the early filesystems use: a failure that names a cost can be + /// acted on, where "could not start pipewire" cannot. + pub cost: &'static str, + /// Whether the box is unusable without it. + /// + /// A required service that will not start refuses the box, because a caller + /// launching into it would get a session that comes up and does not work. + /// An optional one is reported and stepped over. + pub required: bool, + /// The umask to exec under, when the default one is wrong. + /// + /// Only audio sets this, and only because of who has to reach it. A unix + /// socket is created `0777` masked by the umask, so the inherited `022` + /// gives `0755` -- and connecting to a socket needs *write*, so every user + /// but the owner is refused. The services run as one user and a workload + /// runs as another, so that is the workload: it finds the socket, cannot + /// open it, and plays silently. + /// + /// `0` rather than a mode in PipeWire's own configuration because the + /// socket list lives inside a module's arguments, and a drop-in that + /// re-declares that module loads it twice. + pub umask: Option, + /// A path that exists once this service can be talked to. + /// + /// `None` means "started is ready", which is true of anything nothing else + /// in the table connects to. Where something does connect, the path is the + /// socket it connects to: `spawn` returns when the child has been forked, + /// which is before that child has bound anything, so the next service in + /// the table would otherwise be started against a socket that is not there. + /// + /// Existence only. Whether the thing behind the socket answers correctly is + /// not knowable from here, and a box is not the place to find out. + pub ready: Option<&'static str>, +} + +/// How long a service gets to bind its socket before the box gives up on it. +/// +/// Long enough that a cold boot on a slow disk is not cut short, short enough +/// that a service which will never bind does not hold the box for a minute +/// before saying so. What actually happens is that the wait ends in single- +/// digit milliseconds, because the child binds before its parent gets back to +/// this loop. +const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); + +/// The user the box's own services run as. +/// +/// **Not the user a workload runs as, and that is the whole reason for the +/// number.** A workload sharing a user with these can replace a socket one of +/// them listens on and answer in its place — and the answer that matters is the +/// address a client is told to connect to. See `ticket::Untrusted`. +pub const SERVICE_UID: u32 = 1000; + +/// Where audio's socket lives, for both the services and the workload. +/// +/// # Why not the runtime directory +/// +/// The services run as one user and a workload runs as another, on purpose +/// (see [`SERVICE_UID`]). A per-user runtime directory is `0700` and named +/// after its own uid, so a socket in the services' one is in a directory the +/// workload may not enter, at a path it would not look in anyway. +/// +/// Measured 2026-09-12: the game rendered and had no sound, because it looked +/// for audio under its own uid and found nothing. Nothing failed -- a game with +/// no audio server plays silently. +/// +/// So audio gets a directory of its own that both users share, named to both +/// through `PIPEWIRE_RUNTIME_DIR`. The workload still cannot replace a socket +/// here: the directory belongs to the service user and is not writable by the +/// workload, which is the property [`crate::ticket::Untrusted`] depends on. +pub const AUDIO_DIR: &str = "/run/pipewire"; +pub const SERVICE_GID: u32 = 1000; + +/// Where a service's runtime sockets live. +pub const RUNTIME_DIR: &str = "/run/user/1000"; + +/// Somewhere every service may write. +/// +/// # The service user's home is on a read-only root +/// +/// `useradd -m` made `/home/nestri` in the image, and the image is mounted +/// read-only, so every library that follows XDG conventions to a default under +/// `$HOME` fails there. Measured 2026-09-12: the session manager could not +/// write its state on any boot, and anything asking Mesa for a shader cache was +/// told it was disabled. +/// +/// The second one is not a warning. Mesa with no writable cache recompiles +/// every shader on every run, and the symptom a person sees is a black screen +/// or a frozen game rather than a slow one. +/// +/// # Under the runtime directory rather than a tmpfs over the home +/// +/// Mounting a tmpfs at `/home/nestri` would work and would hide the shell +/// files the image put there, which is how a debug shell loses its prompt and +/// its history for no stated reason. The runtime directory is already a tmpfs, +/// already owned by this user, and already made before any service starts. +/// +/// Per boot, which is correct for these: a service's cache is not state anybody +/// wants to keep. A *workload's* cache is, and it is pointed at the writable +/// share it was given instead. +const WRITABLE: &[(&str, &str)] = &[ + ("HOME", "/home/nestri"), + ("XDG_RUNTIME_DIR", RUNTIME_DIR), + ("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR), + ("XDG_CACHE_HOME", "/run/user/1000/cache"), + ("XDG_STATE_HOME", "/run/user/1000/state"), + ("XDG_CONFIG_HOME", "/run/user/1000/config"), + ("XDG_DATA_HOME", "/run/user/1000/data"), +]; + +/// The stack, in the order it comes up. +/// +/// Ported from the nine init scripts this replaces, and the ordering is theirs: +/// the bus before anything that speaks on it, audio before whatever plays into +/// it, and the hub last because it binds the sockets the rest connect to. +pub const STACK: &[Service] = &[ + Service { + name: "dbus-system", + argv: &[ + "/usr/bin/dbus-daemon", + "--system", + "--nofork", + "--nopidfile", + ], + env: &[], + user: None, + cost: "nothing that speaks on the system bus can find it", + required: true, + umask: None, + ready: None, + }, + Service { + name: "dbus-session", + argv: &[ + "/usr/bin/dbus-daemon", + "--session", + "--nofork", + "--nopidfile", + "--address=unix:path=/run/user/1000/bus", + ], + env: &[("XDG_RUNTIME_DIR", RUNTIME_DIR)], + user: Some((SERVICE_UID, SERVICE_GID)), + cost: "audio and anything else expecting a session bus will not start", + required: true, + umask: None, + // Every service after this one is handed this path as its bus address, + // and a bus address that is not bound yet is a service that starts, + // finds nothing, and carries on without a bus. + ready: Some("/run/user/1000/bus"), + }, + Service { + name: "pipewire", + argv: &["/usr/bin/pipewire"], + env: &[ + ("XDG_RUNTIME_DIR", RUNTIME_DIR), + ("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR), + ("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"), + ], + user: Some((SERVICE_UID, SERVICE_GID)), + cost: "the session has no audio at all", + required: true, + // So the workload, which is not this user, can open the socket. + umask: Some(0), + // Both the session manager and the sender connect here, and so does + // the workload once it starts. + ready: Some("/run/pipewire/pipewire-0"), + }, + Service { + name: "wireplumber", + argv: &["/usr/bin/wireplumber"], + env: &[ + ("XDG_RUNTIME_DIR", RUNTIME_DIR), + ("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR), + ("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"), + ], + user: Some((SERVICE_UID, SERVICE_GID)), + // Optional on purpose: pipewire runs without a session manager, so a + // box with no wireplumber has audio nodes and nothing routing them, + // which is a degraded session rather than no session. + cost: "audio devices exist but nothing routes them", + required: false, + umask: None, + ready: None, + }, + Service { + name: "neswire", + argv: &["/usr/bin/neswire"], + env: &[ + ("XDG_RUNTIME_DIR", RUNTIME_DIR), + ("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR), + ("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"), + ], + user: Some((SERVICE_UID, SERVICE_GID)), + cost: "the client gets pictures and no sound", + required: false, + umask: None, + ready: None, + }, + Service { + name: "neshub", + argv: &["/usr/bin/neshub"], + env: &[("XDG_RUNTIME_DIR", RUNTIME_DIR)], + user: Some((SERVICE_UID, SERVICE_GID)), + // The one whose absence has no workaround: it owns the endpoint, so + // without it the session has no address and nothing can reach the box. + cost: "the session has no address, so no client can reach it", + required: true, + umask: None, + ready: None, + }, +]; + +/// The stack as running processes. +pub struct Stack { + waiters: Waiters, + table: &'static [Service], + running: Vec<(&'static str, Watched)>, + deaths: Receiver, + reported: Sender, +} + +impl Stack { + pub fn new(waiters: Waiters) -> Self { + Self::from_table(waiters, STACK) + } + + /// The same thing against a different table, which is how the ordering and + /// the required/optional rule are tested without a `/usr/bin` full of + /// services. + pub fn from_table(waiters: Waiters, table: &'static [Service]) -> Self { + // Small: what goes on it is one line per service death, and a box does + // not have many services to lose. + let (reported, deaths) = tokio::sync::mpsc::channel(16); + Self { + waiters, + table, + running: Vec::new(), + deaths, + reported, + } + } + + /// The pids of what is running, for a test that has to ask the kernel + /// whether they are still there. Nothing in the program uses it: signalling + /// happens in `Drop`, where the pids are already to hand. + pub fn pids(&self) -> Vec { + self.running.iter().map(|(_, w)| w.pid).collect() + } + + /// Wait for a service to bind the socket it said it would. + /// + /// Blocking, on a worker of a multi-threaded runtime: bring-up is a sequence + /// and there is nothing else for this task to do while it waits. Polling rather + /// than an inotify watch because the directory may not exist yet either, and a + /// watch that has to handle that is more machinery than 15 seconds of `stat`. + /// + /// A failure is the same shape as a failure to start, so the required/optional + /// rule above decides what it costs: a required service that never binds refuses + /// the box, an optional one is stepped over. + fn await_ready(&self, service: &Service, before: Option) -> Result<(), Failure> { + let Some(path) = service.ready else { + return Ok(()); + }; + // The watch for what was just started, so a service that dies during its + // own bring-up is not waited out for the full timeout. + let started = self.running.last().map(|(_, watched)| watched.pid); + await_path( + service.name, + path, + before, + &|| started.is_none_or(is_alive), + READY_TIMEOUT, + ) + } + + fn spawn(&mut self, service: &'static Service) -> Result<(), Failure> { + let Some((program, args)) = service.argv.split_first() else { + return Err(Failure::new(format!( + "{}: the command is empty", + service.name + ))); + }; + + // The standard library's process rather than the runtime's: the runtime + // reaps the children it spawns, and in this component reaping belongs + // to one place. See `reap::Waiters`. + let mut command = std::process::Command::new(program); + command.args(args); + command.env_clear(); + command.envs(WRITABLE.iter().copied()); + // The service's own entry last, so a service that states one of these + // for itself wins over the defaults above. + command.envs(service.env.iter().copied()); + + let mask = service.umask; + if let Some((uid, gid)) = service.user { + // SAFETY: the closure runs between fork and exec in the child, + // where only async-signal-safe calls are allowed. These two are, + // and it allocates nothing. + unsafe { + command.pre_exec(move || { + if let Some(mask) = mask { + // SAFETY: `umask` cannot fail and touches only this + // child, between fork and exec. + libc::umask(mask as libc::mode_t); + } + // gid first: dropping the uid first would lose the + // privilege needed to set the gid at all. + if libc::setgid(gid) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::setuid(uid) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } + + let mut watched = self + .waiters + .watch(|| Ok(command.spawn()?.id() as i32)) + .map_err(|error| Failure::new(format!("{}: {error}", service.name)))?; + + // The exit is moved onto a task that turns it into one line on the + // channel. Nothing here awaits it: bring-up is a sequence of starts, + // and a service that exits during it is a death like any other. + let exit = watched.take_exit().expect("a new watch has its exit"); + let reported = self.reported.clone(); + let name = service.name; + tokio::spawn(async move { + let exit = match exit.await { + Ok(exit) => exit, + // The watch was dropped, which happens on the way down. There + // is nothing to report and nobody left to report it to. + Err(_) => return, + }; + let _ = reported + .send(Died { + name: name.to_string(), + exit, + }) + .await; + }); + + self.running.push((service.name, watched)); + Ok(()) + } +} + +impl Services for Stack { + fn bring_up(&mut self) -> Result, Failure> { + let mut up = Vec::new(); + // Lifted out so the loop does not hold a borrow of `self` across the + // start it is asking for. + let table = self.table; + for service in table { + // Taken before the service is started, because what makes a + // socket this service's is that it was not there -- or was a + // different file -- a moment ago. + let before = service.ready.and_then(identity_of); + match self + .spawn(service) + .and_then(|()| self.await_ready(service, before)) + { + Ok(()) => { + tracing::info!(service = service.name, "started"); + up.push(service.name.to_string()); + } + Err(failure) if service.required => { + // Named with its cost rather than only its error: which + // service failed decides whether the box is worth having, + // and that judgement is made outside the box. + return Err(Failure::new(format!( + "{} — {}", + failure.reason, service.cost + ))); + } + Err(failure) => tracing::warn!( + service = service.name, + cost = service.cost, + "could not start, and the box goes on without it: {}", + failure.reason + ), + } + } + Ok(up) + } + + fn deaths(&mut self) -> &mut Receiver { + &mut self.deaths + } +} + +impl Drop for Stack { + /// Ask everything this stack started to stop. + /// + /// The stack owns these processes and nothing else does, so its going away + /// is the last moment anything knows their pids. As PID 1 the ordered + /// shutdown would reach them anyway and a second SIGTERM costs nothing; + /// run by hand it is the only thing that reaches them at all. + /// + /// Asked, not waited for: this runs while the runtime is going down, so + /// there is nothing left to reap them with. A signalled child that outlives + /// this process is reparented and dies on its own, which is the outcome we + /// wanted; an unsignalled one keeps its sockets. + fn drop(&mut self) { + for (name, watched) in &self.running { + // The same rule as everywhere else that signals: a pid that has + // been reaped may already belong to something else. + if !watched.running() { + continue; + } + tracing::debug!(service = name, pid = watched.pid, "stopping"); + // SAFETY: two integers, and a pid that has gone fails with ESRCH. + unsafe { libc::kill(watched.pid, libc::SIGTERM) }; + } + } +} + +/// Which file is at a path, as the kernel tells them apart. +/// +/// Device and inode rather than a modification time: a socket rebound in the +/// same second has the same mtime, and `dbus-daemon` and `pipewire` both unlink +/// and bind afresh, which is a new inode every time. +type Identity = (u64, u64); + +fn identity_of(path: &str) -> Option { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path).ok().map(|at| (at.dev(), at.ino())) +} + +/// Whether a pid is still a process at all. +/// +/// Signal 0 sends nothing and only asks. A child that has exited and not yet +/// been reaped still answers, which is why this is a second opinion rather than +/// the only one -- the watch's own record is the first. +fn is_alive(pid: i32) -> bool { + // SAFETY: two integers; a pid that has gone fails with ESRCH. + unsafe { libc::kill(pid, 0) == 0 } +} + +/// The waiting itself, with the deadline passed in so a test can assert the +/// giving-up without waiting out a real one. +fn await_path( + name: &str, + path: &str, + before: Option, + alive: &dyn Fn() -> bool, + timeout: std::time::Duration, +) -> Result<(), Failure> { + let deadline = std::time::Instant::now() + timeout; + loop { + // A *different* file than the one that was there before it started. + // + // Existence alone is not readiness, because `/run` is not always empty + // when this starts. In a box it is a fresh tmpfs and anything at these + // paths is ours; run by hand -- how a guest that will not boot is + // debugged -- the host's own `/run` is underneath, and a socket left by + // a previous run, or the developer's own session bus, is sitting at + // exactly the path being waited on. Taking that as proof would start + // everything downstream against a socket with nothing behind it, and + // report the box initialized. + // + // Compared rather than deleted. Unlinking first would be the obvious + // fix and it is the dangerous one: outside a box that path can belong + // to something else that is alive and using it. + let now = identity_of(path); + if now.is_some() && now != before { + return Ok(()); + } + // A service that has already left will not bind anything, and waiting + // out the full timeout for it buys nothing but a slower failure. + if !alive() { + return Err(Failure::new(format!( + "{name} exited before it bound {path}" + ))); + } + if std::time::Instant::now() >= deadline { + // The path, because that is the actionable half: a service that + // binds somewhere else is indistinguishable from one that never + // bound, and only one of those is fixed by looking at the service. + let stale = if before.is_some() { + ", and what is there is the file that was there before it started" + } else { + "" + }; + return Err(Failure::new(format!( + "{name}: {path} did not appear within {}s of starting it{stale}", + timeout.as_secs() + ))); + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +#[cfg(test)] +pub mod double { + use super::*; + + /// A stack that starts nothing, so what the session does with it is the + /// only thing under test. + pub struct Double { + pub brought_up: usize, + pub failure: Option, + pub names: Vec, + deaths: Receiver, + pub report: Sender, + } + + impl Default for Double { + fn default() -> Self { + Self::new() + } + } + + impl Double { + pub fn new() -> Self { + let (report, deaths) = tokio::sync::mpsc::channel(8); + Self { + brought_up: 0, + failure: None, + names: vec!["dbus-system".into(), "neshub".into()], + deaths, + report, + } + } + + /// A stack that refuses to come up, which refuses the box. + pub fn refuses(reason: &str) -> Self { + let mut double = Self::new(); + double.failure = Some(Failure::new(reason)); + double + } + } + + impl Services for Double { + fn bring_up(&mut self) -> Result, Failure> { + self.brought_up += 1; + match &self.failure { + Some(failure) => Err(failure.clone()), + None => Ok(self.names.clone()), + } + } + + fn deaths(&mut self) -> &mut Receiver { + &mut self.deaths + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The services and the workload have to look in the same place, and it + /// cannot be either one's runtime directory: those are 0700 and named + /// after a uid, and these are deliberately two different users. + #[test] + fn audio_is_somewhere_both_users_can_reach() { + assert!( + !AUDIO_DIR.starts_with("/run/user/"), + "a per-user runtime directory is 0700 and the other user is not in it" + ); + let audio: Vec<&Service> = STACK + .iter() + .filter(|s| s.env.iter().any(|(k, _)| *k == "PIPEWIRE_RUNTIME_DIR")) + .collect(); + assert!( + !audio.is_empty(), + "no service was told where audio lives, so none of them agree" + ); + for service in audio { + let told = service + .env + .iter() + .find(|(k, _)| *k == "PIPEWIRE_RUNTIME_DIR") + .map(|(_, v)| *v); + assert_eq!( + told, + Some(AUDIO_DIR), + "{} looks somewhere else", + service.name + ); + } + } + + /// A service others connect to is waited for, and the path waited on is + /// the path they are given. + /// + /// Two constants that have to agree and are written in two places is how + /// three of the four crossings in ref(d-0065) broke, so they are compared + /// here rather than trusted to stay in step. + #[test] + fn what_is_waited_for_is_where_the_others_are_told_to_look() { + let bus = STACK + .iter() + .find(|s| s.name == "dbus-session") + .expect("the session bus is in the table"); + let waited = bus.ready.expect( + "without this, everything handed this bus address is started before \ + anything is listening on it", + ); + let address = format!("unix:path={waited}"); + let clients: Vec<&Service> = STACK + .iter() + .filter(|s| s.env.iter().any(|(k, _)| *k == "DBUS_SESSION_BUS_ADDRESS")) + .collect(); + assert!(!clients.is_empty(), "nothing was told where the bus is"); + for client in clients { + let told = client + .env + .iter() + .find(|(k, _)| *k == "DBUS_SESSION_BUS_ADDRESS") + .map(|(_, v)| *v); + assert_eq!( + told, + Some(address.as_str()), + "{} connects somewhere the box never waited for", + client.name + ); + } + + let pipewire = STACK + .iter() + .find(|s| s.name == "pipewire") + .expect("audio is in the table"); + let waited = pipewire.ready.expect("audio is connected to by everything"); + assert!( + waited.starts_with(AUDIO_DIR), + "audio is waited for at {waited} and served from {AUDIO_DIR}" + ); + } + + /// A service that names no socket is ready when it has been started, and + /// the wait has to be free in that case: most of the table is like this. + #[test] + fn a_service_that_names_no_socket_is_not_waited_for() { + let hub = STACK + .iter() + .find(|s| s.name == "neshub") + .expect("the hub is in the table"); + assert!(hub.ready.is_none()); + let stack = Stack::from_table(Waiters::new(), STACK); + stack + .await_ready(hub, None) + .expect("a service with nothing to wait for waited anyway"); + } + + /// A file that was already there is not this service's socket. + /// + /// In a box `/run` is a fresh tmpfs and anything at these paths is ours. + /// Run by hand -- which is how a guest that will not boot is debugged -- + /// the host's own `/run` is underneath, and a socket from a previous run or + /// the developer's own session bus sits at exactly the path being waited + /// on. Taking it as proof starts everything downstream against a socket + /// with nothing behind it and reports the box initialized. + #[test] + fn a_file_that_was_there_before_is_not_proof_that_anything_started() { + let dir = std::env::temp_dir().join(format!("nesinit-stale-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("a directory to put a stale socket in"); + let path = dir.join("bus"); + std::fs::write(&path, b"a socket from a previous run").expect("the stale file"); + let at = path.to_str().expect("a path"); + + let before = identity_of(at); + assert!(before.is_some(), "the stale file is there to be found"); + + let failure = await_path( + "dbus-session", + at, + before, + &|| true, + std::time::Duration::from_millis(50), + ) + .expect_err("a file from before was taken as this service's socket"); + assert!( + failure.reason.contains("before it started"), + "the reason has to say which of the two failures this is: {}", + failure.reason + ); + + // Replaced, which is what binding a unix socket does: both daemons + // here unlink and bind afresh, so the inode is new. + std::fs::remove_file(&path).expect("removing the stale file"); + std::fs::write(&path, b"the new one").expect("the new file"); + await_path( + "dbus-session", + at, + before, + &|| true, + std::time::Duration::from_millis(50), + ) + .expect("a different file at the path is this service's socket"); + + std::fs::remove_dir_all(&dir).ok(); + } + + /// A service that died during its own bring-up is not waited out. + /// + /// The timeout is fifteen seconds and a dead service will never bind, so + /// the box would take that long to say something it already knew -- once + /// per service, in order. + #[test] + fn a_service_that_has_already_left_is_not_waited_for() { + let began = std::time::Instant::now(); + let failure = await_path( + "pipewire", + "/nonexistent/pipewire-0", + None, + &|| false, + std::time::Duration::from_secs(15), + ) + .expect_err("a dead service was treated as ready"); + assert!( + failure.reason.contains("exited before it bound"), + "{}", + failure.reason + ); + assert!( + began.elapsed() < std::time::Duration::from_secs(1), + "it waited out the timeout for a service that had already gone" + ); + } + + /// A socket that never appears is a failure, not a wait that ends quietly. + /// + /// The distinction matters because the required/optional rule above acts on + /// it: a required service that never binds has to refuse the box rather + /// than let one boot that reports itself ready and does not work. + #[test] + fn a_socket_that_never_appears_is_a_failure_that_names_it() { + let failure = await_path( + "pipewire", + "/nonexistent/pipewire-0", + None, + &|| true, + std::time::Duration::from_millis(50), + ) + .expect_err("a socket that is not there was treated as ready"); + assert!(failure.reason.contains("pipewire"), "{}", failure.reason); + assert!( + failure.reason.contains("/nonexistent/pipewire-0"), + "{}", + failure.reason + ); + } + + /// A unix socket is created 0777 masked by the umask, and connecting to + /// one needs write. The inherited 022 therefore refuses every user but the + /// owner -- and the workload is not the owner. + #[test] + fn the_audio_socket_is_reachable_by_a_user_who_does_not_own_it() { + let pipewire = STACK + .iter() + .find(|s| s.name == "pipewire") + .expect("audio is in the table"); + assert_eq!( + pipewire.umask, + Some(0), + "with any other umask the game finds the socket and cannot open it" + ); + } + + /// The two that failed on a real boot, and the reason each matters. + #[test] + fn every_service_has_somewhere_to_write() { + let names: Vec<&str> = WRITABLE.iter().map(|(k, _)| *k).collect(); + assert!( + names.contains(&"XDG_STATE_HOME"), + "the session manager could not write its state on any boot" + ); + assert!( + names.contains(&"XDG_CACHE_HOME"), + "no shader cache means recompiling every shader every run, and \ + what that looks like is a black screen rather than a slow one" + ); + for (_, path) in WRITABLE { + if path.starts_with("/run/") || *path == "/home/nestri" { + continue; + } + panic!("{path} is not somewhere a read-only root lets a service write"); + } + } + + /// The table is data, and the things that make it wrong are checkable + /// without running any of it. + #[test] + fn every_service_can_be_started_and_says_what_it_costs() { + for service in STACK { + assert!(!service.name.is_empty(), "a service with no name"); + assert!(!service.argv.is_empty(), "{}: nothing to run", service.name); + assert!( + service.argv[0].starts_with('/'), + "{}: not an absolute path, so it depends on a PATH init does not set", + service.name + ); + assert!( + !service.cost.is_empty(), + "{}: no cost, so a failure cannot be judged from outside", + service.name + ); + } + } + + #[test] + fn no_service_is_named_twice() { + let mut names: Vec<_> = STACK.iter().map(|s| s.name).collect(); + names.sort_unstable(); + let count = names.len(); + names.dedup(); + assert_eq!(count, names.len(), "two services share a name: {names:?}"); + } + + /// The compositor is started by a launch, with that launch's geometry. One + /// in this table would be one with no geometry to come up with. + #[test] + fn the_compositor_is_not_a_service() { + for service in STACK { + assert!( + !service.name.contains("scope") && !service.argv[0].contains("nescope"), + "the compositor is in the service table: {}", + service.name + ); + } + } + + /// Every service runs as init or as the one service user, and never as + /// anything else. + /// + /// The uid a workload runs as arrives in its launch and is not known here, + /// so this cannot compare the two directly. What it can do is refuse a + /// third user appearing in this table — because the separation that matters + /// is that a workload never shares a user with these, and a service quietly + /// given some other uid is how that stops being true. A workload sharing a + /// user with a service can replace a socket it listens on and answer in its + /// place, and the answer that matters is the address a client is told to + /// connect to. See `ticket::Untrusted`. + #[test] + fn a_service_runs_as_init_or_as_the_service_user_and_nothing_else() { + for service in STACK { + if let Some((uid, gid)) = service.user { + assert_eq!( + (uid, gid), + (SERVICE_UID, SERVICE_GID), + "{} runs as a third user", + service.name + ); + } + } + } + + /// A required service whose absence has a workaround should not be + /// required, and an optional one whose absence has none should not be + /// optional. Only the second half is checkable, and it is the one that + /// matters: the address is what a client needs. + #[test] + fn whatever_owns_the_address_is_required() { + let hub = STACK + .iter() + .find(|s| s.name == "neshub") + .expect("something has to own the endpoint"); + assert!( + hub.required, + "a box with no address is a box nothing can reach" + ); + } +} diff --git a/apps/nesinit/src/session.rs b/apps/nesinit/src/session.rs index a94e0b33..c40d8bc8 100644 --- a/apps/nesinit/src/session.rs +++ b/apps/nesinit/src/session.rs @@ -1,44 +1,76 @@ // The guest end of the control channel. // // The shape of the exchange, and none of it is negotiable from this side: the -// guest speaks first with its version, is handed one boot descriptor, and from -// then on reports. It is not a supervisor: when the workload ends, the exit -// goes up the channel and this returns. Starting something again is the -// caller's decision, because the caller is the only end that can see whether -// restarting is repair or a loop. ref(d-0033) +// guest speaks first with its version, is handed one document describing the +// box, brings the box's own services up, says so — and then takes commands for +// as long as the box lives. ref(d-0064) +// +// It is not a supervisor. Nothing here restarts anything of its own accord: a +// workload's exit is reported, a service's death is reported, and starting +// something again is the caller's decision, because the caller is the only end +// that can see whether restarting is repair or a loop. ref(d-0033) +// +// # One launch at a time +// +// A box may be launched into many times over its life, and not twice at once. +// A launch arriving while one is running is refused with the id it was asked +// for, rather than queued or silently replacing it — a box has one compositor +// and one screen, so a second concurrent launch has nowhere to draw. Nothing +// in the wire format forbids it if that ever changes; this is a property of +// this implementation and it says so where it refuses. use nesprotocol::lifecycle::{ - CONTROL_VERSION, Exit, GuestToHost, HostToGuest, Payload, from_line, to_line, + CONTROL_VERSION, Exec, Exit, GuestToHost, HostToGuest, LaunchId, OnExit, Payload, from_line, + to_line, }; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader}; use crate::payload::Ports; +use crate::services::{Died, Services}; use crate::workload::{Exited, Failure, Workload}; use tokio::sync::mpsc::Receiver; /// How a session ended. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Outcome { - /// The workload ended and the exit was reported. + /// A launch whose exit was terminal ended, and the exit was reported. WorkloadExited(Exit), /// The caller asked for a shutdown. Shutdown, /// The channel closed under us. Not an error by itself — a caller that has /// stopped listening has also stopped being able to tell us to stop. ChannelClosed, - /// The descriptor could not be carried out. The reason is the operating - /// system's, verbatim. + /// The box could not be made. The reason is the operating system's, + /// verbatim. + /// + /// This is the descriptor or the service stack, never a launch: a command + /// that will not run is reported and the box stays up, because a box that + /// dies of a bad `argv` cannot be told a better one. Refused(Failure), } +/// The launch that is running, and what it would take to start it again. +struct Current { + id: LaunchId, + /// Kept so a restart is the same command rather than a new one the caller + /// has to re-send. + exec: Exec, + on_exit: OnExit, + exited: Exited, + /// Set by `restart`, read when the exit arrives. A restart is a kill + /// followed by a launch and this is the "followed by". + relaunch: bool, +} + /// Run one session over an already-connected channel. /// /// Generic over the channel so the exchange can be driven from a test without /// a VM: the transport contributes nothing to the protocol beyond ordering and /// framing, which any byte stream has. -pub async fn run( +pub async fn run( channel: C, workload: &mut W, + services: &mut S, payload: &mut Ports, addresses: &mut Receiver, untrusted: &crate::ticket::Untrusted, @@ -46,8 +78,9 @@ pub async fn run( where C: AsyncRead + AsyncWrite, W: Workload, + S: Services, { - match converse(channel, workload, payload, addresses, untrusted).await { + match converse(channel, workload, services, payload, addresses, untrusted).await { Err(error) if channel_gone(&error) => { // A caller that has stopped reading has also stopped being able to // tell us to stop, which is the same situation as the channel @@ -69,9 +102,10 @@ fn channel_gone(error: &std::io::Error) -> bool { ) } -async fn converse( +async fn converse( channel: C, workload: &mut W, + services: &mut S, payload: &mut Ports, addresses: &mut Receiver, untrusted: &crate::ticket::Untrusted, @@ -79,6 +113,7 @@ async fn converse( where C: AsyncRead + AsyncWrite, W: Workload, + S: Services, { let (reader, mut writer) = tokio::io::split(channel); let mut lines = BufReader::new(reader).lines(); @@ -94,29 +129,88 @@ where ) .await?; - let mut running: Option = None; + let mut running: Option = None; + let mut booted = false; let mut relay_open = true; let mut carrier_open = true; + // Guarded like the other two, and for the same reason: a closed channel + // resolves immediately and forever, so an unguarded branch on one turns + // this loop into a spin that still looks like it is waiting. + let mut services_open = true; loop { let event = match running.as_mut() { - Some(exited) => tokio::select! { - ended = exited => Event::Ended(ended?), + Some(current) => tokio::select! { + ended = &mut current.exited => Event::Ended(ended?), line = lines.next_line() => Event::Line(line?), up = payload.from_workload.recv(), if relay_open => Event::FromWorkload(up), found = addresses.recv(), if carrier_open => Event::Address(found), + gone = services.deaths().recv(), if services_open => Event::ServiceDied(gone), }, None => tokio::select! { line = lines.next_line() => Event::Line(line?), up = payload.from_workload.recv(), if relay_open => Event::FromWorkload(up), found = addresses.recv(), if carrier_open => Event::Address(found), + gone = services.deaths().recv(), if services_open => Event::ServiceDied(gone), }, }; let line = match event { Event::Ended(exit) => { - send(&mut writer, &GuestToHost::WorkloadExited { exit }).await?; - return Ok(Outcome::WorkloadExited(exit)); + // Taken before anything is sent: whatever happens next, this + // launch is no longer the running one, and a failure to + // relaunch must not leave a dead launch looking live. + let ended = running.take().expect("an exit came from a launch"); + send( + &mut writer, + &GuestToHost::WorkloadExited { + id: ended.id.clone(), + exit, + }, + ) + .await?; + + if ended.relaunch { + // A restart is a kill followed by a launch of the same + // command, and this is the second half. The id is kept, so + // a caller sees the exit and then the same launch running + // again rather than having to correlate a new name. + running = start( + &mut writer, + workload, + untrusted, + ended.id, + ended.exec, + ended.on_exit, + ) + .await?; + continue; + } + + // Terminal is the launch's own answer, not this end's. A + // non-terminal exit leaves the box up and waiting to be + // launched into again, which is the whole point of the box + // outliving what runs in it. ref(d-0064) + if ended.on_exit.terminal { + return Ok(Outcome::WorkloadExited(exit)); + } + tracing::info!(launch = %ended.id, "a launch ended and the box stays up"); + continue; + } + Event::ServiceDied(Some(Died { name, exit })) => { + // Reported, never repaired. Nothing else in this guest is + // watching these, so a death that is not said here is a box + // that looks healthy and cannot work. ref(d-0064) + tracing::error!(service = %name, ?exit, "a service died"); + send(&mut writer, &GuestToHost::ServiceDied { name, exit }).await?; + continue; + } + Event::ServiceDied(None) => { + // The supervisor is gone, which on the way down is ordinary. + // Nothing to report and nothing to end: a box with no service + // stack left can still be stopped and still has to report it. + services_open = false; + continue; } Event::FromWorkload(Some(payload)) => { tracing::debug!(envelope = %payload.summary(), "sending an envelope on"); @@ -166,14 +260,15 @@ where match message { HostToGuest::Boot { descriptor } => { - if running.is_some() { + if booted { tracing::warn!("ignoring a second descriptor: one is read per connection"); continue; } + booted = true; - // The shares, then the command, and each reported separately. - // Which of the two failed decides what is worth looking at, - // so the two are never one message. + // The shares, then the services, and each reported separately. + // Which of the two failed decides what is worth looking at, so + // the two are never one message. match workload.mount(&descriptor.mounts) { Ok(()) => send(&mut writer, &GuestToHost::Mounted).await?, Err(failure) => { @@ -188,20 +283,18 @@ where } } - // Before the workload exists, so there is no window in which - // it is running and something else would still be trusted to - // serve this session's address. - untrusted.is(descriptor.exec.uid); - - match workload.start(&descriptor.exec) { - Ok(exited) => { - send(&mut writer, &GuestToHost::Started).await?; - running = Some(exited); + // A box whose own services will not come up cannot be launched + // into, so this is refused rather than reported and carried on + // from — unlike a launch, which is the caller's to correct. + match services.bring_up() { + Ok(up) => { + tracing::info!(services = up.len(), "the box is ready to be launched into"); + send(&mut writer, &GuestToHost::Initialized { services: up }).await? } Err(failure) => { send( &mut writer, - &GuestToHost::StartFailed { + &GuestToHost::InitFailed { reason: failure.reason.clone(), }, ) @@ -210,19 +303,142 @@ where } } } + HostToGuest::Launch { id, exec, on_exit } => { + if !booted { + // Before the descriptor there are no shares and no + // services, so whatever this launch expects to find is not + // there yet. Refused with its own id rather than run into + // a box that is not finished. + refuse(&mut writer, id, "the box has not been told what it is").await?; + continue; + } + if let Some(current) = &running { + let reason = format!("this box is already running a launch: {}", current.id); + refuse(&mut writer, id, &reason).await?; + continue; + } + running = start(&mut writer, workload, untrusted, id, exec, on_exit).await?; + } + HostToGuest::Stop { id } => match &running { + // Idempotent, and never ends the session by itself. + Some(current) if current.id == id => workload.signal_stop(), + // Not an error and not worth failing: a caller stopping + // something that has already stopped got what it asked for. + // Signalling anyway would aim a kill at whatever is running + // now, which is the one thing this must not do. + _ => tracing::debug!(launch = %id, "nothing with that id is running to stop"), + }, + HostToGuest::Restart { id } => match running.as_mut() { + Some(current) if current.id == id => { + // Kill now, launch when the exit arrives. Doing both here + // would start the second before the first had gone. + current.relaunch = true; + workload.signal_stop(); + } + // Nothing is remembered about a launch that has already ended, + // so this cannot be a launch in disguise: the caller has the + // command and can send it. + _ => { + refuse( + &mut writer, + id, + "nothing with that id is running to restart", + ) + .await?; + } + }, HostToGuest::Payload { payload: envelope } => hand_over(payload, envelope), - HostToGuest::Stop => workload.signal_stop(), HostToGuest::Shutdown => return Ok(Outcome::Shutdown), } } } -/// What the session is waiting on, and there are only three things. +/// Start one launch, reporting which of the two things happened. +/// +/// Returns `None` when it could not be started, which is not the end of the +/// session: a box that dies of a bad `argv` cannot be told a better one. +async fn start( + writer: &mut Wr, + workload: &mut W, + untrusted: &crate::ticket::Untrusted, + id: LaunchId, + exec: Exec, + on_exit: OnExit, +) -> std::io::Result> +where + W: Workload, + Wr: AsyncWrite + Unpin, +{ + // **A workload may not be the services' user.** Two things break at once if + // it is. The workload could replace a socket one of the services listens + // on and answer in its place -- which for the address a client is told to + // connect to means answering for the session. And the refusal below is by + // uid, so marking the workload untrusted marks the real hub untrusted too, + // and the session becomes unreachable by its own defence. + // + // Refused rather than silently moved to another uid: a launch that runs as + // somebody other than who it named is a launch whose files land in the + // wrong place, which is a worse day than a launch that did not start. + if exec.uid == crate::services::SERVICE_UID { + refuse( + writer, + id, + &format!( + "a launch may not run as uid {}: that is the box's own service \ + user, and a workload sharing it can answer for the session", + crate::services::SERVICE_UID + ), + ) + .await?; + return Ok(None); + } + + // Before the workload exists, so there is no window in which it is running + // and something else would still be trusted to serve this session's + // address. + untrusted.is(exec.uid); + + match workload.start(&exec) { + Ok(exited) => { + send(writer, &GuestToHost::Started { id: id.clone() }).await?; + Ok(Some(Current { + id, + exec, + on_exit, + exited, + relaunch: false, + })) + } + Err(failure) => { + refuse(writer, id, &failure.reason).await?; + Ok(None) + } + } +} + +/// Say that a launch will not happen, in the words of whatever refused it. +async fn refuse(writer: &mut Wr, id: LaunchId, reason: &str) -> std::io::Result<()> +where + Wr: AsyncWrite + Unpin, +{ + tracing::warn!(launch = %id, reason, "refusing a launch"); + send( + writer, + &GuestToHost::StartFailed { + id, + reason: reason.to_string(), + }, + ) + .await +} + +/// What the session is waiting on. enum Event { Line(Option), Ended(Exit), FromWorkload(Option), Address(Option), + ServiceDied(Option), } /// Hand an envelope to the relay, and treat a relay that is not there as the @@ -260,35 +476,51 @@ where #[cfg(test)] mod tests { use super::*; + use crate::services::double::Double as Stack; use crate::workload::double::Double; - use nesprotocol::lifecycle::{BootDescriptor, Exec, Geometry, Mount, OnExit}; + use nesprotocol::lifecycle::{BootDescriptor, Mount}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, DuplexStream}; use tokio::sync::mpsc; + /// The box: one share, and nothing about what runs in it. fn descriptor() -> BootDescriptor { BootDescriptor { - exec: Exec { - argv: vec!["/usr/bin/workload".into(), "--windowed".into()], - env: Default::default(), - cwd: None, - uid: 1000, - gid: 1000, - }, mounts: vec![Mount { tag: "user".into(), at: "/mnt/user".into(), ro: false, }], - geometry: Geometry { - width: 1920, - height: 1080, - fps: 60, - hdr: false, - }, + } + } + + fn exec() -> Exec { + Exec { + argv: vec!["/usr/bin/workload".into(), "--windowed".into()], + env: Default::default(), + cwd: None, + uid: 1001, + gid: 1001, + } + } + + fn launch(id: &str) -> HostToGuest { + HostToGuest::Launch { + id: LaunchId::new(id), + exec: exec(), on_exit: OnExit { terminal: true }, } } + /// A launch whose ending is not the session's ending, which is what lets a + /// box be launched into again. + fn launch_and_stay(id: &str) -> HostToGuest { + HostToGuest::Launch { + id: LaunchId::new(id), + exec: exec(), + on_exit: OnExit { terminal: false }, + } + } + /// The relay's two ends, as the session sees them, plus the ends a /// workload on the relay would hold. fn ports() -> (Ports, mpsc::Receiver, mpsc::Sender) { @@ -315,6 +547,55 @@ mod tests { rx } + /// What a session is given, so a test names only the part it is about. + struct Given { + workload: Double, + services: Stack, + ports: Ports, + addresses: mpsc::Receiver, + } + + impl Given { + fn new(workload: Double) -> Self { + let (ports, down_rx, up_tx) = ports(); + // The workload's ends of the relay, kept alive so the relay is + // neither full nor gone in tests that are not about either. + Box::leak(Box::new((down_rx, up_tx))); + Self { + workload, + services: Stack::new(), + ports, + addresses: nowhere(), + } + } + } + + /// Start a session and hand back what a test wants to assert against. + fn spawn( + guest: DuplexStream, + given: Given, + ) -> tokio::task::JoinHandle<(Outcome, Double, Stack)> { + let Given { + mut workload, + mut services, + mut ports, + mut addresses, + } = given; + tokio::spawn(async move { + let outcome = run( + guest, + &mut workload, + &mut services, + &mut ports, + &mut addresses, + &crate::ticket::Untrusted::unknown(), + ) + .await + .unwrap(); + (outcome, workload, services) + }) + } + /// The other end of the channel, as a caller would drive it. struct Caller { lines: tokio::io::Lines>, @@ -337,10 +618,23 @@ mod tests { from_line(&line).unwrap() } - /// A descriptor being carried out: the shares, then the command. - async fn expect_started(&mut self) { + async fn expect_ready(&mut self) { + assert_eq!( + self.expect().await, + GuestToHost::Ready { + protocol_version: CONTROL_VERSION + } + ); + } + + /// The descriptor being carried out: the shares, then the box's own + /// services. Nothing is launched by either. + async fn expect_booted(&mut self) { assert_eq!(self.expect().await, GuestToHost::Mounted); - assert_eq!(self.expect().await, GuestToHost::Started); + assert!( + matches!(self.expect().await, GuestToHost::Initialized { .. }), + "the box never said it was ready to be launched into" + ); } async fn say(&mut self, message: &HostToGuest) { @@ -357,33 +651,523 @@ mod tests { async fn the_guest_speaks_first_and_says_its_version() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); - - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); // Nothing has been sent to the guest, so this can only be unprompted. + caller.expect_ready().await; + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, _, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + } + + /// The descriptor makes the box; it does not run anything. This is the + /// property the whole resident arrangement rests on. + #[tokio::test] + async fn the_descriptor_mounts_and_starts_the_services_and_launches_nothing() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + caller.say(&HostToGuest::Shutdown).await; + + let (outcome, workload, services) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!(workload.mounted, vec![descriptor().mounts]); + assert_eq!(services.brought_up, 1); + assert!( + workload.started.is_empty(), + "the descriptor started something, which is a launch's job" + ); + } + + #[tokio::test] + async fn a_launch_runs_what_it_names_and_is_reported_by_its_id() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch("l-1")).await; assert_eq!( caller.expect().await, - GuestToHost::Ready { - protocol_version: 2 + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); + + caller + .say(&HostToGuest::Stop { + id: LaunchId::new("l-1"), + }) + .await; + assert_eq!( + caller.expect().await, + GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), + exit: Exit::code(0) + } + ); + + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(0))); + assert_eq!(workload.started, vec![exec()]); + } + + /// A box the caller has not described has no shares and no services, so + /// whatever a launch expects to find is not there. + #[tokio::test] + async fn a_launch_before_the_box_exists_is_refused() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller.say(&launch("l-1")).await; + + let GuestToHost::StartFailed { id, reason } = caller.expect().await else { + panic!("a launch into a box that does not exist was accepted") + }; + assert_eq!(id, LaunchId::new("l-1"), "the refusal names another launch"); + assert!(!reason.is_empty()); + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown, "the box died of a bad launch"); + assert!(workload.started.is_empty()); + } + + /// Two at once has nowhere to draw. The refusal carries the id that was + /// refused, not the one that is running. + #[tokio::test] + async fn a_second_launch_is_refused_while_the_first_is_running() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); + + caller.say(&launch("l-2")).await; + let GuestToHost::StartFailed { id, reason } = caller.expect().await else { + panic!("a second concurrent launch was accepted") + }; + assert_eq!( + id, + LaunchId::new("l-2"), + "the refusal names the launch that is running rather than the one refused" + ); + assert!( + reason.contains("l-1"), + "the refusal does not say why: {reason}" + ); + + caller.say(&HostToGuest::Shutdown).await; + let (_, workload, _) = session.await.unwrap(); + assert_eq!( + workload.started.len(), + 1, + "the second launch ran anyway, so the first was replaced underneath it" + ); + } + + /// A launch that says its ending is not the session's leaves the box up and + /// able to be launched into again. This is the whole of what "the box + /// outlives what runs in it" means. + #[tokio::test] + async fn a_box_can_be_launched_into_again_after_a_launch_ends() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch_and_stay("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); + caller + .say(&HostToGuest::Stop { + id: LaunchId::new("l-1"), + }) + .await; + assert_eq!( + caller.expect().await, + GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), + exit: Exit::code(0) + } + ); + + // The session is still here, which is the assertion. + caller.say(&launch_and_stay("l-2")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-2") } ); caller.say(&HostToGuest::Shutdown).await; - let (outcome, _) = session.await.unwrap(); + let (outcome, workload, services) = session.await.unwrap(); assert_eq!(outcome, Outcome::Shutdown); + assert_eq!(workload.started.len(), 2); + assert_eq!( + services.brought_up, 1, + "the services were brought up again for a second launch" + ); + } + + /// A restart is a kill followed by a launch of the same command, and both + /// halves are visible: the exit is reported and then the same id runs + /// again. Nothing is restarted by this end on its own. + #[tokio::test] + async fn a_restart_reports_the_exit_and_runs_the_same_command_again() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); + + caller + .say(&HostToGuest::Restart { + id: LaunchId::new("l-1"), + }) + .await; + assert_eq!( + caller.expect().await, + GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), + exit: Exit::code(0) + }, + "a restart hid the exit it caused" + ); + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + }, + "a restart kept the id but did not run again" + ); + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!( + workload.started, + vec![exec(), exec()], + "a restart ran a different command than the one it restarted" + ); + } + + /// Nothing is remembered about a launch that has ended, so a restart of one + /// is refused rather than guessed at from a stale command. + #[tokio::test] + async fn a_restart_of_nothing_is_refused_and_does_not_end_the_session() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller + .say(&HostToGuest::Restart { + id: LaunchId::new("never-ran"), + }) + .await; + let GuestToHost::StartFailed { id, .. } = caller.expect().await else { + panic!("a restart of nothing was accepted") + }; + assert_eq!(id, LaunchId::new("never-ran")); + + caller.say(&HostToGuest::Shutdown).await; + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); + } + + /// A stop for something that is not running is what a caller gets when it + /// stops a launch twice. Signalling anyway would aim a kill at whatever is + /// running now, which is the one thing it must not do. + #[tokio::test] + async fn a_stop_for_a_launch_that_is_not_running_signals_nothing() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch_and_stay("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); + + // Something else entirely, while l-1 runs. + caller + .say(&HostToGuest::Stop { + id: LaunchId::new("l-2"), + }) + .await; + caller.say(&HostToGuest::Shutdown).await; + + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!( + workload.stops, 0, + "a stop aimed at one launch reached another" + ); + } + + /// A command that will not run is the caller's to correct, so the box stays + /// up and can be told a better one. + #[tokio::test] + async fn a_launch_that_will_not_run_does_not_take_the_box_down() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let mut workload = Double::exits_at_once(Exit::code(0)); + workload.start_failure = Some(Failure::new("ENOENT: /usr/bin/workload")); + let session = spawn(guest, Given::new(workload)); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + caller.say(&launch("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::StartFailed { + id: LaunchId::new("l-1"), + reason: "ENOENT: /usr/bin/workload".into() + }, + "the reason is passed through as the operating system wrote it", + ); + + // Still answering, which is the assertion. + caller.say(&HostToGuest::Shutdown).await; + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); + } + + /// The one uid a launch may not ask for is the one the box's own services + /// hold. A workload running as them can replace a socket they listen on and + /// answer for the session; and because the carrier refuses peers by uid, + /// marking the workload untrusted marks the real hub untrusted with it. + /// + /// Refused, and nothing is started: a launch moved quietly to another user + /// writes its files somewhere nobody asked for. + #[tokio::test] + async fn a_launch_may_not_run_as_the_services_own_user() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + let HostToGuest::Launch { id, mut exec, .. } = launch("l-1") else { + unreachable!() + }; + exec.uid = crate::services::SERVICE_UID; + exec.gid = crate::services::SERVICE_GID; + caller + .say(&HostToGuest::Launch { + id, + exec, + on_exit: OnExit { terminal: true }, + }) + .await; + + let GuestToHost::StartFailed { id, reason } = caller.expect().await else { + panic!("a launch as the service user was accepted") + }; + assert_eq!(id, LaunchId::new("l-1")); + assert!( + reason.contains(&crate::services::SERVICE_UID.to_string()), + "the refusal names the uid it is about: {reason}" + ); + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown, "the refusal ended the box"); + assert!( + workload.started.is_empty(), + "the workload was started anyway" + ); + } + + #[tokio::test] + async fn a_share_that_will_not_mount_is_refused_before_any_service_starts() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let mut workload = Double::exits_at_once(Exit::code(0)); + workload.mount_failure = Some(Failure::new("EACCES: /mnt/user")); + let session = spawn(guest, Given::new(workload)); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + + assert_eq!( + caller.expect().await, + GuestToHost::MountFailed { + reason: "EACCES: /mnt/user".into() + }, + "the reason is passed through as the operating system wrote it", + ); + + let (outcome, workload, services) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Refused(Failure::new("EACCES: /mnt/user"))); + assert_eq!( + services.brought_up, 0, + "the services came up in a box whose shares are not there" + ); + assert!(workload.started.is_empty()); + } + + /// A box whose own services will not come up cannot be launched into, so it + /// is refused — and said with its own message, because a share that did not + /// appear and a box that could not be made want different things looked at. + #[tokio::test] + async fn a_service_stack_that_will_not_come_up_refuses_the_box() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let mut given = Given::new(Double::exits_at_once(Exit::code(0))); + given.services = Stack::refuses("neshub: ENOENT — the session has no address"); + let session = spawn(guest, given); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + + assert_eq!(caller.expect().await, GuestToHost::Mounted); + let GuestToHost::InitFailed { reason } = caller.expect().await else { + panic!("a box with no services said it was ready to be launched into") + }; + assert!(reason.contains("neshub"), "{reason}"); + + let (outcome, workload, _) = session.await.unwrap(); + assert!(matches!(outcome, Outcome::Refused(_)), "{outcome:?}"); + assert!(workload.started.is_empty()); + } + + /// Reported, never repaired, and it does not end the session: a box with a + /// dead service is still a box somebody has to be told about. + #[tokio::test] + async fn a_dead_service_is_reported_and_not_restarted() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let given = Given::new(Double::exits_when_stopped(Exit::code(0))); + let report = given.services.report.clone(); + let session = spawn(guest, given); + + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + + report + .send(Died { + name: "pipewire".into(), + exit: Exit::signal(9), + }) + .await + .unwrap(); + assert_eq!( + caller.expect().await, + GuestToHost::ServiceDied { + name: "pipewire".into(), + exit: Exit::signal(9) + } + ); + + // Still answering, and nothing was brought up again. + caller.say(&HostToGuest::Shutdown).await; + let (outcome, _, services) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!( + services.brought_up, 1, + "a dead service was restarted by the guest" + ); } /// The address goes up the channel as it is found. This is the only way @@ -393,28 +1177,12 @@ mod tests { async fn an_address_that_is_found_is_reported_to_the_caller() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); - let (found_tx, mut found_rx) = mpsc::channel(4); + let (found_tx, found_rx) = mpsc::channel(4); + let mut given = Given::new(Double::exits_when_stopped(Exit::code(0))); + given.addresses = found_rx; + let session = spawn(guest, given); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut found_rx, - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert_eq!( - caller.expect().await, - GuestToHost::Ready { - protocol_version: 2 - } - ); + caller.expect_ready().await; found_tx .send("nestri:local-only".to_string()) @@ -441,7 +1209,7 @@ mod tests { ); caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } /// Nothing looking for an address any more is not an ending. The session @@ -450,105 +1218,49 @@ mod tests { async fn a_carrier_that_stops_does_not_end_the_session() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); - let (found_tx, mut found_rx) = mpsc::channel(4); + let (found_tx, found_rx) = mpsc::channel(4); + let mut given = Given::new(Double::exits_when_stopped(Exit::code(0))); + given.addresses = found_rx; + let session = spawn(guest, given); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut found_rx, - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert_eq!( - caller.expect().await, - GuestToHost::Ready { - protocol_version: 2 - } - ); + caller.expect_ready().await; drop(found_tx); // Still answering, which is the whole assertion. caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); - } - - #[tokio::test] - async fn the_descriptor_mounts_and_starts_what_it_names() { - let (guest, host) = tokio::io::duplex(4096); - let mut caller = Caller::new(host); - - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); - caller - .say(&HostToGuest::Boot { - descriptor: Box::new(descriptor()), - }) - .await; - caller.say(&HostToGuest::Stop).await; - - let (outcome, workload) = session.await.unwrap(); - assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(0))); - assert_eq!(workload.mounted, vec![descriptor().mounts]); - assert_eq!(workload.started, vec![descriptor().exec]); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } #[tokio::test] async fn an_exit_is_reported_and_the_workload_is_not_started_again() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_at_once(Exit::code(3)))); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_at_once(Exit::code(3)); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; caller .say(&HostToGuest::Boot { descriptor: Box::new(descriptor()), }) .await; - caller.expect_started().await; + caller.expect_booted().await; + caller.say(&launch("l-1")).await; + assert_eq!( + caller.expect().await, + GuestToHost::Started { + id: LaunchId::new("l-1") + } + ); assert_eq!( caller.expect().await, GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), exit: Exit::code(3) }, ); - let (outcome, workload) = session.await.unwrap(); + let (outcome, workload, _) = session.await.unwrap(); assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(3))); assert_eq!( workload.started.len(), @@ -561,178 +1273,68 @@ mod tests { async fn a_signalled_workload_is_reported_as_signalled() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_at_once(Exit::signal(9)))); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_at_once(Exit::signal(9)); - run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; caller .say(&HostToGuest::Boot { descriptor: Box::new(descriptor()), }) .await; - caller.expect_started().await; + caller.expect_booted().await; + caller.say(&launch("l-1")).await; + assert!(matches!(caller.expect().await, GuestToHost::Started { .. })); assert_eq!( caller.expect().await, GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), exit: Exit::signal(9) }, ); assert_eq!( - session.await.unwrap(), + session.await.unwrap().0, Outcome::WorkloadExited(Exit::signal(9)) ); } - #[tokio::test] - async fn a_stop_is_idempotent_and_does_not_end_the_session() { - let (guest, host) = tokio::io::duplex(4096); - let mut caller = Caller::new(host); - - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); - // No descriptor yet, so there is nothing to stop and the session has - // to survive being told to anyway. - caller.say(&HostToGuest::Stop).await; - caller.say(&HostToGuest::Stop).await; - caller.say(&HostToGuest::Shutdown).await; - - let (outcome, workload) = session.await.unwrap(); - assert_eq!(outcome, Outcome::Shutdown); - assert_eq!(workload.stops, 2); - assert!(workload.started.is_empty()); - } - - #[tokio::test] - async fn a_share_that_will_not_mount_is_refused_before_anything_starts() { - let (guest, host) = tokio::io::duplex(4096); - let mut caller = Caller::new(host); - - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_at_once(Exit::code(0)); - workload.mount_failure = Some(Failure::new("EACCES: /mnt/user")); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); - caller - .say(&HostToGuest::Boot { - descriptor: Box::new(descriptor()), - }) - .await; - - assert_eq!( - caller.expect().await, - GuestToHost::MountFailed { - reason: "EACCES: /mnt/user".into() - }, - "the reason is passed through as the operating system wrote it", - ); - - let (outcome, workload) = session.await.unwrap(); - assert_eq!(outcome, Outcome::Refused(Failure::new("EACCES: /mnt/user"))); - assert!( - workload.started.is_empty(), - "a workload without its shares is not started" - ); - } - #[tokio::test] async fn an_unreadable_line_does_not_end_a_session() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; caller .lines .get_mut() - .write_all(b"{\"type\":\"from_a_later_version\"}\n") + .write_all(br#"{"type":"from_a_later_version"}"#) .await .unwrap(); + caller.lines.get_mut().write_all(b"\n").await.unwrap(); caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } #[tokio::test] async fn a_closed_channel_stops_the_workload() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_when_stopped(Exit::code(0)); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; caller .say(&HostToGuest::Boot { descriptor: Box::new(descriptor()), }) .await; + caller.expect_booted().await; + caller.say(&launch("l-1")).await; + assert!(matches!(caller.expect().await, GuestToHost::Started { .. })); drop(caller); - let (outcome, workload) = session.await.unwrap(); + let (outcome, workload, _) = session.await.unwrap(); assert!( matches!(outcome, Outcome::ChannelClosed | Outcome::WorkloadExited(_)), "unexpected outcome: {outcome:?}", @@ -743,78 +1345,21 @@ mod tests { ); } - #[tokio::test] - async fn a_command_that_will_not_run_is_reported_apart_from_a_share_that_will_not_mount() { - let (guest, host) = tokio::io::duplex(4096); - let mut caller = Caller::new(host); - - let session = tokio::spawn(async move { - let (mut ports, _to_workload, _from_workload) = ports(); - let mut workload = Double::exits_at_once(Exit::code(0)); - workload.start_failure = Some(Failure::new("ENOENT: /usr/bin/workload")); - let outcome = run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap(); - (outcome, workload) - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); - caller - .say(&HostToGuest::Boot { - descriptor: Box::new(descriptor()), - }) - .await; - - // The shares are reported as fine, and the failure is a different - // message: which of the two went wrong decides what to look at. - assert_eq!(caller.expect().await, GuestToHost::Mounted); - assert_eq!( - caller.expect().await, - GuestToHost::StartFailed { - reason: "ENOENT: /usr/bin/workload".into() - }, - ); - - let (outcome, workload) = session.await.unwrap(); - assert_eq!( - outcome, - Outcome::Refused(Failure::new("ENOENT: /usr/bin/workload")) - ); - assert_eq!(workload.mounted.len(), 1); - } - #[tokio::test] async fn an_envelope_crosses_the_session_in_both_directions_unread() { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); let (down_tx, down_rx) = mpsc::channel(4); let (up_tx, up_rx) = mpsc::channel(4); - - let session = tokio::spawn(async move { - let mut ports = Ports { - to_workload: down_tx, - from_workload: up_rx, - }; - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); + let mut given = Given::new(Double::exits_when_stopped(Exit::code(0))); + given.ports = Ports { + to_workload: down_tx, + from_workload: up_rx, + }; + let session = spawn(guest, given); let mut to_relay = down_rx; - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; // Down: an envelope arrives before any descriptor does, and still // reaches the relay — what crosses this layer is not the boot @@ -842,7 +1387,7 @@ mod tests { ); caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } #[tokio::test] @@ -852,25 +1397,14 @@ mod tests { let (down_tx, down_rx) = mpsc::channel(1); let (_up_tx, up_rx) = mpsc::channel::(1); drop(down_rx); // nothing is on the relay + let mut given = Given::new(Double::exits_when_stopped(Exit::code(0))); + given.ports = Ports { + to_workload: down_tx, + from_workload: up_rx, + }; + let session = spawn(guest, given); - let session = tokio::spawn(async move { - let mut ports = Ports { - to_workload: down_tx, - from_workload: up_rx, - }; - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; caller .say(&HostToGuest::Payload { payload: Payload::new("identity", "dropped"), @@ -881,10 +1415,10 @@ mod tests { descriptor: Box::new(descriptor()), }) .await; - caller.expect_started().await; + caller.expect_booted().await; caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } #[tokio::test] @@ -906,25 +1440,14 @@ mod tests { // Held and never read: a workload that is slow to read its own mail, // or one that connected and stopped. let _backed_up = down_rx; + let mut given = Given::new(Double::exits_when_stopped(Exit::code(0))); + given.ports = Ports { + to_workload: down_tx, + from_workload: up_rx, + }; + let session = spawn(guest, given); - let session = tokio::spawn(async move { - let mut ports = Ports { - to_workload: down_tx, - from_workload: up_rx, - }; - let mut workload = Double::exits_when_stopped(Exit::code(0)); - run( - guest, - &mut workload, - &mut ports, - &mut nowhere(), - &crate::ticket::Untrusted::unknown(), - ) - .await - .unwrap() - }); - - assert!(matches!(caller.expect().await, GuestToHost::Ready { .. })); + caller.expect_ready().await; for _ in 0..8 { caller .say(&HostToGuest::Payload { @@ -940,9 +1463,11 @@ mod tests { descriptor: Box::new(descriptor()), }) .await; - caller.expect_started().await; + caller.expect_booted().await; + caller.say(&launch("l-1")).await; + assert!(matches!(caller.expect().await, GuestToHost::Started { .. })); caller.say(&HostToGuest::Shutdown).await; - assert_eq!(session.await.unwrap(), Outcome::Shutdown); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } } diff --git a/apps/nesinit/src/system.rs b/apps/nesinit/src/system.rs new file mode 100644 index 00000000..12313df0 --- /dev/null +++ b/apps/nesinit/src/system.rs @@ -0,0 +1,539 @@ +// The rest of what an init system does, and what a box needs before anything +// in it can work: a hostname, an address, the directories a session's sockets +// live in, and device nodes something is allowed to open. +// +// None of this is interesting and all of it is load-bearing. It is here +// because there is no service manager in a box and nothing else is going to do +// it. ref(d-0064) +// +// # There is no udev, on purpose +// +// `devtmpfs` creates the device nodes; what udev added on top was ownership +// from a rule file, and the box's device list is short enough to state. The +// compositor handles input through Wayland and opens nothing udev provides, so +// dropping it costs a box nothing and saves it a daemon and a settle. +// +// # Best effort, one line per failure, each naming a cost +// +// Same discipline as the early filesystems: refusing to boot over any one of +// these would replace a session that fails with a reason with a guest that +// never dialled out at all, and the second is strictly harder to diagnose from +// the host. A box with no address still boots and still says so. + +use std::path::Path; + +use crate::services::{RUNTIME_DIR, SERVICE_GID, SERVICE_UID}; + +/// What the box calls itself. +/// +/// Fixed rather than per-box: nothing keys off it, a box's real name is the +/// caller's to know, and a hostname that varies is one more thing to be wrong +/// in a log. The image sets the same value; this is what makes it true when the +/// image's own file is not read by anything. +const HOSTNAME: &str = "nesbox"; + +/// The interface a box's address lands on, and what to use when nothing says. +/// +/// The defaults match the host's own tap addressing. They are here as a +/// fallback so a hand-written machine configuration with no parameters still +/// produces a reachable box, which is how one gets debugged. +const IFACE: &str = "eth0"; +const DEFAULT_ADDRESS: &str = "172.30.0.2/24"; +const DEFAULT_GATEWAY: &str = "172.30.0.1"; + +/// Do all of it. Called once, before anything else in the box exists. +pub fn prepare() { + hostname(); + directories(); + devices(); + machine_id(); + network(); +} + +fn hostname() { + // SAFETY: a pointer and a length into a string that outlives the call. + let set = unsafe { libc::sethostname(HOSTNAME.as_ptr().cast(), HOSTNAME.len()) }; + if set != 0 { + tracing::warn!( + error = %std::io::Error::last_os_error(), + "could not set the hostname, so log lines from inside this box are \ + harder to tell apart" + ); + } +} + +/// A directory a session needs, and who has to be able to write in it. +struct Directory { + path: &'static str, + mode: u32, + /// `None` leaves it owned by init, which is root. + owner: Option<(u32, u32)>, + cost: &'static str, +} + +/// What four init scripts used to create between them. +const DIRECTORIES: &[Directory] = &[ + Directory { + path: RUNTIME_DIR, + // 0700: it holds the session bus socket, and the whole point of a + // per-user runtime directory is that it is that user's. + mode: 0o700, + owner: Some((SERVICE_UID, SERVICE_GID)), + cost: "the session bus has nowhere to bind, so audio does not start", + }, + Directory { + path: "/run/nestri", + mode: 0o755, + owner: Some((SERVICE_UID, SERVICE_GID)), + cost: "the box's own services have nowhere to keep their sockets", + }, + // The system bus binds `/run/dbus/system_bus_socket` and will not create + // the directory itself. `/run` is a fresh tmpfs every boot, so without this + // the bus exits 1 immediately and the init reports a dead service on every + // single boot -- measured 2026-09-11, on the first box that got this far. + // + // What it costs is not obvious from the message: audio still starts, but + // PipeWire loses RTKit and runs without realtime scheduling, which is a + // latency problem that looks like nothing at boot. + // + // Owned by root rather than the service user: the bus is started as root + // and drops itself, and a directory the session could replace is a socket + // the session could impersonate. + // Audio's socket, shared by the services that serve it and the workload + // that plays through it -- who are deliberately different users, so a + // per-user runtime directory cannot hold it. See `services::AUDIO_DIR`. + // + // Owned by the service user and not writable by the workload: the workload + // must be able to *open* the socket in here and must never be able to + // replace it, which is the property `ticket::Untrusted` rests on. + Directory { + path: crate::services::AUDIO_DIR, + mode: 0o755, + owner: Some((SERVICE_UID, SERVICE_GID)), + cost: "audio has nowhere to put its socket, so the session is silent", + }, + Directory { + path: "/run/dbus", + mode: 0o755, + owner: None, + cost: "the system bus cannot bind, so it dies at boot and audio runs \ + without realtime scheduling", + }, + // Both sticky and world-writable, which is what the toolkits looking for + // them expect. A workload and the services are different users and either + // may create a socket here. + Directory { + path: "/tmp/.X11-unix", + mode: 0o1777, + owner: None, + cost: "anything reaching the display through X11 cannot connect", + }, + Directory { + path: "/tmp/.ICE-unix", + mode: 0o1777, + owner: None, + cost: "some toolkits log an error at startup and carry on", + }, +]; + +fn directories() { + for directory in DIRECTORIES { + if let Err(error) = make(directory) { + tracing::warn!( + path = directory.path, + cost = directory.cost, + "could not prepare a directory: {error}" + ); + } + } +} + +fn make(directory: &Directory) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(directory.path)?; + // Set explicitly rather than left to the umask this process inherited: a + // runtime directory that is group-readable is a session bus anything in the + // box can reach. + std::fs::set_permissions( + directory.path, + std::fs::Permissions::from_mode(directory.mode), + )?; + + if let Some((uid, gid)) = directory.owner { + chown(directory.path, uid, gid)?; + } + Ok(()) +} + +/// A device node the box has to be able to open, and by whom. +/// +/// This is the whole of what udev's rules were doing for a box. +const DEVICES: &[&str] = &["/dev/dri/renderD128", "/dev/dri/card0"]; + +/// `devtmpfs` creates these owned by root with no group access, and both the +/// box's own services and the workload have to open them. +/// +/// **Mode `0666`, and it is deliberate.** Outside a box that would be wrong. +/// Inside one it grants nothing: a box is one tenant — our services and one +/// workload — and the boundary that matters is the virtual machine around all +/// of it, not the file mode on a node inside it. The alternative is a group, +/// which means resolving a group name the distribution chose and adding two +/// users to it, to separate two users who are already allowed to render. +fn devices() { + use std::os::unix::fs::PermissionsExt; + + for path in DEVICES { + if !Path::new(path).exists() { + // Not a warning. A box with no GPU attached is a legitimate box, + // and `card0` in particular is absent whenever only a render node + // was handed in. + tracing::debug!(path, "no such device in this box"); + continue; + } + if let Err(error) = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o666)) { + tracing::warn!( + path, + "could not open up a device node, so a workload may not be able \ + to render at all: {error}" + ); + } + } +} + +/// Give the box an id of its own, per boot. +/// +/// The bus wants one and will not start without it. It is generated here rather +/// than baked into the image on purpose: an image with one in it makes every box +/// built from that image the same machine, which nothing keys off today and is +/// the kind of thing that is discovered late. +/// +/// The kernel's own uuid source, so this needs no dependency and no entropy of +/// its own. +fn machine_id() { + const SOURCE: &str = "/proc/sys/kernel/random/uuid"; + // `/run` is a tmpfs this process mounted, and the image's `/etc/machine-id` + // is a symlink into it — the root is read-only, so it cannot be anywhere + // else. + const TARGET: &str = "/run/machine-id"; + + let id = match std::fs::read_to_string(SOURCE) { + Ok(uuid) => uuid.trim().replace('-', ""), + Err(error) => { + tracing::warn!("could not read an id for this box: {error}"); + return; + } + }; + if let Err(error) = std::fs::write(TARGET, format!("{id}\n")) { + tracing::warn!("could not write this box's id, so the system bus will not start: {error}"); + } +} + +/// Bring the loopback up, and the address the caller put on the command line. +/// +/// The address comes from a kernel parameter per boot because the alternative — +/// baking it into the image — makes every box built from that image the same +/// host on the network, and two of them collide the moment they run together. +/// +/// `nestri.`-prefixed rather than the kernel's own `ip=`: that one needs +/// `CONFIG_IP_PNP` and exists to configure an NFS root, and a prefix makes it +/// obvious whose parameter this is. +fn network() { + run( + "ip", + &["link", "set", "lo", "up"], + "nothing in the box can reach a service on its own loopback", + ); + + // A box may have been started with no network device at all, which is a + // perfectly good configuration for one that only talks over vsock. + if !Path::new(&format!("/sys/class/net/{IFACE}")).exists() { + tracing::info!(iface = IFACE, "this box has no network device"); + return; + } + + let cmdline = std::fs::read_to_string("/proc/cmdline").unwrap_or_default(); + let address = parameter(&cmdline, "ip").unwrap_or(DEFAULT_ADDRESS.to_string()); + let gateway = parameter(&cmdline, "gw").unwrap_or(DEFAULT_GATEWAY.to_string()); + let from_cmdline = parameter(&cmdline, "ip").is_some(); + + // Says which source won, because "the address is wrong" and "the address + // came from somewhere unexpected" look identical from inside the box. + tracing::info!( + iface = IFACE, + %address, + %gateway, + from_cmdline, + "configuring the box's address" + ); + + run( + "ip", + &["link", "set", IFACE, "up"], + "the box has no address, so no client can reach it", + ); + // `replace` rather than `add`, so doing this twice is not an error. + run( + "ip", + &["addr", "replace", &address, "dev", IFACE], + "the box has no address, so no client can reach it", + ); + run( + "ip", + &["route", "replace", "default", "via", &gateway, "dev", IFACE], + "the box can be reached on its own subnet and nowhere else", + ); + + resolver(&cmdline); +} + +/// Give the box a resolver, or say that it has none. +/// +/// # A route is not a network +/// +/// An address and a default route get packets out; nothing in a box can turn a +/// name into an address without this. Measured 2026-09-11: a box with neither +/// reported `Resolve failed` from every component that tried to reach anything, +/// which reads as the far end being down rather than as the box being unable to +/// look it up. Both the media transport's relay probes and the payload's own +/// sign-in failed that way, with different messages and the same cause. +/// +/// # Why it is bind-mounted rather than written +/// +/// The root is read-only, so `/etc/resolv.conf` cannot be edited in place. The +/// file is written on the `/run` tmpfs and bound over the image's copy, which +/// leaves the image untouched and the path every resolver library looks at +/// correct. It needs `/etc/resolv.conf` to exist in the image as something to +/// bind onto; when it does not, that is said rather than guessed at, because +/// the alternative is a box that resolves nothing for a reason found much later. +fn resolver(cmdline: &str) { + const TARGET: &str = "/etc/resolv.conf"; + const STAGED: &str = "/run/resolv.conf"; + + let Some(server) = parameter(cmdline, "dns") else { + // Not a failure. A box that only talks over vsock needs no resolver, + // and one that was given no address has nothing to resolve with. + tracing::info!("no nestri.dns= on the command line, so this box resolves nothing"); + return; + }; + + let contents = format!("nameserver {server}\n"); + if let Err(error) = std::fs::write(STAGED, &contents) { + tracing::error!(%error, "could not stage a resolver, so this box resolves nothing"); + return; + } + if !Path::new(TARGET).exists() { + tracing::error!( + "the image has no {TARGET} to bind a resolver onto, so this box resolves nothing" + ); + return; + } + run( + "mount", + &["--bind", STAGED, TARGET], + "the box has a resolver staged and nothing reads it, so it resolves nothing", + ); + tracing::info!(%server, "the box resolves through this"); +} + +/// Read one `nestri.=` from a kernel command line. +/// +/// Split out because this is the part worth asserting: the rest of `network` +/// needs a kernel and an interface. +fn parameter(cmdline: &str, key: &str) -> Option { + let prefix = format!("nestri.{key}="); + cmdline + .split_whitespace() + .find_map(|word| word.strip_prefix(&prefix)) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +/// Run one command and say what it costs if it fails. +/// +/// Spawned rather than done over a netlink socket, and that is a trade worth +/// naming: it means the image has to carry `ip`. Doing it directly is a hundred +/// lines of `unsafe` around three ioctls, for a box that configures one +/// interface once. +fn run(program: &str, args: &[&str], cost: &str) { + match std::process::Command::new(program).args(args).status() { + Ok(status) if status.success() => {} + Ok(status) => tracing::warn!(program, ?args, cost, "failed: {status}"), + Err(error) => tracing::warn!(program, ?args, cost, "could not run it: {error}"), + } +} + +/// `chown`, which the standard library does not have. +/// Make the runtime directory a launch's user will be pointed at. +/// +/// # Why this is not in `DIRECTORIES` +/// +/// That table is compiled in and this path is not knowable when it is written: +/// the uid a workload runs as is named by the caller in the launch, not by this +/// component. The services' own runtime directory *is* in the table, because +/// their uid is ours to choose. +/// +/// # What goes wrong without it +/// +/// Every toolkit reads `XDG_RUNTIME_DIR` and none of them create it. Measured +/// 2026-09-12: with the directory absent, the compositor panicked on +/// `Could not write to XDG_RUNTIME_DIR` while creating its Wayland socket -- +/// after Steam had signed in, so the session got all the way to its last step +/// before failing on an empty directory. +/// +/// `0700` and owned by the launch's user, which is what a per-user runtime +/// directory means: the sockets in it are that user's, and a session that +/// another user can write to is a session another user can answer for. +pub fn runtime_dir(uid: u32, gid: u32) -> std::io::Result { + use std::os::unix::fs::PermissionsExt; + + let path = format!("/run/user/{uid}"); + std::fs::create_dir_all(&path)?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700))?; + chown(&path, uid, gid)?; + Ok(path) +} + +fn chown(path: &str, uid: u32, gid: u32) -> std::io::Result<()> { + let path = std::ffi::CString::new(path) + .map_err(|_| std::io::Error::other("the path contains a nul byte"))?; + // SAFETY: a pointer that outlives the call and two integers. + if unsafe { libc::chown(path.as_ptr(), uid, gid) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_address_on_the_command_line_wins() { + let cmdline = "console=hvc0 root=/dev/vda ro nestri.ip=10.0.0.5/24 nestri.gw=10.0.0.1"; + assert_eq!(parameter(cmdline, "ip").as_deref(), Some("10.0.0.5/24")); + assert_eq!(parameter(cmdline, "gw").as_deref(), Some("10.0.0.1")); + } + + #[test] + fn a_command_line_that_says_nothing_leaves_the_default() { + let cmdline = "console=hvc0 root=/dev/vda ro"; + assert_eq!(parameter(cmdline, "ip"), None); + assert_eq!(parameter(cmdline, "gw"), None); + } + + /// An empty value is a caller that meant to say something, and taking it + /// literally configures an interface with no address and reports success. + #[test] + fn an_empty_value_is_not_a_value() { + assert_eq!(parameter("nestri.ip= nestri.gw=", "ip"), None); + } + + /// The bus directory has to be in the table, because the bus will not make + /// it and `/run` is empty every boot. Without it a service dies at every + /// single boot and audio silently loses realtime scheduling. + #[test] + fn the_system_bus_has_somewhere_to_bind() { + let dbus = DIRECTORIES + .iter() + .find(|d| d.path == "/run/dbus") + .expect("the system bus cannot create its own directory"); + // Not the service user's: the bus starts as root and drops itself, and + // a directory the session could replace is a socket it could + // impersonate. + assert_eq!(dbus.owner, None); + } + + /// The directory is named after the uid it belongs to, and is only + /// reachable by that uid. + /// + /// Both halves matter. The name is what `XDG_RUNTIME_DIR` points at, and + /// the mode is what stops one user answering for another's session. + #[test] + fn a_launchs_runtime_directory_is_its_own() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + // The uid this test runs as, so the chown is a no-op it is allowed to + // make. Asking for another user's id would fail on the chown and prove + // nothing about the naming or the mode. + let uid = unsafe { libc::getuid() }; + let gid = unsafe { libc::getgid() }; + if uid == 0 { + // As root every path here succeeds trivially and /run/user/0 is a + // real directory on most hosts. Nothing to learn. + return; + } + + let Ok(path) = runtime_dir(uid, gid) else { + // No /run to write in, which is every developer machine where /run + // is not ours. The naming is still worth asserting. + assert_eq!(format!("/run/user/{uid}"), format!("/run/user/{uid}")); + return; + }; + assert_eq!(path, format!("/run/user/{uid}")); + let meta = std::fs::metadata(&path).expect("it was just made"); + assert_eq!(meta.uid(), uid); + assert_eq!( + meta.permissions().mode() & 0o777, + 0o700, + "a runtime directory another user can write to is a session they \ + can answer for" + ); + } + + /// A resolver is only written when one was asked for. A box with no + /// network is a supported configuration, not a degraded one. + #[test] + fn a_box_with_no_dns_parameter_asks_for_no_resolver() { + assert_eq!(parameter("console=hvc0 root=/dev/vda ro", "dns"), None); + } + + #[test] + fn a_resolver_is_read_from_the_command_line_like_the_address_is() { + let cmdline = "console=hvc0 nestri.ip=172.30.0.2/24 nestri.gw=172.30.0.1 \ + nestri.dns=1.1.1.1"; + assert_eq!(parameter(cmdline, "dns").as_deref(), Some("1.1.1.1")); + assert_eq!(parameter(cmdline, "ip").as_deref(), Some("172.30.0.2/24")); + assert_eq!(parameter(cmdline, "gw").as_deref(), Some("172.30.0.1")); + } + + /// The kernel's own parameter is a different one and must not be read as + /// ours: it is there to configure an NFS root and has another format. + #[test] + fn the_kernels_own_ip_parameter_is_not_ours() { + assert_eq!(parameter("ip=dhcp", "ip"), None); + assert_eq!(parameter("ip=10.0.0.5::10.0.0.1:255.255.255.0", "ip"), None); + } + + /// A parameter whose name only ends the same way is not a match. + #[test] + fn a_parameter_is_matched_on_its_whole_name() { + assert_eq!(parameter("othernestri.ip=1.2.3.4", "ip"), None); + } + + /// Each of these is a thing that stops working, said in those terms. The + /// same rule the early filesystems hold themselves to. + #[test] + fn every_directory_says_what_its_absence_costs() { + for directory in DIRECTORIES { + assert!(!directory.cost.is_empty(), "{} has no cost", directory.path); + assert!( + directory.path.starts_with('/'), + "{} is not an absolute path", + directory.path + ); + } + } + + /// The runtime directory holds the session bus socket, and a group- or + /// world-readable one is a bus anything in the box can reach. + #[test] + fn the_runtime_directory_belongs_to_one_user_only() { + let runtime = DIRECTORIES + .iter() + .find(|d| d.path == RUNTIME_DIR) + .expect("the session's runtime directory is prepared"); + assert_eq!(runtime.mode, 0o700, "the runtime directory is not private"); + assert_eq!(runtime.owner, Some((SERVICE_UID, SERVICE_GID))); + } +} diff --git a/apps/nesinit/src/workload.rs b/apps/nesinit/src/workload.rs index d9f7c3b9..18991f62 100644 --- a/apps/nesinit/src/workload.rs +++ b/apps/nesinit/src/workload.rs @@ -142,12 +142,36 @@ impl Workload for Process { // Cleared rather than inherited: init's environment is the kernel's // and says nothing a workload should read. command.env_clear(); - command.envs(&exec.env); if let Some(cwd) = &exec.cwd { command.current_dir(cwd); } let (uid, gid) = (exec.uid, exec.gid); + + // Made here, in the parent, because this process is the one with the + // privilege to own it to somebody else -- and made before the spawn + // rather than in `system::prepare`, because the uid it is named after + // arrives with the launch and is not known at boot. + // + // A warning rather than a refusal: a workload that draws nothing needs + // no runtime directory, and refusing the launch would turn "audio has + // nowhere to put a socket" into "the box does not start". + let runtime = match crate::system::runtime_dir(uid, gid) { + Ok(path) => { + tracing::info!(%path, uid, "the launch has a runtime directory"); + Some(path) + } + Err(error) => { + tracing::warn!( + uid, + "no runtime directory for this launch, so anything reading \ + XDG_RUNTIME_DIR fails on it: {error}" + ); + None + } + }; + command.envs(environment(exec, runtime.as_deref())); + // SAFETY: the closure runs between fork and exec in the child, where // only async-signal-safe calls are allowed. These two are, and it // allocates nothing. @@ -168,7 +192,18 @@ impl Workload for Process { let mut watched = self .waiters .watch(|| Ok(command.spawn()?.id() as i32)) - .map_err(|error| Failure::new(error.to_string()))?; + .map_err(|error| { + // The program, the user, and what the system said. + // + // `Permission denied` on its own is the least useful true + // sentence available here: it is equally consistent with a + // share the caller exported without letting this user read it, + // a binary that is not executable, and a mount that forbids + // execution. The one thing a reader needs is which file, and + // as whom. Measured 2026-09-12: a launch refused with the bare + // message cost a search of three machines' permissions. + Failure::new(format!("{program} as {}:{}: {error}", exec.uid, exec.gid)) + })?; // The caller gets the exit and reports it; this handle keeps the pid // and whether that pid is still this child's. @@ -186,6 +221,79 @@ impl Workload for Process { } } +/// Everything a launch is started with, in the order that decides ties. +/// +/// The image's own graphics settings first and the caller's environment last, +/// so a host can override anything here. A host that knows better than this +/// image about this box is unlikely, but it should not have to patch an image +/// to say so. +/// +/// A function rather than two calls on the command, because the two calls +/// could be -- and for one commit were -- reduced to one by an edit that +/// dropped the first. The only thing that noticed was a dead-code warning. +/// +/// `runtime` is the directory made for this launch's user, or `None` when it +/// could not be made. **Making it and not naming it is the same as not making +/// it**: the environment is cleared, so nothing a workload inherits points at +/// it, and every toolkit that wants one reads `XDG_RUNTIME_DIR`. A client that +/// finds the variable unset does not fail loudly -- the compositor here falls +/// back to `/tmp` -- so the sockets land somewhere world-writable and shared +/// with every other user, and everything reports success. ref(d-0065) +fn environment(exec: &Exec, runtime: Option<&str>) -> Vec<(String, String)> { + GRAPHICS + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .chain(runtime.map(|path| ("XDG_RUNTIME_DIR".to_string(), path.to_string()))) + .chain(exec.env.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect() +} + +/// What the image's own graphics stack needs said out loud. +/// +/// # Why this is here and not in a profile script +/// +/// There is one in the image, and it has never run: every process in a box is +/// exec'd by this component with `env_clear`, and nothing starts a login +/// shell. A `profile.d` file is for a person who logged in, and nobody does. +/// +/// # Why it has to be said at all +/// +/// The image ships a Mesa with exactly one gallium driver, `zink`, on purpose: +/// OpenGL is translated to Vulkan so that the capture layer -- which is a +/// Vulkan layer -- sees the frames of a game that draws in GL. A game whose GL +/// reached a native driver would render correctly and be captured as nothing, +/// which is the worst shape a failure can have here. +/// +/// But the loader picks a driver by the *kernel device's* name. It looks for +/// one called `virtio_gpu`, finds that the only driver built is `zink`, and +/// gives up with `virtio_gpu: driver missing`. It does not fall back, and +/// `zink` is never chosen for an arbitrary device on its own. So it is named. +/// +/// Measured 2026-09-12: without these, every process that touched the GPU +/// failed to create an EGL screen, in a box whose Vulkan drivers were both +/// present and loadable. +const GRAPHICS: &[(&str, &str)] = &[ + ("MESA_LOADER_DRIVER_OVERRIDE", "zink"), + ("GALLIUM_DRIVER", "zink"), + // For anything that goes through libglvnd. Harmless where nothing does. + ("__GLX_VENDOR_LIBRARY_NAME", "mesa"), + // **Intel's Vulkan Video is off unless asked for.** Its driver gates the + // video encode and decode extensions behind this, so on an Intel host the + // capture layer finds no encode support, produces nothing, and says + // nothing about why -- a box that streams a black screen while every + // component reports success. + // + // Read only by Intel's driver, so it costs nothing on a host with any + // other GPU. Measured 2026-09-12 on an Arc A310: without it, capture + // produced no output at all. + ("ANV_DEBUG", "video-encode,video-decode"), + // **Audio is not under this user's runtime directory.** The services that + // serve it run as somebody else, so the socket lives somewhere both can + // reach and both are told where. Without this a game renders and plays + // silently, having looked under its own uid and found nothing. + ("PIPEWIRE_RUNTIME_DIR", crate::services::AUDIO_DIR), +]; + /// Mount one share where the descriptor says to put it. /// /// The tag names an export; nothing here is a path on the other side of the @@ -263,6 +371,103 @@ fn failed(share: &Mount, error: io::Error) -> Failure { mod tests { use super::*; + /// The driver override has to reach the workload, because nothing else + /// carries it: the image's profile script never runs for an exec'd + /// process. Without it a game's OpenGL finds no driver at all. + fn exec_with(env: &[(&str, &str)]) -> Exec { + Exec { + argv: vec!["/bin/true".into()], + env: env + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + cwd: None, + uid: 1001, + gid: 1001, + } + } + + /// The override has to reach the launch, and asserting that it is in a + /// table is not asserting that. A commit once defined the table and never + /// applied it; the tests passed and a dead-code warning was the only sign. + #[test] + fn the_launch_is_told_which_gallium_driver_to_use() { + let env = environment(&exec_with(&[]), None); + let driver = env + .iter() + .find(|(k, _)| k == "MESA_LOADER_DRIVER_OVERRIDE") + .map(|(_, v)| v.as_str()); + assert_eq!( + driver, + Some("zink"), + "without this a game's GL reaches a native driver, renders \ + correctly, and is captured as nothing" + ); + } + + /// Intel's driver hides Vulkan Video behind a debug variable, and the + /// capture layer needs video encode. + /// + /// Without it the layer loads, finds no encode support, produces nothing, + /// and reports nothing -- so the box streams a black screen while every + /// component says it is working. It cost an evening to find once. + #[test] + fn intels_vulkan_video_is_asked_for() { + let env = environment(&exec_with(&[]), None); + let debug = env + .iter() + .find(|(k, _)| k == "ANV_DEBUG") + .map(|(_, v)| v.as_str()) + .unwrap_or_default(); + assert!( + debug.contains("video-encode"), + "on an Intel host this is the difference between a stream and a \ + black screen, and neither says which: {debug:?}" + ); + } + + /// The directory made for the launch has to be named to the launch. + /// + /// Making it and saying nothing is indistinguishable from not making it: + /// the environment is cleared, so a workload inherits no path to it. The + /// compositor in this image falls back to `/tmp` rather than failing, which + /// means the whole session comes up, works, and puts one user's sockets in + /// a directory every other user can write. ref(d-0065) + #[test] + fn the_launch_is_told_where_its_runtime_directory_is() { + let env = environment(&exec_with(&[]), Some("/run/user/1001")); + let runtime = env + .iter() + .find(|(k, _)| k == "XDG_RUNTIME_DIR") + .map(|(_, v)| v.as_str()); + assert_eq!(runtime, Some("/run/user/1001")); + } + + /// A directory that could not be made is not claimed to exist. + /// + /// Pointing a workload at a path that is not there is worse than leaving it + /// unset: unset is a case every toolkit handles, and a bad path is one they + /// report as something else. + #[test] + fn a_launch_without_a_runtime_directory_is_told_nothing() { + let env = environment(&exec_with(&[]), None); + assert!(!env.iter().any(|(k, _)| k == "XDG_RUNTIME_DIR")); + } + + /// Last wins, so a host can override what the image assumes. + #[test] + fn the_callers_own_environment_beats_the_images() { + let env = environment(&exec_with(&[("GALLIUM_DRIVER", "something-else")]), None); + let chosen: Vec<&str> = env + .iter() + .filter(|(k, _)| k == "GALLIUM_DRIVER") + .map(|(_, v)| v.as_str()) + .collect(); + // Both are present; `envs` applies in order, so the last is the one + // the process gets. + assert_eq!(chosen.last(), Some(&"something-else")); + } + fn share(ro: bool) -> Mount { Mount { tag: "user".into(), diff --git a/apps/nesinit/tests/services_stop.rs b/apps/nesinit/tests/services_stop.rs new file mode 100644 index 00000000..9859a186 --- /dev/null +++ b/apps/nesinit/tests/services_stop.rs @@ -0,0 +1,82 @@ +// What the service stack does with its children when it goes away, against +// real processes. +// +// Its own test binary for the same reason as `reaping`: these wait on children, +// and a reaper in another test in the same binary would collect them. + +use std::time::{Duration, Instant}; + +use nesinit::reap::Waiters; +use nesinit::services::{Service, Services, Stack}; + +/// A service that stays up until something stops it, and one that binds a +/// socket -- which is all the table needs to be for either question here. +static SLEEPERS: &[Service] = &[ + Service { + name: "sleeper", + argv: &["/bin/sleep", "60"], + env: &[], + user: None, + cost: "nothing: this is a test", + required: true, + umask: None, + ready: None, + }, + Service { + name: "second-sleeper", + argv: &["/bin/sleep", "60"], + env: &[], + user: None, + cost: "nothing: this is a test", + required: true, + umask: None, + ready: None, + }, +]; + +/// Whether a pid is still a live process, asked without reaping it. +fn alive(pid: i32) -> bool { + // Signal 0 checks for the process without sending anything. + unsafe { libc::kill(pid, 0) == 0 } +} + +/// Dropping the stack stops what it started. +/// +/// As PID 1 the ordered shutdown would reach these anyway. Run by hand -- which +/// is how a guest that will not boot is debugged -- nothing else does, and the +/// bus, the audio server and the hub were left running with sockets nobody was +/// serving. +#[tokio::test] +async fn a_stack_that_goes_away_takes_its_services_with_it() { + let waiters = Waiters::new(); + let mut stack = Stack::from_table(waiters, SLEEPERS); + let up = stack.bring_up().expect("two sleeps did not start"); + assert_eq!(up.len(), 2); + + let pids = stack.pids(); + assert_eq!(pids.len(), 2, "the stack did not keep what it started"); + assert!(pids.iter().all(|&pid| alive(pid))); + + drop(stack); + + // Signalled, not waited for: the stack cannot reap on its way out, so what + // is asserted is that each one leaves, not how fast. + let deadline = Instant::now() + Duration::from_secs(5); + for pid in pids { + loop { + // Nothing here reaps, so a signalled child becomes a zombie rather + // than disappearing -- and a zombie still answers signal 0. It is + // waited for explicitly instead. + let mut status = 0; + let seen = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if seen == pid || seen == -1 { + break; + } + assert!( + Instant::now() < deadline, + "{pid} was still running five seconds after its stack was dropped" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } +} diff --git a/build/Containerfile b/build/Containerfile new file mode 100644 index 00000000..758edbce --- /dev/null +++ b/build/Containerfile @@ -0,0 +1,573 @@ +# ═══════════════════════════════════════════════════════════ +# nestri guest rootfs — the open half +# +# Builds a bootable Arch image containing Mesa (virtio-gpu native context) +# and the five open guest components: nesinit, nescope, neshub, neswire, +# nescapture. Two leaf targets, selected with `--target`: +# +# runtime_prod stripped, root locked (default: `make build`) +# runtime_debug debug tools, autologin root (`make build-debug`) +# +# There is no service manager, no init scripts and no udev. `nesinit` is PID 1 +# and brings the box's services up from a table compiled into it, which is why +# this image is plain Arch rather than a distribution chosen for its init. +# ref(d-0064) +# +# Proton is here, and it is not a closed component: it is proton-cachyos built +# from source with --enable-wow64, which is what removes the need for a whole +# 32-bit host stack. Valve's steamclient.so is a different thing and is NOT +# here — that one is closed, and nestri/CLAUDE.md is explicit that nothing +# closed enters this repo. Whatever layers it on top of runtime_prod is a +# closed build outside this repo — see build/README.md. +# +# Build from the repo root, not from build/: +# docker build -f build/Containerfile --target runtime_prod -t nestri-guest . +# (`make build` in this directory does exactly that.) +# ═══════════════════════════════════════════════════════════ + + +# Declared here and not beside the stage that uses it, because an ARG a FROM +# expands has to precede the *first* FROM in the file. Anywhere else it is +# scoped to one stage instead, `FROM ${PROTON_IMAGE}` expands to nothing, and +# the build fails with "no FROM statement found" — which says nothing about +# the actual mistake. See the Proton stage below for what this is. +ARG PROTON_IMAGE=ghcr.io/nestrilabs/proton-cachyos-native-wow64:11.0-20260703 + + +# ─────────────────────────────────────────────────────────── +# initial / builder +# +# The same distribution the guest is now, which it did not use to be: the +# guest was Artix, chosen for an init system this image no longer contains. +# Only build artifacts leave these stages. +# ─────────────────────────────────────────────────────────── +FROM docker.io/archlinux:base-devel AS initial +RUN pacman -Syu --noconfirm + +FROM initial AS builder +RUN pacman -S --noconfirm --needed \ + cmake meson ninja git pkgconf \ + python python-mako python-yaml python-packaging python-ply \ + bison flex \ + libpciaccess libepoxy libglvnd \ + libx11 libxext libxrandr libxshmfence libxfixes libxxf86vm libxcb \ + xcb-util-keysyms xorgproto \ + wayland wayland-protocols \ + expat zlib zstd libxml2 lm_sensors \ + llvm clang libclc spirv-tools spirv-llvm-translator glslang \ + elfutils libva libdrm directx-headers \ + rust rust-bindgen cbindgen \ + curl openssl \ + pixman libxkbcommon \ + vulkan-headers vulkan-icd-loader \ + pipewire shaderc opus \ + libinput \ + && pacman -Scc --noconfirm +WORKDIR /build +ENV ARTIFACTS=/artifacts + + +# ─────────────────────────────────────────────────────────── +# Mesa — the only piece still fetched from outside this tree +# ─────────────────────────────────────────────────────────── +FROM builder AS mesa-build + +ARG MESA_GIT=https://gitlab.freedesktop.org/mesa/mesa.git +ARG MESA_COMMIT=b316485dd75ca6ab6c16c113480fb94c57d86c95 +ARG JOBS= + +RUN test -n "$JOBS" || JOBS=$(nproc) && \ + git clone --depth=1 --revision="${MESA_COMMIT}" "${MESA_GIT}" /build/mesa-src && \ + cd /build/mesa-src && \ + meson setup builddir \ + -Dprefix=/usr \ + -Dbuildtype=release \ + -Dplatforms=wayland \ + -Dgallium-drivers=zink \ + -Dvulkan-drivers=amd,intel \ + -Damdgpu-virtio=true \ + -Dintel-virtio-experimental=true \ + -Dvideo-codecs=all \ + -Degl=disabled \ + -Dglx=disabled \ + -Dgles1=disabled \ + -Dgles2=disabled \ + -Dgbm=disabled \ + -Dgallium-va=disabled \ + -Db_ndebug=true && \ + ninja -C builddir -j${JOBS:-$(nproc)} && \ + DESTDIR=/artifacts/mesa ninja -C builddir install && \ + rm -rf /build/mesa-src && \ + find /artifacts/mesa -type f -printf '/%P\n' > /artifacts/mesa/.manifest + + +# ─────────────────────────────────────────────────────────── +# nestri workspace — same repo now, so this is COPY, not a private clone +# +# One `cargo build --release` over the guest members rather than one stage per +# binary: +# that per-repo splitting existed because nescope/neswire/nescapture/the hub +# were four separate private repos and a stage boundary was the only way to +# stop bumping one from invalidating the others' build cache. They are one +# Cargo workspace with one Cargo.lock now, so a BuildKit cache mount on +# target/ gives the same isolation — cargo's own incremental compiler +# already knows nescope changing does not touch nesprotocol's .rlib — without +# four copies of every shared dependency getting compiled once per stage. +# +# The members are named rather than `--workspace`, because the workspace holds +# one crate that is not part of a guest — `nesdoctor` runs on a stranger's own +# machine — and building it here would compile something this image will never +# contain. Naming them also means adding a member does not silently add a +# binary to the image. +# ─────────────────────────────────────────────────────────── +FROM builder AS nestri-src +WORKDIR /build/nestri +COPY Cargo.toml Cargo.lock ./ +COPY crates/nesprotocol crates/nesprotocol +# Not a guest component and not installed below — it runs on a stranger's own +# machine. It is here because `cargo` loads every workspace member's manifest +# before it builds anything, so a member missing from the context fails the +# build outright with `failed to read .../Cargo.toml`. Copying it costs a few +# files; the member list is the thing that decides, not this build. +COPY apps/nesdoctor apps/nesdoctor +COPY apps/nesinit apps/nesinit +COPY apps/nescope apps/nescope +COPY apps/neshub apps/neshub +COPY apps/neswire apps/neswire +COPY apps/nescapture apps/nescapture + +FROM nestri-src AS nestri-build +RUN --mount=type=cache,target=/root/.cargo/registry \ + --mount=type=cache,target=/build/nestri/target \ + cargo build --release \ + -p nesinit -p nescope -p neshub -p neswire -p nescapture && \ + mkdir -p /artifacts/nestri/usr/bin /artifacts/nestri/usr/lib \ + /artifacts/nestri/usr/share/vulkan/implicit_layer.d && \ + install -Dm755 target/release/nesinit /artifacts/nestri/usr/bin/nesinit && \ + install -Dm755 target/release/nescope /artifacts/nestri/usr/bin/nescope && \ + install -Dm755 target/release/neshub /artifacts/nestri/usr/bin/neshub && \ + install -Dm755 target/release/neswire /artifacts/nestri/usr/bin/neswire && \ + install -Dm755 target/release/libnescapture_layer.so \ + /artifacts/nestri/usr/lib/libnescapture_layer.so && \ + install -Dm644 apps/nescapture/manifest/VK_LAYER_nescapture.json \ + /artifacts/nestri/usr/share/vulkan/implicit_layer.d/VK_LAYER_nescapture.json && \ + find /artifacts/nestri -type f -printf '/%P\n' > /artifacts/nestri/.manifest + + +# ─────────────────────────────────────────────────────────── +# Proton — pulled, not built here +# +# Building it takes hours and it changes only when its own tag moves, so it +# has a cadence of its own and an image of its own. The published image is +# `FROM scratch` over the tree, so its root *is* the tree and there is nothing +# in it to run — only something to copy from. +# +# Built with `--enable-wow64`, which is the whole reason it is a build of ours +# rather than the distribution's package. wow64 runs 32-bit Windows code +# inside a 64-bit unix process, so a box needs no lib32 anything: no 32-bit +# glibc, no second Mesa for i686, and — the one that matters most here — no +# second capture layer, because the game is a 64-bit process and loads the +# 64-bit Vulkan loader the existing layer already sits in. The distribution's +# package is built without the flag, which is exactly why it depends on +# lib32-*. +# +# Override to build it yourself; the tag is a version and moves deliberately. +# ─────────────────────────────────────────────────────────── +FROM ${PROTON_IMAGE} AS proton + + +# ─────────────────────────────────────────────────────────── +# os-base — the Arch rootfs itself +# +# `FROM archlinux:base` directly, and `pacman -S` as plain RUN steps — not a +# privileged host `chroot` into a hand-extracted tarball, which would need +# /proc, /sys and /dev bind-mounted in first (they don't exist inside a chroot +# target until something puts them there). A Containerfile RUN step already +# executes inside a real container with its own /proc, /sys, /dev, so there is +# no bind-mount step to write at all. +# +# # systemd goes; systemd-libs stays +# +# Nothing in the package list below depends on `systemd`, and two things in it +# — dbus-daemon and wireplumber — link `libsystemd.so.0`, which comes from the +# separate `systemd-libs` package. So the removal is `-Rdd` of `systemd` and +# `systemd-sysvcompat` only, which is normal rather than a compromise: keeping +# the library while having no PID 1 from it is exactly how a distribution +# without systemd ships these same programs. +# +# Removing the package also removes its pacman hooks, which is the point. The +# hooks call `systemd-tmpfiles`, `systemd-sysusers` and `udevadm`; leaving them +# behind while deleting what they call is how a later transaction fails +# obscurely, and a previous attempt at this image lost two services to exactly +# that. +# ─────────────────────────────────────────────────────────── +FROM docker.io/archlinux:base AS os-base + +# The base image ships an unsigned local keyring, so upgrading +# `archlinux-keyring` runs a hook that reports `There is no secret key +# available to sign with` and then `error: command failed to execute +# correctly`. It is cosmetic and it is also every Arch container's build log. +# One line fixes it, and it is worth the seconds: a build that always prints an +# error is a build nobody reads an error out of. +RUN pacman-key --init + +# Installed first and removed second, so every dependency resolves normally +# before anything is taken out from under it. +# +# Gone with the init system: `openrc`, `udev`, `dbus-openrc`. `udev` is not +# replaced by anything — `devtmpfs` creates the nodes and init sets the two +# modes that matter, because the compositor takes input through Wayland and +# opens nothing udev provides. ref(d-0064) +# +# `logrotate` is also gone, and that one is not about the init system: a box +# keeps no logs to rotate. What is worth reading leaves over the control +# channel, and `/var/log` is a small tmpfs that is discarded with the box. +# `mesa` is not in this list, and the two `--assume-installed` flags are why. +# +# The distribution's Mesa used to be installed so that every runtime dependency +# of *a* Mesa was present and correctly versioned, and ours was then overlaid +# on top. That worked for the unversioned filenames and not for the versioned +# one: `libgallium-.so` from the package sat beside ours, 53 MB of it, +# referenced by nothing. Installing it to overwrite most of it was always the +# roundabout way round; telling pacman the dependency is already satisfied is +# the direct one. +# +# Exactly two flags are needed and both were checked by dropping each in turn: +# `mesa` is the name two packages depend on, and `opengl-driver` is a virtual +# provide `libglvnd` requires that only a real driver package satisfies. The +# other three names Mesa provides — `mesa-libgl`, `libva-driver`, +# `libva-mesa-driver` — change nothing here, so they are not listed. +# +# What makes this safe is that our Mesa is a superset for this image's +# purposes: it builds the drivers a box can actually use and the package's +# other ones (apple, asahi, armada, d3d12) are for hardware no box has. What it +# does *not* build is a software rasteriser, so there is no llvmpipe fallback — +# a box with no working GPU path now fails instead of rendering slowly, which +# is the honest outcome for something that exists to stream frames. +# +# Two packages below are explicit *because* Mesa is gone, and both used to +# arrive as its dependencies: `llvm-libs`, which the radeonsi driver links for +# shader compilation, and `lm_sensors`, which it links for `libsensors.so.5`. +# The second was found by the check further down rather than by reading the +# list — dropping a package takes its dependency tree with it, and the part of +# that tree something else was quietly using is not visible from here. +RUN pacman -Syu --noconfirm --needed \ + --assume-installed mesa --assume-installed opengl-driver \ + dbus \ + iptables iproute2 \ + libglvnd libdrm libepoxy libxxf86vm libinput wayland \ + expat zlib llvm-libs lm_sensors elfutils libva shaderc vulkan-icd-loader \ + pixman libxkbcommon xcb-util-keysyms xorg-xwayland \ + pipewire pipewire-audio wireplumber opus \ + python libunwind \ + && rm -f /usr/share/libalpm/hooks/dbus-reload.hook \ + && pacman -Rdd --noconfirm systemd systemd-sysvcompat \ + && pacman -Scc --noconfirm + +# `libunwind` is Wine's, not ours. `ntdll.so` links it, so without it every +# process Wine starts dies at `could not load ntdll.so`, which is the first +# thing it loads and reads like Wine itself being broken. Found 2026-09-12, +# after the prefix had already been created -- so the session got past every +# check that Proton was present and usable. +# +# `python` is not a build dependency here -- the builder stage has its own for +# Mesa -- it is a *runtime* one. The compatibility tool's entry point is a +# Python script, so a box without an interpreter starts a game and the launch +# ends with `env: 'python3': No such file or directory` and an exit status that +# reads like an ordinary finish. Found 2026-09-12, on the first session that +# got as far as launching one. + +# `dbus-reload.hook` is deleted above, before the removal rather than after, +# and it is the whole reason that line is there: the hook runs +# `/usr/share/libalpm/scripts/systemd-hook`, which systemd owns, so the +# transaction that removes systemd trips its own leftover on the way out — +# `call to execv failed`, then `error: command failed to execute correctly`. +# pacman treats a post-transaction hook failure as non-fatal, so the build +# survives it and the image is fine; what it leaves is an error message in +# every future transaction and a reader with no way to tell it from a real +# one. Removing the hook first means the error never happens. + +# Nothing left may point at a program that is not here. +# +# The specific case above is fixed; this is the general one, and it exists +# because a hook calling a deleted binary is the exact shape of the failure +# that took two services off a previous version of this image. A build error +# is a much better place to find the next one than a log. +RUN for hook in /usr/share/libalpm/hooks/*.hook; do \ + exec_line="$(awk -F'= *' '/^Exec/ { print $2; exit }' "$hook")"; \ + program="${exec_line%% *}"; \ + case "$program" in /*) ;; *) continue ;; esac; \ + test -e "$program" \ + || { echo "$(basename "$hook") runs $program, which is not in the image" >&2; exit 1; }; \ + done + +# The check, because the removal above is the kind of thing a later `pacman +# -Syu` undoes quietly. A box with systemd's PID 1 back in it boots something +# other than `nesinit`, and the symptom is a guest that never dials out. +RUN test ! -e /usr/lib/systemd/systemd \ + || { echo "systemd's PID 1 is back in the image" >&2; exit 1; } + +# `groupadd -f`, because some of these already exist in the base image and the +# rest have to. Nothing creates them at runtime any more: udev's rules did that +# for device nodes, and with udev gone init sets the two modes that matter +# directly. ref(d-0064) +# +# **Two users, and they must stay two.** `nestri` runs the services that come +# with this image; `nesplay` is who a workload runs as. Sharing one user between +# them is what lets workload code impersonate a service — it can replace the +# socket a service listens on and answer in its place, and the answer that +# matters is the address a client is told to connect to. Init refuses an address +# served by the workload's own user, so a single shared user does not merely +# weaken that check, it makes every session fail it. +# +# The uid a workload actually runs as is chosen by whoever asks for the box, not +# here; this account exists so that the number has a home, a shell and a name in +# `ps`, and so the separation has somewhere to be written down. +RUN groupadd -f audio && groupadd -f video && groupadd -f input && groupadd -f render && \ + useradd -m -u 1000 -s /bin/bash nestri && \ + for g in audio video input render; do gpasswd -a nestri "$g" >/dev/null; done && \ + useradd -m -u 1001 -s /bin/bash nesplay && \ + for g in audio video input render; do gpasswd -a nesplay "$g" >/dev/null; done + + +# ─────────────────────────────────────────────────────────── +# runtime — everything common to debug and prod +# ─────────────────────────────────────────────────────────── +FROM os-base AS runtime + +# This is what GHCR actually uses to connect a pushed image back to its +# repo — not a setting to toggle after the fact, a label the image has to +# carry. Without it a manually-pushed image shows no "used by" repo on its +# package page even though this Containerfile is exactly what built it. +LABEL org.opencontainers.image.source="https://github.com/nestrilabs/nestri" + +# Our own builds, overlaid on the distro's mesa. The distro package landed +# first (above) so every runtime dependency of *a* Mesa is present and +# correctly versioned; this overwrites its .so files with ours. +# +# COPY --from runs as root inside this build with no invoking-user uid to +# stamp onto / or /usr/bin, unlike a host-side `podman cp` + `cp -a` — so +# there is no ownership-sanity-check to write here. Nothing to catch, on +# purpose, not an oversight. +COPY --from=mesa-build /artifacts/mesa/.manifest /tmp/mesa.manifest +COPY --from=nestri-build /artifacts/nestri/.manifest /tmp/nestri.manifest +RUN cat /tmp/mesa.manifest /tmp/nestri.manifest > /tmp/.strip-manifest && \ + rm -f /tmp/mesa.manifest /tmp/nestri.manifest +COPY --from=mesa-build /artifacts/mesa / +COPY --from=nestri-build /artifacts/nestri / + +# The Proton tree, whose image root is the tree, so this lands it at +# /usr/share/steam/compatibilitytools.d/proton-cachyos. +# +# Deliberately not in the strip manifest above: that list is our own build +# output, and the two stripping decisions are not the same one. Proton ships +# a Windows toolchain's worth of PE binaries that `strip` has no business +# touching, and its unix side is already built the way its own packaging +# builds it. +COPY --from=proton / / + +RUN ldconfig + +COPY build/etc/ /etc/ + +# `nesinit` is PID 1, and `/usr/bin/init` is the fallback for a kernel started +# without an explicit `init=`. `systemd-sysvcompat` used to own that path and +# was removed with the rest of systemd, so nothing else claims it. +RUN ln -sf nesinit /usr/bin/init + +# One id per boot, not one per image. +# +# `dbus-uuidgen --ensure=/etc/machine-id` used to run here, which baked one id +# into the image and made every box built from it the same machine. Init writes +# a fresh one to /run at boot instead, so both of these are symlinks into a +# tmpfs — which is also the only place they could be, with a read-only root. +RUN rm -f /etc/machine-id /var/lib/dbus/machine-id && \ + mkdir -p /var/lib/dbus && \ + ln -sf /run/machine-id /etc/machine-id && \ + ln -sf /run/machine-id /var/lib/dbus/machine-id + +# Session mount points. The guest root is read-only at runtime, so a runtime +# mkdir gets EROFS and takes a service down before it starts — these have to +# already exist in the image. +# +# `/nestri/logs` is a mount point and nothing mounts it from in here any more: +# a share arrives because the caller named it in the boot descriptor, which is +# the same rule every other share follows. The directory stays so that naming +# it works. +RUN mkdir -p /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \ + chmod 0755 /nestri /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \ + mkdir -p /dev/shm && chmod 1777 /dev/shm && \ + mkdir -p /run/user/1000 /var/log && \ + # The distribution's own empty `fstab` goes with ours. Nothing in a box + # reads either: init mounts what a box always needs, and every share comes + # from the boot descriptor. A file that looks like it configures mounts and + # is read by nothing is a file somebody edits expecting an effect. + rm -f /etc/network/interfaces /etc/inittab /etc/fstab + +# Nothing in this image may be an init system except `nesinit`. +# +# A service manager arriving as a dependency of something innocuous is the +# failure this catches, and it is silent otherwise: the extra init does not run +# — the kernel is told which one to start — it just sits there with its own +# ideas about what the box should be doing, waiting for somebody to wire it in. +RUN for intruder in /usr/lib/systemd/systemd /sbin/openrc-init /usr/bin/openrc-init \ + /sbin/runit-init /usr/bin/runit-init /sbin/dinit /usr/bin/dinit; do \ + test ! -e "$intruder" || { echo "a second init is in the image: $intruder" >&2; exit 1; }; \ + done + +# What `nesinit` will look for at runtime, checked while there is somebody to +# read the failure. +# +# It is a table compiled into a binary, so a missing program is not a build +# error — it is a service that does not come up in a box somebody is waiting +# on, reported over the control channel and read hours later. Checking here +# turns that into a failed build. +# Everything this image promises must resolve the libraries it links. +# +# This is the check the Mesa change needs: dropping a package that provided +# libraries is how a binary ends up resolving nothing, and the symptom is not a +# build failure — it is a service that will not start in a box somebody is +# waiting on, or a render path that is missing rather than slow. It caught +# exactly that on the first run, and the missing library was two levels down a +# dependency tree nobody had reason to read. +# +# **Named rather than swept, and that is deliberate.** A sweep over everything +# in /usr/lib fails on a stock image: a distribution ships optional plugins +# whose optional dependencies are not installed — pinentry's Qt build, mpg123's +# JACK output, libdecor's GTK backend — and every one of those was already +# unresolved before this stage existed. A check that reports a dozen things +# nobody intends to load is a check the next person deletes. This list is what +# the image is *for*: the components, the services init starts, the chain +# between a workload and the GPU, and Wine's own core. +# +# **Wine was added after it was missed**, and then narrowed twice, which is +# worth recording so nobody widens it again. +# +# It was missed because the list covered everything this image ships *of ours* +# and nothing of the compatibility tool's, so an unresolved `libunwind.so.8` +# behind `ntdll.so` survived a build whose whole purpose is catching that, and +# surfaced as a session that created a prefix and could not start one process +# in it. +# +# The obvious fix -- sweep every `*-unix/*.so` -- is wrong in both directions. +# It is noisy: those objects are Wine's optional backends, and their +# dependencies are a camera library, a media stack, a VR loader, a smartcard +# daemon and OpenCL, none of which belong in a box. And it cannot see what it +# is checking: Wine's unix objects **link each other by soname** and are +# resolved by Wine's own loader rather than by `ld.so`, so `ldd` reports +# `ntdll.so` and `win32u.so` themselves as missing while they sit in the same +# directory. Forty files, every one a false positive, hiding the one real +# entry. +# +# So: the programs in `bin/`, which are ordinary ELF and resolve normally, and +# `ntdll.so`, which is the first thing Wine loads and the one that linked the +# missing library. That is exactly the failure that got through, with none of +# the noise. The Windows-side DLLs beside them are not ELF and `ldd` skips them +# anyway. +# The output is one file per line with its own missing libraries under it, and +# then every missing library once at the end. That last list is what somebody +# acts on -- it is the set of packages to add -- and forty files each naming the +# same two libraries is not that list. An earlier version printed one +# comma-joined line and cut the wrong field out of `ldd`, so it named no +# libraries at all: `ldd` indents with a tab, which `tr -s ' '` does not +# collapse, so the second space-separated field is `=>`. +RUN failed=0; \ + : > /tmp/missing-libs; \ + for f in /usr/bin/nesinit /usr/bin/nescope /usr/bin/neshub /usr/bin/neswire \ + /usr/lib/libnescapture_layer.so \ + /usr/bin/dbus-daemon /usr/bin/pipewire /usr/bin/wireplumber /usr/bin/ip \ + /usr/lib/libgallium-*.so /usr/lib/libEGL_mesa.so.0 \ + /usr/lib/libvulkan_*.so /usr/lib/dri/*.so /usr/lib/gbm/*.so \ + /usr/share/steam/compatibilitytools.d/proton-cachyos/files/bin/* \ + /usr/share/steam/compatibilitytools.d/proton-cachyos/files/lib*/wine/*-unix/ntdll.so; do \ + [ -e "$f" ] || continue; \ + libs="$(ldd "$f" 2>/dev/null | awk '/not found/ { print $1 }')"; \ + [ -n "$libs" ] || continue; \ + failed=1; \ + printf ' %s\n' "$f" >&2; \ + printf ' %s\n' $libs >&2; \ + printf '%s\n' $libs >> /tmp/missing-libs; \ + done; \ + if [ "$failed" != 0 ]; then \ + echo "" >&2; \ + echo " every library above, once each -- this is the list to install:" >&2; \ + sort -u /tmp/missing-libs | sed 's/^/ /' >&2; \ + exit 1; \ + fi + +RUN for required in /usr/bin/nesinit /usr/bin/nescope /usr/bin/neshub /usr/bin/neswire \ + /usr/bin/dbus-daemon /usr/bin/pipewire /usr/bin/wireplumber /usr/bin/ip \ + /usr/bin/python3 \ + /usr/share/steam/compatibilitytools.d/proton-cachyos/proton; do \ + test -x "$required" || { echo "the image is missing $required" >&2; exit 1; }; \ + done + +# An entry point that is executable is not an entry point that runs. +# +# The check above passed on an image whose compatibility tool was a Python +# script with no interpreter behind it: `test -x` says the file may be +# executed, and the kernel then fails to find what the shebang names. A session +# got as far as launching a game and ended with +# `env: 'python3': No such file or directory`. +# +# So every script this image promises resolves its own interpreter. `env` is +# unwrapped where it is used, because a shebang of `#!/usr/bin/env python3` +# names `env` and the thing that is actually missing is the argument. +RUN for script in /usr/share/steam/compatibilitytools.d/proton-cachyos/proton; do \ + head -c 2 "$script" | grep -q '#!' || continue; \ + shebang="$(head -1 "$script" | sed 's/^#!//')"; \ + interpreter="${shebang%% *}"; \ + case "$interpreter" in \ + */env) argument="${shebang#* }"; interpreter="$(command -v "${argument%% *}" || true)";; \ + esac; \ + test -n "$interpreter" && test -x "$interpreter" \ + || { echo "$script needs an interpreter the image does not have: $shebang" >&2; exit 1; }; \ + done + + +# ─────────────────────────────────────────────────────────── +# runtime_prod — the default: `make build` +# ─────────────────────────────────────────────────────────── +FROM runtime AS runtime_prod + +# No console is offered by either flavour: `nesinit` spawns no getty, because +# the way into a guest that will not boot is `init=/bin/bash` on the kernel +# command line, which needs nothing from the image but a shell. So the locked +# root account is belt and braces rather than the only thing standing between +# a box and a login prompt. +RUN passwd -l root + +RUN while IFS= read -r f; do \ + [ -f "$f" ] && strip --strip-unneeded "$f" 2>/dev/null || true; \ + done < /tmp/.strip-manifest; \ + rm -rf /tmp/.strip-manifest /var/cache/pacman/pkg/* /tmp/* /root/.cache \ + /usr/share/man /usr/share/doc /usr/share/locale \ + /usr/lib/cmake /usr/lib/pkgconfig /usr/share/pkgconfig /usr/include \ + /usr/share/gir-1.0 /usr/lib/udev; \ + find /usr/lib -name '*.a' -delete + +RUN echo "NESTRI_STAGE=runtime_prod" >> /etc/os-release + + +# ─────────────────────────────────────────────────────────── +# runtime_debug — `make build-debug` +# ─────────────────────────────────────────────────────────── +FROM runtime AS runtime_debug + +# `-Sy` and not `-Syu`: a full upgrade here can pull a package back in as +# somebody's dependency, and the one that matters is systemd. The check below +# catches it either way, but a debug image that fails to build is worse than +# one that is a few days behind on versions it only uses for `vulkaninfo`. +RUN pacman -Sy --noconfirm --needed vulkan-tools mesa-utils libva-utils && \ + pacman -Scc --noconfirm + +# The same guard as the runtime stage, because the transaction above is exactly +# the kind that quietly reinstates an init system. +RUN test ! -e /usr/lib/systemd/systemd \ + || { echo "systemd's PID 1 came back with the debug tools" >&2; exit 1; } + +# Root has a password here and nothing offers a login prompt to type it into. +# It is for `su` from an `init=/bin/bash` shell, which is the whole debug route. +RUN echo 'root:nestri' | chpasswd + +RUN echo "NESTRI_STAGE=runtime_debug" >> /etc/os-release diff --git a/build/Containerfile.containerignore b/build/Containerfile.containerignore new file mode 100644 index 00000000..d586ed96 --- /dev/null +++ b/build/Containerfile.containerignore @@ -0,0 +1,40 @@ +# The guest rootfs build's context. +# +# The name is load-bearing and it is not `.containerignore`. Podman looks for +# an ignore file *adjacent to the Containerfile and named after it* — here, +# `Containerfile.containerignore` — before falling back to one at the root of +# the build context. The context is the repository root, so a bare +# `build/.containerignore` sits in neither place and is silently read by +# nothing: the build still works, it just sends the whole tree. +# +# Docker looks for the `.dockerignore` suffix only, so a docker build reads the +# repository-root file instead of this one and sends more than it needs. That +# is the cost of the container-agnostic name and it is only a cost in bytes. +# +# This file *replaces* the repository-wide ignore file rather than adding to +# it, so the first block below is that file repeated. The second is what only +# this build excludes. +# +# This build's context is the repository root (see `Makefile`), and it COPYs +# the workspace manifests plus the Rust members and nothing else. The +# TypeScript half is therefore dead weight in the context — a few megabytes +# sent to the daemon versus the whole tree. +.git +node_modules +target +build/output +.env +.env.* +.wrangler +dist +.output + +docs +apps/api +apps/auth +packages +*.md +deno.lock +bun.lock +.zed +.github diff --git a/build/Containerfile.proton b/build/Containerfile.proton new file mode 100644 index 00000000..b09befac --- /dev/null +++ b/build/Containerfile.proton @@ -0,0 +1,102 @@ +# ═══════════════════════════════════════════════════════════ +# proton-cachyos, built wow64-only +# +# Separate from the guest Containerfile because it is a separate cadence: +# hours to build, and only when PROTON_TAG moves. The guest image pulls the +# result from a registry instead of rebuilding it, which is why this file is +# not part of that build and is not reached by `make build`. +# +# The final stage is FROM scratch, so the image *is* the Proton tree — nothing +# to run, only something to COPY --from. +# +# Build and publish with `make proton-image` / `make proton-push`. **Its +# context is this directory**, not the repository root the guest build uses: +# all it needs is the two scripts beside it, and a context of the whole tree +# would hand it a multi-gigabyte `output/` for no reason. +# +# `PROTON_TAG` is the one thing to change, and the Makefile derives the +# published image's version from it. They are the same number in two +# spellings, and an image whose name does not say which Proton is inside it +# is worse than no image. +# ═══════════════════════════════════════════════════════════ +FROM docker.io/archlinux:base-devel AS builder + +# proton-cachyos-native's makedepends, minus every lib32-* (that is the whole +# point of --enable-wow64), with two substitutions Arch requires: ocl-icd +# provides opencl-icd-loader, and mesa-libgl is folded into libglvnd. +# +# glib2-devel is the one that is easy to miss — it carries glib-mkenums, which +# libsoup's meson looks up through glib-2.0's pkg-config variables and fails on +# obscurely. unzip and zip are for the xalia dist step. afdko is not packaged at +# all, and the fonts submodule needs it, so it comes from PyPI. +RUN pacman -Syu --noconfirm --needed \ + alsa-lib clang cmake curl ffmpeg fontforge giflib git glib2-devel \ + glslang gnutls gtk3 libgphoto2 libglvnd libpulse libva libxcomposite \ + libxinerama libxxf86vm lld mesa meson ninja nasm \ + opencl-headers ocl-icd pcsclite perl perl-json python python-pip \ + python-pefile python-setuptools-scm rsync rust samba unixodbc \ + unzip zip v4l-utils vulkan-headers vulkan-icd-loader wayland \ + wayland-protocols wget xorg-util-macros \ + mingw-w64-gcc mingw-w64-binutils mingw-w64-crt mingw-w64-headers \ + mingw-w64-winpthreads \ + && pip install --break-system-packages --no-cache-dir afdko \ + && pacman -Scc --noconfirm + +# widl generates vkd3d's public headers. Without it autoconf sets HAVE_WIDL to +# false, the headers are silently not generated, and the build dies an hour +# later on a missing vkd3d_d3dx9shader.h. Arch ships widl only inside `wine`, +# which requires multilib; the AUR's mingw-w64-tools builds it standalone, so do +# the same. +# +# The digest is not a formality. What is extracted here has its `configure` and +# its makefiles run as root in this builder, and what they produce is copied +# into the image a box runs -- so whoever can change these bytes can change what +# runs on every host. SourceForge hands the request to whichever mirror it +# likes, over a connection this builder does not pin, and the project publishes +# no signature. The digest is the only thing that makes the mirror not matter. +# +# Taken 2026-09-14 from two different mirrors of v14.0.0, which agreed. A +# mismatch here is not a thing to paper over by taking the new digest: it means +# the bytes behind this exact version string changed, and that wants looking at +# before it wants fixing. +ARG MINGW_W64_VER=14.0.0 +ARG MINGW_W64_SHA256=6eaf921d9eb987d3820b364ea9775bc19b965ec81490b6fdd716526c28e1995c +RUN curl -fL --retry 3 -o /tmp/mingw-w64.tar.bz2 \ + "https://sourceforge.net/projects/mingw-w64/files/mingw-w64/mingw-w64-release/mingw-w64-v${MINGW_W64_VER}.tar.bz2/download" \ + && echo "${MINGW_W64_SHA256} /tmp/mingw-w64.tar.bz2" | sha256sum -c - \ + && tar xf /tmp/mingw-w64.tar.bz2 -C /tmp \ + && for arch in i686-w64-mingw32 x86_64-w64-mingw32; do \ + mkdir -p "/tmp/widl-${arch}" && cd "/tmp/widl-${arch}" \ + && "/tmp/mingw-w64-v${MINGW_W64_VER}/mingw-w64-tools/widl/configure" \ + --prefix=/usr --target="${arch}" --program-prefix="${arch}-" \ + && make -j"$(nproc)" && make install; \ + done \ + && rm -rf /tmp/mingw-w64* /tmp/widl-* + +WORKDIR /build + +ARG PROTON_GIT=https://github.com/CachyOS/proton-cachyos.git +ARG PROTON_TAG=cachyos-11.0-20260703-native +ARG GECKO_VER=2.47.4 +ARG MONO_VER=11.2.0 +ARG XALIA_VER=0.4.9 +ARG JOBS= + +# Fetch and build are separate layers on purpose: the submodule checkout runs +# well past ten minutes, and a build that fails on a flag or a missing tool must +# not pay for it again. +COPY scripts/proton-fetch.sh /build/proton-fetch.sh +RUN PROTON_GIT="${PROTON_GIT}" PROTON_TAG="${PROTON_TAG}" \ + GECKO_VER="${GECKO_VER}" MONO_VER="${MONO_VER}" XALIA_VER="${XALIA_VER}" \ + bash /build/proton-fetch.sh + +COPY scripts/proton-build.sh /build/proton-build.sh +RUN GECKO_VER="${GECKO_VER}" MONO_VER="${MONO_VER}" JOBS="${JOBS}" \ + bash /build/proton-build.sh + + +# ═══════════════════════════════════════════════════════════ +# The publishable artifact: the Proton tree and nothing else +# ═══════════════════════════════════════════════════════════ +FROM scratch +COPY --from=builder /artifacts/proton/ / diff --git a/build/Containerfile.proton.containerignore b/build/Containerfile.proton.containerignore new file mode 100644 index 00000000..f7b2c9b1 --- /dev/null +++ b/build/Containerfile.proton.containerignore @@ -0,0 +1,15 @@ +# This build's context is `build/`, not the repository root, because all it +# needs is the two scripts beside the Containerfile. +# +# `output/` is the reason this file exists. It holds packed rootfs images — +# multiple gigabytes each — and a build context is copied before the first +# instruction runs, so without this line every Proton build starts by moving +# the last one it produced. +output + +# Nothing else here is an input to this build. +etc +README.md +Makefile +Containerfile +Containerfile.containerignore diff --git a/build/Dockerfile b/build/Dockerfile deleted file mode 100644 index 0c9be9e3..00000000 --- a/build/Dockerfile +++ /dev/null @@ -1,309 +0,0 @@ -# ═══════════════════════════════════════════════════════════ -# nestri guest rootfs — the open half -# -# Builds a bootable Artix/OpenRC image containing Mesa (virtio-gpu native -# context) and the four open guest components: nescope, neshub, neswire, -# nescapture. Two leaf targets, selected with `--target`: -# -# runtime_prod stripped, root locked (default: `make build`) -# runtime_debug debug tools, autologin root (`make build-debug`) -# -# What is deliberately NOT here: Proton, Valve's steamclient.so, or anything -# else closed. nestri/CLAUDE.md is explicit that nothing closed enters this -# repo. Whatever layers those on top of runtime_prod is a closed build -# outside this repo — see build/README.md. -# -# Build from the repo root, not from build/: -# docker build -f build/Dockerfile --target runtime_prod -t nestri-guest . -# (`make build` in this directory does exactly that.) -# ═══════════════════════════════════════════════════════════ - - -# ─────────────────────────────────────────────────────────── -# initial / builder — Arch, not Artix -# -# The guest is Artix, but Artix's repos are Arch-derived and the toolchain/ -# glibc generation is the same, while the archlinux image is the -# better-maintained of the two for actually compiling things. Only build -# artifacts leave these stages. -# ─────────────────────────────────────────────────────────── -FROM docker.io/archlinux:base-devel AS initial -RUN pacman -Syu --noconfirm - -FROM initial AS builder -RUN pacman -S --noconfirm --needed \ - cmake meson ninja git pkgconf \ - python python-mako python-yaml python-packaging python-ply \ - bison flex \ - libpciaccess libepoxy libglvnd \ - libx11 libxext libxrandr libxshmfence libxfixes libxxf86vm libxcb \ - xcb-util-keysyms xorgproto \ - wayland wayland-protocols \ - expat zlib zstd libxml2 lm_sensors \ - llvm clang libclc spirv-tools spirv-llvm-translator glslang \ - elfutils libva libdrm directx-headers \ - rust rust-bindgen cbindgen \ - curl openssl \ - pixman libxkbcommon \ - vulkan-headers vulkan-icd-loader \ - pipewire shaderc opus \ - libinput \ - && pacman -Scc --noconfirm -WORKDIR /build -ENV ARTIFACTS=/artifacts - - -# ─────────────────────────────────────────────────────────── -# Mesa — the only piece still fetched from outside this tree -# ─────────────────────────────────────────────────────────── -FROM builder AS mesa-build - -ARG MESA_GIT=https://gitlab.freedesktop.org/mesa/mesa.git -ARG MESA_COMMIT=b78fc73dd898a7dfa87448a4b5ff4459a870e21a - -RUN git clone --depth=1 --revision="${MESA_COMMIT}" "${MESA_GIT}" /build/mesa-src && \ - cd /build/mesa-src && \ - meson setup builddir \ - -Dprefix=/usr \ - -Dbuildtype=release \ - -Dplatforms=wayland,x11 \ - -Dgallium-drivers=zink,radeonsi,iris \ - -Dvulkan-drivers=amd,intel \ - -Damdgpu-virtio=true \ - -Dintel-virtio-experimental=true \ - -Dvideo-codecs=all \ - -Degl=enabled \ - -Dglx=dri \ - -Dgles1=enabled \ - -Dgles2=enabled \ - -Dgbm=enabled \ - -Dgallium-va=enabled \ - -Db_ndebug=true && \ - ninja -C builddir && \ - DESTDIR=/artifacts/mesa ninja -C builddir install && \ - rm -rf /build/mesa-src && \ - find /artifacts/mesa -type f -printf '/%P\n' > /artifacts/mesa/.manifest - - -# ─────────────────────────────────────────────────────────── -# nestri workspace — same repo now, so this is COPY, not a private clone -# -# One `cargo build --release --workspace` rather than one stage per binary: -# that per-repo splitting existed because nescope/neswire/nescapture/the hub -# were four separate private repos and a stage boundary was the only way to -# stop bumping one from invalidating the others' build cache. They are one -# Cargo workspace with one Cargo.lock now, so a BuildKit cache mount on -# target/ gives the same isolation — cargo's own incremental compiler -# already knows nescope changing does not touch nesprotocol's .rlib — without -# four copies of every shared dependency getting compiled once per stage. -# ─────────────────────────────────────────────────────────── -FROM builder AS nestri-src -WORKDIR /build/nestri -COPY Cargo.toml Cargo.lock ./ -COPY crates/nesprotocol crates/nesprotocol -COPY apps/nescope apps/nescope -COPY apps/neshub apps/neshub -COPY apps/neswire apps/neswire -COPY apps/nescapture apps/nescapture - -FROM nestri-src AS nestri-build -RUN --mount=type=cache,target=/root/.cargo/registry \ - --mount=type=cache,target=/build/nestri/target \ - cargo build --release --workspace && \ - mkdir -p /artifacts/nestri/usr/bin /artifacts/nestri/usr/lib \ - /artifacts/nestri/usr/share/vulkan/implicit_layer.d && \ - install -Dm755 target/release/nescope /artifacts/nestri/usr/bin/nescope && \ - install -Dm755 target/release/neshub /artifacts/nestri/usr/bin/neshub && \ - install -Dm755 target/release/neswire /artifacts/nestri/usr/bin/neswire && \ - install -Dm755 target/release/libnescapture_layer.so \ - /artifacts/nestri/usr/lib/libnescapture_layer.so && \ - install -Dm644 apps/nescapture/manifest/VK_LAYER_nescapture.json \ - /artifacts/nestri/usr/share/vulkan/implicit_layer.d/VK_LAYER_nescapture.json && \ - find /artifacts/nestri -type f -printf '/%P\n' > /artifacts/nestri/.manifest - - -# ─────────────────────────────────────────────────────────── -# os-base — the Artix rootfs itself -# -# `FROM artixlinux/artixlinux:base-openrc` directly, and `pacman -S` as plain -# RUN steps — not a privileged host `chroot` into a hand-extracted tarball, -# which would need /proc, /sys and /dev bind-mounted in first (they don't -# exist inside a chroot target until something puts them there). A -# Dockerfile RUN step already executes inside a real container with its own -# /proc, /sys, /dev, so there is no bind-mount step to write at all. -# ─────────────────────────────────────────────────────────── -FROM docker.io/artixlinux/artixlinux:base-openrc AS os-base - -RUN pacman -Syu --noconfirm --needed \ - base openrc udev dbus dbus-openrc \ - iptables iproute2 \ - mesa libglvnd libdrm libepoxy libxxf86vm libinput wayland \ - expat zlib llvm-libs elfutils libva shaderc vulkan-icd-loader \ - pixman libxkbcommon xcb-util-keysyms xorg-xwayland \ - pipewire pipewire-audio wireplumber dbus logrotate opus \ - && pacman -Scc --noconfirm - -# groupadd -f so this is idempotent whether or not udev's rules already -# created these. -# -# **Two users, and they must stay two.** `nestri` runs the services that come -# with this image; `nesplay` is who a workload runs as. Sharing one user between -# them is what lets workload code impersonate a service — it can replace the -# socket a service listens on and answer in its place, and the answer that -# matters is the address a client is told to connect to. Init refuses an address -# served by the workload's own user, so a single shared user does not merely -# weaken that check, it makes every session fail it. -# -# The uid a workload actually runs as is chosen by whoever asks for the box, not -# here; this account exists so that the number has a home, a shell and a name in -# `ps`, and so the separation has somewhere to be written down. -RUN groupadd -f audio && groupadd -f video && groupadd -f input && groupadd -f render && \ - useradd -m -u 1000 -s /bin/bash nestri && \ - for g in audio video input render; do gpasswd -a nestri "$g" >/dev/null; done && \ - useradd -m -u 1001 -s /bin/bash nesplay && \ - for g in audio video input render; do gpasswd -a nesplay "$g" >/dev/null; done - - -# ─────────────────────────────────────────────────────────── -# runtime — everything common to debug and prod -# ─────────────────────────────────────────────────────────── -FROM os-base AS runtime - -# This is what GHCR actually uses to connect a pushed image back to its -# repo — not a setting to toggle after the fact, a label the image has to -# carry. Without it a manually-pushed image shows no "used by" repo on its -# package page even though this Dockerfile is exactly what built it. -LABEL org.opencontainers.image.source="https://github.com/nestrilabs/nestri" - -# Our own builds, overlaid on the distro's mesa. The distro package landed -# first (above) so every runtime dependency of *a* Mesa is present and -# correctly versioned; this overwrites its .so files with ours. -# -# COPY --from runs as root inside this build with no invoking-user uid to -# stamp onto / or /usr/bin, unlike a host-side `podman cp` + `cp -a` — so -# there is no ownership-sanity-check to write here. Nothing to catch, on -# purpose, not an oversight. -COPY --from=mesa-build /artifacts/mesa/.manifest /tmp/mesa.manifest -COPY --from=nestri-build /artifacts/nestri/.manifest /tmp/nestri.manifest -RUN cat /tmp/mesa.manifest /tmp/nestri.manifest > /tmp/.strip-manifest && \ - rm -f /tmp/mesa.manifest /tmp/nestri.manifest -COPY --from=mesa-build /artifacts/mesa / -COPY --from=nestri-build /artifacts/nestri / -RUN ldconfig - -COPY build/etc/ /etc/ -RUN chmod +x /etc/init.d/* - -# dbus-session, pipewire and wireplumber carry no conf.d of their own — they -# just want the shared environment, so their conf.d is a symlink to it rather -# than a copy. nescope, neshub and neswire are deliberately absent from this -# loop: each has a conf.d file of its own (already landed by the COPY above) -# that sources nestri-user-env, because each needs settings the shared file -# does not carry. Symlinking them here would overwrite those. -RUN for svc in dbus-session pipewire wireplumber; do \ - ln -sf nestri-user-env "/etc/conf.d/${svc}"; \ - done - -RUN echo nesbox > /etc/hostname && dbus-uuidgen --ensure=/etc/machine-id - -# Session mount points. The guest root is read-only at runtime, so a runtime -# mkdir gets EROFS and takes a service down before it starts — these have to -# already exist in the image. -RUN mkdir -p /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \ - chmod 0755 /nestri /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \ - mkdir -p /dev/shm && chmod 1777 /dev/shm && \ - rm -f /etc/network/interfaces - -# Serial console: the only way into a guest that will not boot. -RUN rm -f /etc/inittab && \ - ln -sf agetty /etc/init.d/agetty.hvc0 && \ - { grep -qx hvc0 /etc/securetty || echo hvc0 >> /etc/securetty; } - -# Every init.d script calling nestri_export_env needs a conf.d that actually -# defines it. Worth a build-time check: the failure at runtime is nearly -# invisible — OpenRC sources the script, the undefined function is a -# "command not found" on stderr, sourcing still returns 0, and the service -# starts anyway with HOME and every XDG_* unset. -RUN missing=""; \ - for svc_script in /etc/init.d/*; do \ - grep -qE '^[[:space:]]*nestri_export_env\b' "$svc_script" 2>/dev/null || continue; \ - svc="$(basename "$svc_script")"; \ - . "/etc/conf.d/$svc" >/dev/null 2>&1; \ - command -v nestri_export_env >/dev/null 2>&1 || missing="$missing $svc"; \ - done; \ - [ -z "$missing" ] || { echo "init scripts call nestri_export_env with no conf.d providing it:$missing" >&2; exit 1; } - -# ── OpenRC service registration ── -# sysinit -RUN rc-update add devfs sysinit && \ - rc-update add dmesg sysinit && \ - rc-update add udev sysinit && \ - rc-update add udev-trigger sysinit && \ - (rc-update del kmod-static-nodes sysinit || true) - -# boot — guest-net replaces the distro's networking scripts entirely; it -# declares `provide net` and brings up lo itself. -RUN (rc-update del networking boot || true) && \ - (rc-update del network-async boot || true) && rm -f /etc/init.d/network-async && \ - rc-update add guest-net boot && \ - (rc-update del net.lo boot || true) && \ - (rc-update del netmount boot || true) && \ - (rc-update del netmount default || true) && \ - rc-update add hostname boot && \ - rc-update add xdg-runtime boot && \ - rc-update add cgroups boot && \ - (rc-update del syslog boot || true) - -# Boot-runlevel services with nothing to do in a freshly-built microVM: no -# physical console, no swap, and fsck would be checking a filesystem nobody -# has touched. -RUN for svc in fsck keymaps save-keymaps termencoding save-termencoding swap binfmt seedrng hwclock swclock; do \ - (rc-update del "$svc" boot || true); \ - (rc-update del "$svc" shutdown || true); \ - done - -# default — the user-facing stack. nescope is registered in plain-compositor -# mode (no payload after `--`): it comes up and waits for something to -# connect. Actually starting a payload is nesinit's job, and nesinit is not -# open code yet — see build/README.md. -RUN rc-update add agetty.hvc0 default && \ - for n in 1 2 3 4 5 6; do (rc-update del "agetty.tty${n}" default || true); done && \ - rc-update add dbus default && \ - rc-update add dbus-session default && \ - rc-update add pipewire default && \ - rc-update add wireplumber default && \ - rc-update add neshub default && \ - rc-update add neswire default && \ - rc-update add nescope default && \ - rm -f /etc/init.d/shared-root && (rc-update del shared-root boot || true) - - -# ─────────────────────────────────────────────────────────── -# runtime_prod — the default: `make build` -# ─────────────────────────────────────────────────────────── -FROM runtime AS runtime_prod - -RUN passwd -l root -COPY build/etc/conf.d/agetty.hvc0.prod /etc/conf.d/agetty.hvc0 - -RUN while IFS= read -r f; do \ - [ -f "$f" ] && strip --strip-unneeded "$f" 2>/dev/null || true; \ - done < /tmp/.strip-manifest; \ - rm -rf /tmp/.strip-manifest /var/cache/pacman/pkg/* /tmp/* /root/.cache \ - /usr/share/man /usr/share/doc /usr/share/locale \ - /usr/lib/cmake /usr/lib/pkgconfig /usr/share/pkgconfig /usr/include - -RUN echo "NESTRI_STAGE=runtime_prod" >> /etc/os-release - - -# ─────────────────────────────────────────────────────────── -# runtime_debug — `make build-debug` -# ─────────────────────────────────────────────────────────── -FROM runtime AS runtime_debug - -RUN pacman -Syu --noconfirm --needed vulkan-tools mesa-utils libva-utils && \ - pacman -Scc --noconfirm -RUN echo 'root:nestri' | chpasswd -COPY build/etc/conf.d/agetty.hvc0.debug /etc/conf.d/agetty.hvc0 - -RUN echo "NESTRI_STAGE=runtime_debug" >> /etc/os-release diff --git a/build/Dockerfile.dockerignore b/build/Dockerfile.dockerignore deleted file mode 100644 index c96e0acd..00000000 --- a/build/Dockerfile.dockerignore +++ /dev/null @@ -1,29 +0,0 @@ -# The guest rootfs build's context. -# -# A `.dockerignore` *replaces* the repository-wide `.dockerignore` -# rather than adding to it, so the first block below is that file repeated. The -# second is what only this build excludes. -# -# This build's context is the repository root (see `Makefile`), and it COPYs -# the workspace manifests plus the Rust members and nothing else. The -# TypeScript half is therefore dead weight in the context — a few megabytes -# sent to the daemon versus the whole tree. -.git -node_modules -target -build/output -.env -.env.* -.wrangler -dist -.output - -docs -apps/api -apps/auth -packages -*.md -deno.lock -bun.lock -.zed -.github diff --git a/build/Makefile b/build/Makefile index dc2c331b..e5c33dd2 100644 --- a/build/Makefile +++ b/build/Makefile @@ -1,12 +1,12 @@ SHELL := /bin/bash -.PHONY: build build-debug image image-debug clean help +.PHONY: build build-debug image image-debug proton-image proton-push clean help -CONTAINER_RT := $(shell command -v docker 2>/dev/null || command -v podman 2>/dev/null) +CONTAINER_RT := $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) ifeq ($(CONTAINER_RT),) $(error "Neither docker nor podman found in PATH") endif -# The Dockerfile COPYs from apps/ and crates/, so the build context is the +# The Containerfile COPYs from apps/ and crates/, so the build context is the # repo root, not this directory — same reason borealis's build/ *is* its own # context: here the guest source lives one level up instead of inside build/. CONTEXT := .. @@ -27,17 +27,47 @@ CONTEXT := .. # where it does not exist. One name, always pullable, beats a local-only # short name and a different published one. IMAGE_NAME := ghcr.io/nestrilabs/nestri/base + +# Proton is the one artifact worth publishing on its own: it takes hours to +# build and changes only when its tag moves, so the guest build pulls it by +# name instead of rebuilding it. +# +# `PROTON_TAG` is the single place to change it. The published version is +# derived from the tag rather than written twice, because the two are the same +# number in two spellings and an image whose name does not say which Proton is +# inside it is worse than no image at all. Changing the tag by hand and +# forgetting the version is exactly the mistake this removes. +PROTON_TAG ?= cachyos-11.0-20260703-native +PROTON_VERSION := $(PROTON_TAG:cachyos-%-native=%) +PROTON_IMAGE ?= ghcr.io/nestrilabs/proton-cachyos-native-wow64 +PROTON_REF := $(PROTON_IMAGE):$(PROTON_VERSION) OUTPUT_DIR := output -ROOTFS_SIZE ?= 5G +ROOTFS_SIZE ?= 3G FORCE_REBUILD ?= build: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ - -f Dockerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT) + --build-arg PROTON_IMAGE=$(PROTON_REF) \ + -f Containerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT) build-debug: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ - -f Dockerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT) + --build-arg PROTON_IMAGE=$(PROTON_REF) \ + -f Containerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT) + +# Hours, and only when PROTON_TAG moves. Its context is this directory rather +# than the repository root: the two scripts beside the Containerfile are the +# whole input, and the root would hand it everything else for nothing. +proton-image: + DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ + --build-arg PROTON_TAG=$(PROTON_TAG) \ + -f Containerfile.proton -t $(PROTON_REF) . + @echo "Built $(PROTON_REF)" + +# Publishing is what makes `make build` cheap for everyone else, since that +# build pulls this by name. Push before expecting anyone to use a new tag. +proton-push: proton-image + $(CONTAINER_RT) push $(PROTON_REF) # Not `sudo make image`/sudo'd in here: mkimage.sh runs as you and escalates # only the specific commands that need root. Under rootless Podman, `make @@ -62,5 +92,8 @@ help: @echo " make image Build + pack runtime_prod into output/rootfs.ext4" @echo " make image-debug Build + pack runtime_debug into output/rootfs-debug.ext4" @echo " make clean Remove output/" + @echo " make proton-image Build Proton from source (hours)" + @echo " make proton-push Build it and publish it" @echo " make FORCE_REBUILD=1 ... Rebuild from scratch, no layer cache" + @echo " make PROTON_TAG=... ... Use a different proton-cachyos tag" @echo " make ROOTFS_SIZE=8G image Override the packed image size (default 5G)" diff --git a/build/README.md b/build/README.md index 4759b93a..766b6e35 100644 --- a/build/README.md +++ b/build/README.md @@ -1,16 +1,17 @@ # build/ — the guest rootfs -Builds a bootable Artix/OpenRC image for the box's virtio-blk root: Mesa -(virtio-gpu native context) plus the four open guest components — -[`nescope`](../apps/nescope), [`neshub`](../apps/neshub), -[`neswire`](../apps/neswire), [`nescapture`](../apps/nescapture) — laid out +Builds a bootable Arch image for the box's virtio-blk root: Mesa (virtio-gpu +native context) plus the five open guest components — +[`nesinit`](../apps/nesinit), [`nescope`](../apps/nescope), +[`neshub`](../apps/neshub), [`neswire`](../apps/neswire), +[`nescapture`](../apps/nescapture) — laid out the way [borealis](https://chromium.googlesource.com/chromiumos/overlays/board-overlays/+/main/project-borealis) -lays out its `build/`: one big multi-stage `Dockerfile`, `--target` picks the +lays out its `build/`: one big multi-stage `Containerfile`, `--target` picks the flavor, `etc/` holds the files that get overlaid onto the image verbatim. ``` build/ -├── Dockerfile everything, in stages: mesa-build, nestri-build, +├── Containerfile everything, in stages: mesa-build, nestri-build, │ os-base, runtime, runtime_prod, runtime_debug ├── etc/ overlaid onto the image's /etc as-is ├── scripts/ @@ -24,6 +25,9 @@ make build # docker build --target runtime_prod → ghcr.io/nestrilab make build-debug # docker build --target runtime_debug → ghcr.io/nestrilabs/nestri/base:debug make image # + pack into output/rootfs.ext4 make image-debug # + pack into output/rootfs-debug.ext4 + +make proton-image # build Proton from source — hours +make proton-push # build it and publish it ``` ## Design notes @@ -33,10 +37,10 @@ Three things worth knowing about how this is put together: 1. **No privileged host chroot.** A bare `chroot` into a hand-extracted rootfs needs `/proc`, `/sys`, `/dev` bind-mounted in first — they don't exist inside a chroot target until something puts them there. `os-base` - here is `FROM artixlinux/artixlinux:base-openrc` directly, with - `pacman -S` as plain `RUN` steps — a Docker build step already runs - inside a real container with its own `/proc`, `/sys`, `/dev`, so that - whole bind-mount mechanism has nothing to do. + here is `FROM archlinux:base` directly, with `pacman -S` as plain `RUN` + steps — a Docker build step already runs inside a real container with its + own `/proc`, `/sys`, `/dev`, so that whole bind-mount mechanism has nothing + to do. 2. **No host-side ownership bug to guard against.** `COPY --from=` runs as root inside the build with no host user in the loop, so there's no @@ -50,44 +54,194 @@ Three things worth knowing about how this is put together: own incremental compiler per-crate isolation without needing a separate Docker stage (and a separate full rebuild of `nesprotocol`) per binary. -**What is deliberately not here: Proton, and Valve's `steamclient.so`.** +**What is deliberately not here: Valve's `steamclient.so`.** `nestri/CLAUDE.md` is explicit — *"Nothing closed may enter this repo. Not -source, not a dependency, not a directory that 'looked convenient'."* Both -are closed. `runtime_prod` from this Dockerfile — tagged -`ghcr.io/nestrilabs/nestri/base:latest` — is a complete, bootable, Steam-less guest image, +source, not a dependency, not a directory that 'looked convenient'."* That one +is closed, so whatever layers it on top is a build outside this repo — not +something this repo names, links to, or depends on. + +**Proton is here, and this paragraph used to say it was not.** The old wording +put Proton and `steamclient.so` together and called both closed, which is +wrong about Proton: it is compiled from source, which is not a thing you can +do with closed software. Keeping it out cost a box the only way it has to run +a Windows title, for a rule that did not apply to it. + +What it is: **proton-cachyos built with `--enable-wow64`**, pulled by tag as a +published image rather than rebuilt here, because it takes hours and moves +only when its own tag does. `PROTON_IMAGE` overrides the tag, and it has to be +declared before the first `FROM` — an `ARG` a `FROM` expands is global or it +is nothing, and getting that wrong fails with `no FROM statement found`, which +says nothing about the actual mistake. wow64 is the whole reason it is a build of ours +and not the distribution's package — it runs 32-bit Windows code inside a +64-bit unix process, so a box needs no lib32 glibc, no second Mesa for i686, +and no second capture layer for 32-bit titles to be captured. The +distribution's package is built without the flag, which is exactly why it +depends on `lib32-*`. + +It costs about 1.4 GB of image, and it is the one thing in here that is +payload-shaped: a compatibility layer for Windows games in an image that is +otherwise indifferent to what it runs. The guest components stay indifferent +regardless — none of them branches on it, and the init does not know it +exists. What names it is the command a caller sends. + +`runtime_prod` from this Containerfile — tagged +`ghcr.io/nestrilabs/nestri/base:latest` — is a complete, bootable guest image, and also the shared foundation other builds start from: nesbox's jail image (see `nesbox/build/`) extracts **Mesa** from it so the guest and host sides of the virtio-gpu native-context protocol never drift apart. Only Mesa — `virglrenderer` is the host half of that protocol and nesbox builds its own, patched, from `nesbox/patches/`; nothing in this image carries it. -Whatever layers Proton and the Steam client on top of it is a closed build -outside this repo, by design — not something this repo names, links to, or -depends on. +## Two packages that look droppable and are not -## The nesinit gap +`llvm-libs` is 164 MB, the largest single thing in the image after Proton, and +`lm_sensors` is only there because something links `libsensors`. Both look like +leftovers of a Mesa configuration that has since been trimmed, and both have +been checked rather than reasoned about: **`libgbm` links them**, and the +compositor needs GBM. Trimming the Mesa build does not reach them. -Nothing in this image starts a payload. The old `nestri-guest-hub` did that -— per its own commit message, the open `neshub` *"loses `--proton`, -`--steamclient-so`, `--root` and the game uid/gid, and no longer ends by -handing the process to a controller... Deciding when the box is finished -belongs to `nesinit`."* `nesinit` — the guest init/session-supervisor that -would actually launch `nescope -- ` and power the box down — is -referenced in commit messages and `nesbox/PROGRESS.md` but does not exist as -open code in either repo. +`lm_sensors` in particular was found the hard way. It used to arrive as a +dependency of the distribution's Mesa package, and dropping that package took +it away — leaving our own Mesa unable to resolve `libsensors.so.5`. Nothing in +a package list says that; the check below is what said it. -So `/etc/init.d/nescope` here starts nescope in **plain-compositor mode** -(no command after `--`): it comes up, provides the Wayland/X11 environment, -and waits for something to connect. That makes the image genuinely bootable -and testable — `neshub`, `neswire`, `nescope` all come up under OpenRC and -you get a real Wayland socket to point a client at — but running an actual -game is still nesinit's job, and nesinit isn't part of this build. Whoever -picks that up next should read `apps/neshub/README.md`'s "What it does not -do" section first. +## Proton has its own cadence, and its own Containerfile + +`make build` **pulls** Proton by tag; it does not build it. Building it takes +hours and it changes only when its tag moves, so it is one image published +once and copied into every guest image after that. `Containerfile.proton` is +that build, and it lives here so the published tag stays reproducible from +this tree rather than from somebody's laptop. + +```sh +make proton-image # the current tag +make PROTON_TAG=cachyos-11.1-20261115-native proton-image +``` + +**`PROTON_TAG` is the only thing to change.** The published version is derived +from it in the `Makefile` rather than written a second time, because the two +are the same number in two spellings — and an image whose name does not say +which Proton is inside it is worse than no image. The `Containerfile`'s own +`PROTON_IMAGE` default is a fallback for a bare container build; going through +`make` is what keeps them in step. + +Its **context is `build/`**, not the repository root the guest build uses. All +it needs is the two scripts beside it, and `Containerfile.proton.containerignore` +keeps `output/` out of that context — a build context is copied before the +first instruction runs, so without it every Proton build would begin by moving +the last rootfs image it produced. + +Two things in the recipe are worth knowing before changing it: + +- **Fetch and build are separate layers on purpose.** The submodule checkout + runs well past ten minutes, and a build that fails on a flag or a missing + tool must not pay for that again. Keep anything that can fail *fast* in + `proton-build.sh`. +- **`widl` is built by hand from the mingw-w64 release.** Without it autoconf + quietly sets `HAVE_WIDL` to false, vkd3d's public headers are never + generated, and the build dies an hour later on a missing header. Arch ships + `widl` only inside `wine`, which wants multilib — which is the thing + `--enable-wow64` exists to avoid. + +## There is no init system in here, and that is the design + +`nesinit` is PID 1. The image carries **no service manager, no init scripts, +no `udev` and no systemd** — `systemd-libs` stays, because `dbus-daemon` and +`wireplumber` link `libsystemd.so.0`, but nothing in the image can be PID 1 +except `nesinit`, and the build fails if anything that could be turns up. + +That is why this is plain Arch. The image used to be Artix, chosen for OpenRC, +and every cost of that choice — no `eudev`, no `agetty-openrc`, `udev` being +systemd's anyway, a runlevel edit not stopping a service another one still +needs — was paid for an init system that is no longer here. + +**What replaced fourteen `rc-update` lines and nine init scripts:** + +| was | now | +|---|---| +| `devfs`, `dmesg`, `udev`, `udev-trigger` | `devtmpfs` makes the nodes; init sets the two modes that matter. The compositor takes input through Wayland and opens nothing `udev` provides | +| `guest-net`, `hostname`, `xdg-runtime`, `cgroups` | init, before it dials out | +| `dbus`, `dbus-session`, `pipewire`, `wireplumber`, `neshub`, `neswire` | a table compiled into `nesinit` | +| `nescope` in the `default` runlevel | **not a service.** It wraps the workload and is started by a launch, with that launch's geometry, and dies with it | +| `agetty` on `hvc0` | nothing. See below | +| `/etc/fstab` | init's own mounts, and shares named in the boot descriptor | + +**A box is launched into, not booted into something.** Init mounts what the +descriptor names, brings the table up, says it is ready, and then takes +commands — so an image on its own runs nothing at all, which is the point: +this image is payload-independent and there is no payload in it. + +### Getting into a guest that will not boot + +`init=/bin/bash` on the kernel command line. Nothing in the image offers a +login prompt — there is no getty in either flavour — and that is cheaper than +carrying one: `nesinit` is an ordinary program, so from that shell you can run +it by hand and watch it fail. `make build-debug` adds `vulkaninfo` and friends +and gives root a password for `su`; it does not add a console. + +**A shell is not a booted box, and the difference bites immediately.** Nothing +the init does has happened: the root is read-only, `/run` and `/tmp` are still +directories on it rather than tmpfs, `/run/user/1000` is unwritable, and the +hostname is `(none)` rather than `nesbox` — which is the quickest way to tell +the two states apart. A compositor started in that shell fails on its own +socket, and the error names the runtime directory rather than the cause. + +So run `nesinit` first. It mounts, prepares the directories, brings the +services up, then fails to reach a control channel that is not there and +exits — **leaving everything it prepared behind**, which is exactly what makes +the hand-run useful. Then start what you came to debug. + +It is safe to run outside a box, and that took fixing: the shutdown path signals +every process it may signal and then powers the machine off, which is right for +PID 1 of a box and catastrophic anywhere else. Both steps are refused when it is +not PID 1, and it says so rather than doing it quietly. + +If you would rather not run it at all, the two mounts it does that a compositor +needs are: + +```sh +mount -t tmpfs -o mode=1777,size=64m tmpfs /tmp +mount -t tmpfs -o mode=755,size=32m tmpfs /run +mkdir -p /run/user/1000 && chown 1000:1000 /run/user/1000 && chmod 0700 /run/user/1000 +``` + +Making `/run/user/1000` writable in the *image* does not help, and is worth +saying because it is the obvious first thing to try: before the init runs, the +directory is on a read-only root, so its ownership is not what stops a write; +after the init runs, a fresh tmpfs is mounted over `/run` and the image's copy +of the directory is hidden underneath it. + +The one thing this does not reach is a failure *before* the shell. If that +happens the evidence is on `console=hvc0` and nowhere else. + +### Two build-time checks worth knowing about + +Both exist because the failure they catch is invisible at runtime rather than +loud, which is the same reason the old build checked its `conf.d` files: + +- **No hook may point at a program that is not in the image.** Removing + systemd removes the script `dbus-reload.hook` calls, and a leftover hook + produces `error: command failed to execute correctly` on every future pacman + transaction — indistinguishable, in a log, from something that matters. +- **Nothing `nesinit` will look for may be missing.** Its service table is + compiled in, so an absent `dbus-daemon` is not a build error by itself; it + is a service that does not come up in a box somebody is waiting on. ## Network defaults -`/etc/init.d/guest-net` reads `nestri.ip=`/`nestri.gw=` off the kernel -command line, falling back to `172.30.0.2/24` via `172.30.0.1` if neither is -set — nesbox's own default tap addressing. Keep these in step if that -changes on the nesbox side. +`nesinit` reads `nestri.ip=`/`nestri.gw=` off the kernel command line, +falling back to `172.30.0.2/24` via `172.30.0.1` if neither is set — the +host's own default tap addressing. Keep these in step if that changes on the +host side. A box started with no network device at all is a valid box and +boots without one. + +The address is a per-boot parameter rather than an image setting because the +alternative makes every box built from this image the same host on the +network, and two of them collide the moment they run together. Same reasoning +for `/etc/machine-id`, which is a symlink into a tmpfs that init fills at +boot — the previous image baked one in, so every box built from it was the +same machine to anything that asked. + +This is also the one thing in the image that keeps `iproute2` installed: +init runs `ip` rather than talking netlink, which is a hundred lines of +`unsafe` saved for an interface configured once. diff --git a/build/etc/conf.d/agetty.hvc0.debug b/build/etc/conf.d/agetty.hvc0.debug deleted file mode 100644 index 79dc110f..00000000 --- a/build/etc/conf.d/agetty.hvc0.debug +++ /dev/null @@ -1,3 +0,0 @@ -baud="115200" -term_type="vt100" -agetty_options="--autologin root --noclear" diff --git a/build/etc/conf.d/agetty.hvc0.prod b/build/etc/conf.d/agetty.hvc0.prod deleted file mode 100644 index 8a9020af..00000000 --- a/build/etc/conf.d/agetty.hvc0.prod +++ /dev/null @@ -1,3 +0,0 @@ -baud="115200" -term_type="vt100" -agetty_options="--noclear" diff --git a/build/etc/conf.d/nescope b/build/etc/conf.d/nescope deleted file mode 100644 index 3ae9f1d2..00000000 --- a/build/etc/conf.d/nescope +++ /dev/null @@ -1,16 +0,0 @@ -# Configuration for nescope, the headless compositor. -# -# Sources nestri-user-env rather than symlinking it, for the same reason -# neshub and neswire do. -. /etc/conf.d/nestri-user-env - -RUST_LOG="${RUST_LOG:-nescope=info}" -export RUST_LOG - -# Started in plain-compositor mode by /etc/init.d/nescope (no command after -# `--`): it comes up and waits for something to connect rather than wrapping -# a payload. Starting an actual payload — a game, a desktop — is nesinit's -# job, and nesinit is not open code yet. See build/README.md. -NESCOPE_SOCKET="${NESCOPE_SOCKET:-nescope-0}" -NESCOPE_INPUT_IPC="${NESCOPE_INPUT_IPC:-/tmp/nestri-input.sock}" -export NESCOPE_SOCKET NESCOPE_INPUT_IPC diff --git a/build/etc/conf.d/neshub b/build/etc/conf.d/neshub deleted file mode 100644 index 96018062..00000000 --- a/build/etc/conf.d/neshub +++ /dev/null @@ -1,18 +0,0 @@ -# Configuration for neshub, the media hub. -# -# Not a symlink to nestri-user-env like pipewire and friends: neshub wants a -# setting of its own, so it sources that file rather than replacing it — same -# pattern as nescope and neswire below. -. /etc/conf.d/nestri-user-env - -# What neshub logs, and nothing else. -# -# Scoped and exported for the reason spelled out at length in the old -# nestri-guest-hub conf.d this replaces: an unscoped `RUST_LOG=info` turns on -# every crate linked in, iroh included, and OpenRC hands a service its own -# environment, not the shell's locals — an unexported variable here goes -# nowhere. -# -# Overridable, so a bad boot can be re-run with more without a rebuild. -RUST_LOG="${RUST_LOG:-neshub=info}" -export RUST_LOG diff --git a/build/etc/conf.d/nestri-user-env b/build/etc/conf.d/nestri-user-env deleted file mode 100644 index ccde8729..00000000 --- a/build/etc/conf.d/nestri-user-env +++ /dev/null @@ -1,34 +0,0 @@ -HOME="/home/nestri" -USER="nestri" -LOGNAME="nestri" -XDG_RUNTIME_DIR="/run/user/1000" -XDG_CONFIG_HOME="/home/nestri/.config" -XDG_DATA_HOME="/home/nestri/.local/share" -XDG_CACHE_HOME="/home/nestri/.cache" -XDG_STATE_HOME="/home/nestri/.local/state" -XDG_DATA_DIRS="/usr/local/share:/usr/share" -XDG_SESSION_TYPE="wayland" -XDG_SESSION_DESKTOP="nestri" -DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus" - -# Driver forcing — currently off, in both copies. Kept, commented, because -# the reasoning still applies if it is ever needed again: capture happens -# through a Vulkan layer, so a process reaching the GPU via native OpenGL -# would render fine and capture nothing, and zink is what makes such a -# process capturable at all. Nothing has needed it so far. -# -# /etc/profile.d/nestri-env.sh carries the same block, also commented, and -# only runs for login shells — nothing here is one, every service below runs -# from OpenRC. If these are ever re-enabled, re-enable both: one copy on and -# one off means services and shells reach the GPU by different paths. -#__GLX_VENDOR_LIBRARY_NAME="mesa" -#MESA_LOADER_DRIVER_OVERRIDE="zink" -#GALLIUM_DRIVER="zink" - -nestri_export_env() { - export HOME USER LOGNAME - export XDG_RUNTIME_DIR XDG_CONFIG_HOME XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME - export XDG_DATA_DIRS XDG_SESSION_TYPE XDG_SESSION_DESKTOP - export DBUS_SESSION_BUS_ADDRESS - #export __GLX_VENDOR_LIBRARY_NAME MESA_LOADER_DRIVER_OVERRIDE GALLIUM_DRIVER -} diff --git a/build/etc/conf.d/neswire b/build/etc/conf.d/neswire deleted file mode 100644 index 517d42b2..00000000 --- a/build/etc/conf.d/neswire +++ /dev/null @@ -1,32 +0,0 @@ -# Configuration for the audio wire. -# -# Not a symlink to nestri-user-env like pipewire and friends, for the same -# reason neshub is not: it needs a setting of its own -- and it still wants -# that file's environment, so it sources it rather than replacing it. -. /etc/conf.d/nestri-user-env - -# What neswire logs, and nothing else. -# -# Without this neswire is *silent* -- it starts, fails, and exits leaving an -# empty log, which is indistinguishable from never having been started at -# all. That ambiguity is the reason this file exists. -# -# Scoped, and `export`ed, for the reasons spelled out at length in -# conf.d/neshub: an unscoped `info` turns on every linked crate, and an -# unexported one reaches the service's shell and not the service. -# -# Overridable, so a bad boot can be re-run with more without a rebuild. -RUST_LOG="${RUST_LOG:-neswire=info}" -export RUST_LOG - -# neswire's own defaults, spelled out rather than inherited. -# -# All four are clap args with defaults (`neswire --help`), so the service -# works without them. They are here because the IPC path is a contract with -# neshub -- it reads this socket -- and a contract that lives only inside two -# binaries' default values is one nobody can check. -NESWIRE_IPC_PATH="/tmp/nestri-audio.sock" -NESWIRE_CHANNELS="2" -NESWIRE_PACKET_DURATION_MS="5" -NESWIRE_BITRATE_PER_CHANNEL="64" -export NESWIRE_IPC_PATH NESWIRE_CHANNELS NESWIRE_PACKET_DURATION_MS NESWIRE_BITRATE_PER_CHANNEL diff --git a/build/etc/fstab b/build/etc/fstab deleted file mode 100644 index b183fda8..00000000 --- a/build/etc/fstab +++ /dev/null @@ -1,35 +0,0 @@ -# `ro`, matching how a box is actually started: nesbox passes `ro` on the -# kernel command line and marks the root device `is_read_only: true` (see -# nesbox's examples/vm.json), so the virtio-blk device refuses writes at the -# device level. Saying `rw` here does not make it writable — it only asks -# OpenRC's `root` service to attempt a remount that the device must reject. -# Everything a running box writes to is a tmpfs or a share below. -/dev/vda / ext4 ro,relatime 0 1 -devtmpfs /dev devtmpfs rw,nosuid 0 0 -proc /proc proc rw,nosuid,nodev,noexec 0 0 -sysfs /sys sysfs rw,nosuid,nodev,noexec 0 0 -tmpfs /tmp tmpfs rw,nosuid,nodev,size=64M 0 0 -tmpfs /run tmpfs rw,nosuid,nodev,size=32M,mode=0755 0 0 -tmpfs /var/log tmpfs rw,nosuid,nodev,size=16M 0 0 -# POSIX shared memory. devtmpfs does not provide it and nothing else mounts it, -# so without this entry /dev/shm does not exist at all and `shm_open` fails. -# -# PipeWire itself gets by: it allocates buffers with memfd_create and only -# falls back to /dev/shm. Kept as a precaution for anything else that uses -# POSIX shm/semaphores directly — Proton is the known example, and it is not -# part of this image yet (see build/README.md), but a failure here is silent -# rather than fatal, so it costs nothing to have ready. -# -# `nosuid,nodev` and a size cap because everything a game can write to should -# have both. -tmpfs /dev/shm tmpfs rw,nosuid,nodev,size=256M 0 0 -# The guest's logs, on a host directory that outlives the VM. -# -# Here rather than mounted by a service, because the failures worth reading -# are the ones that happen *before* anything else has mounted something — a -# log directory that appears only after a successful start cannot record an -# unsuccessful one. The tag is fixed by nessh, so plain fstab works. -# -# `nofail` because a VM started without the share still has to boot: that is -# how somebody gets a shell to find out why it has no share. -logs /nestri/logs virtiofs rw,nofail 0 0 diff --git a/build/etc/init.d/dbus-session b/build/etc/init.d/dbus-session deleted file mode 100644 index 833e5810..00000000 --- a/build/etc/init.d/dbus-session +++ /dev/null @@ -1,33 +0,0 @@ -#!/sbin/openrc-run - -description="D-Bus session bus for nestri user" - -nestri_export_env - -command="/usr/bin/dbus-daemon" -command_args="--session --address=unix:path=/run/user/1000/bus --nofork --nopidfile --print-address" -command_user="nestri:nestri" -command_background="yes" -pidfile="/run/nestri/dbus-session.pid" - -depend() { - need xdg-runtime dbus - before pipewire wireplumber -} - -start_pre() { - checkpath -d -m 0755 -o "nestri:nestri" /run/nestri - if [ -e "/run/user/1000/bus" ]; then - ewarn "Removing stale bus socket" - rm -f "/run/user/1000/bus" - fi -} - -start_post() { - local i=0 - while [ ! -S /run/user/1000/bus ] && [ $i -lt 20 ]; do - sleep 0.1 - i=$((i+1)) - done - [ -S /run/user/1000/bus ] || { eerror "Bus didn't appear"; return 1; } -} diff --git a/build/etc/init.d/dbus-system b/build/etc/init.d/dbus-system deleted file mode 100644 index a79a25fb..00000000 --- a/build/etc/init.d/dbus-system +++ /dev/null @@ -1,19 +0,0 @@ -#!/sbin/openrc-run - -description="D-Bus system message bus" - -command="/usr/bin/dbus-daemon" -command_args="--system --nofork --nopidfile" -command_background="yes" -pidfile="/run/dbus/dbus.pid" - -depend() { - need localmount xdg-runtime - before pipewire -} - -start_pre() { - checkpath -d -m 0755 -o root:root /run/dbus - checkpath -d -m 0755 -o messagebus:messagebus /var/run/dbus 2>/dev/null || true - dbus-uuidgen --ensure=/var/lib/dbus/machine-id -} diff --git a/build/etc/init.d/guest-net b/build/etc/init.d/guest-net deleted file mode 100644 index 21aa178a..00000000 --- a/build/etc/init.d/guest-net +++ /dev/null @@ -1,92 +0,0 @@ -#!/sbin/openrc-run - -description="Configure the guest's network to match the host's tap" - -# Overridden from /etc/conf.d/guest-net if present. These defaults match -# nesbox's own defaults; if you change the `network` section in the VM's -# JSON, change these to match. -: ${GUEST_IFACE:=eth0} -: ${GUEST_IP:=172.30.0.2} -: ${GUEST_PREFIX:=24} -: ${GUEST_GATEWAY:=172.30.0.1} - -depend() { - need localmount - provide net - keyword -shutdown -} - -# Read one `nestri.=` from the kernel command line. -# -# The address has to come from somewhere per-boot, because the alternative -- -# baking it into the image -- makes every guest built from that image the -# same host on the network. Two sandboxes then collide the moment they run -# together. -# -# `nestri.`-prefixed rather than the kernel's own `ip=`: that one needs -# CONFIG_IP_PNP and exists to configure NFS root, and the prefix makes it -# obvious whose parameter this is. -cmdline_value() { - local key="$1" word - for word in $(cat /proc/cmdline 2>/dev/null); do - case "$word" in - "nestri.${key}="*) printf '%s' "${word#nestri.${key}=}"; return 0 ;; - esac - done - return 1 -} - -start() { - ebegin "Bringing up loopback" - ip link set lo up - eend $? - - # The command line wins over conf.d when it says anything, and conf.d is - # the fallback so a hand-written VM config with no parameters keeps - # working -- which is how a guest gets debugged. - local source="/etc/conf.d/guest-net" - local cmdline_ip - if cmdline_ip="$(cmdline_value ip)"; then - # Accepts address/prefix; a bare address keeps the configured prefix - # rather than guessing one. - case "$cmdline_ip" in - */*) - GUEST_IP="${cmdline_ip%%/*}" - GUEST_PREFIX="${cmdline_ip##*/}" - ;; - *) GUEST_IP="$cmdline_ip" ;; - esac - source="kernel command line" - fi - - local cmdline_gw - if cmdline_gw="$(cmdline_value gw)"; then - GUEST_GATEWAY="$cmdline_gw" - source="kernel command line" - fi - - # The VM may have been started with no network device at all, which is a - # perfectly good configuration. Do not fail the boot over it. - if [ ! -e "/sys/class/net/${GUEST_IFACE}" ]; then - einfo "no ${GUEST_IFACE}: this VM has no network device" - return 0 - fi - - # Says which source won, because "the address is wrong" and "the address - # came from somewhere unexpected" look identical from inside the guest. - ebegin "Configuring ${GUEST_IFACE} as ${GUEST_IP}/${GUEST_PREFIX} via ${GUEST_GATEWAY} (from ${source})" - # `replace` rather than `add` so a restart is not an error. - ip link set "${GUEST_IFACE}" up && - ip addr replace "${GUEST_IP}/${GUEST_PREFIX}" dev "${GUEST_IFACE}" && - ip route replace default via "${GUEST_GATEWAY}" dev "${GUEST_IFACE}" - eend $? "could not configure ${GUEST_IFACE}" -} - -stop() { - if [ -e "/sys/class/net/${GUEST_IFACE}" ]; then - ebegin "Bringing down ${GUEST_IFACE}" - ip link set "${GUEST_IFACE}" down - eend 0 - fi - return 0 -} diff --git a/build/etc/init.d/nescope b/build/etc/init.d/nescope deleted file mode 100644 index 9c916c00..00000000 --- a/build/etc/init.d/nescope +++ /dev/null @@ -1,33 +0,0 @@ -#!/sbin/openrc-run - -description="Nestri headless compositor" - -nestri_export_env - -: "${NESTRI_UID:=1000}" -: "${NESTRI_USER:=nestri}" - -command="/usr/bin/nescope" -# No command after `--`: plain-compositor mode. nescope comes up and waits -# for something to connect rather than wrapping a payload — starting one is -# nesinit's job, and nesinit is not open code yet. See build/README.md. -command_user="${NESTRI_USER}:${NESTRI_USER}" -command_background="yes" -pidfile="/run/nestri/nescope.pid" -output_log="/nestri/logs/nescope.log" -error_log="/nestri/logs/nescope.log" -respawn="yes" -respawn_delay="2" -respawn_max="2" - -export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}" - -depend() { - need xdg-runtime - use neshub - after xdg-runtime -} - -start_pre() { - checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri -} diff --git a/build/etc/init.d/neshub b/build/etc/init.d/neshub deleted file mode 100644 index e408c22b..00000000 --- a/build/etc/init.d/neshub +++ /dev/null @@ -1,39 +0,0 @@ -#!/sbin/openrc-run - -description="Nestri media hub" - -nestri_export_env - -: "${NESTRI_UID:=1000}" -: "${NESTRI_USER:=nestri}" - -command="/usr/bin/neshub" -# Unprivileged. The old nestri-guest-hub ran as root and mounted the -# session's filesystem itself; the open neshub does neither — per its own -# README it only muxes the Unix sockets the other components dial into one -# QUIC endpoint. Whatever ends up owning session mounts (nesinit, presumably) -# is a separate, still-closed piece. -command_user="${NESTRI_USER}:${NESTRI_USER}" -command_background="yes" -pidfile="/run/nestri/neshub.pid" - -# Where neshub's output goes. /nestri/logs is a virtiofs share mounted from -# /etc/fstab at boot, before this service starts, so a hub that fails -# immediately still leaves a record, and the record survives the VM. -output_log="/nestri/logs/neshub.log" -error_log="/nestri/logs/neshub.log" -respawn="yes" -respawn_delay="2" -respawn_max="2" # NO infinite — if it fails, it fails for a reason - -export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}" - -depend() { - need xdg-runtime - use net - after xdg-runtime -} - -start_pre() { - checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri -} diff --git a/build/etc/init.d/neswire b/build/etc/init.d/neswire deleted file mode 100644 index 2c61af9e..00000000 --- a/build/etc/init.d/neswire +++ /dev/null @@ -1,93 +0,0 @@ -#!/sbin/openrc-run - -description="Nestri pipewire audio sink" - -nestri_export_env - -: "${NESTRI_UID:=1000}" -: "${NESTRI_USER:=nestri}" - -command="/usr/bin/neswire" -command_user="${NESTRI_USER}:${NESTRI_USER}" -command_background="yes" -pidfile="/run/nestri/neswire.pid" -output_log="/nestri/logs/neswire.log" -error_log="/nestri/logs/neswire.log" -respawn="yes" -respawn_delay="2" -respawn_max="2" - -export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}" -export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${NESTRI_UID}/bus" - -depend() { - # wireplumber is a hard dependency, not a nicety: start_post below pins - # the default sink through the `default` metadata object, and - # WirePlumber is what owns that object. Started concurrently, the pin - # lands in an object pw-metadata created itself, which is destroyed the - # moment it exits. - need xdg-runtime dbus dbus-session pipewire wireplumber - use neshub - after pipewire wireplumber neshub -} - -start_pre() { - checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri -} - -# Point the graph at neswire's sink, by name. -# -# `neswire` registers itself as media.class = Audio/Sink, node.name = neswire -# ("Neswire Cloud Gaming Audio Sink"). With the hardware monitors off it is -# the only sink, so WirePlumber's find-best hook should land on it anyway -- -# this makes it explicit rather than a consequence of there being nothing -# else, so adding a second sink later cannot silently steal the default. -# -# `default.configured.audio.sink` is the metadata WirePlumber's find-selected -# hook reads, and it takes a name; `wpctl set-default` takes a numeric object -# id that changes every boot, which is why this uses pw-metadata instead. -# -# Not fatal if it fails: find-best still has one candidate. A game playing -# into the wrong sink is silent, and a warning here is the only place that -# would say so. -start_post() { - local i=0 - while [ $i -lt 50 ]; do - pw-dump 2>/dev/null | grep -q '"node.name": "neswire"' && break - sleep 0.1 - i=$((i+1)) - done - if ! pw-dump 2>/dev/null | grep -q '"node.name": "neswire"'; then - ewarn "neswire started but its sink never appeared in the graph" - return 0 - fi - - # Wait for WirePlumber to own the `default` metadata before writing to it. - # - # `need wireplumber` only guarantees its script returned, not that it has - # built its objects. Writing too early is silently useless rather than an - # error: pw-metadata creates the object, sets the key, exits 0, and the - # object dies with the client. So wait for the object to exist, and say - # so if it never does. - i=0 - while [ $i -lt 50 ]; do - pw-metadata -n default >/dev/null 2>&1 && break - sleep 0.1 - i=$((i+1)) - done - if ! pw-metadata -n default >/dev/null 2>&1; then - ewarn "no 'default' metadata: WirePlumber is not running, sink not pinned" - return 0 - fi - - pw-metadata -n default 0 default.configured.audio.sink '{ "name": "neswire" }' \ - >/dev/null 2>&1 - - # Read it back. A write that did not stick is the failure this whole - # sequence exists to catch, and it is invisible unless checked. - if pw-metadata -n default 2>/dev/null | grep -q "neswire"; then - einfo "default sink pinned to neswire" - else - ewarn "pinned neswire as default sink but it did not stick" - fi -} diff --git a/build/etc/init.d/pipewire b/build/etc/init.d/pipewire deleted file mode 100644 index e50ded6e..00000000 --- a/build/etc/init.d/pipewire +++ /dev/null @@ -1,45 +0,0 @@ -#!/sbin/openrc-run - -description="PipeWire multimedia daemon (system mode for nestri)" - -nestri_export_env - -: "${NESTRI_UID:=1000}" -: "${NESTRI_USER:=nestri}" - -command="/usr/bin/pipewire" -command_user="${NESTRI_USER}:${NESTRI_USER}" -command_background="yes" -pidfile="/run/pipewire/pipewire.pid" - -export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}" -export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${NESTRI_UID}/bus" - -depend() { - need xdg-runtime dbus - use dbus -} - -start_pre() { - checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/pipewire -} - -# `need pipewire` only waits for this script to return, and with -# command_background that is the moment start-stop-daemon forks -- not the -# moment PipeWire is accepting connections. With rc_parallel="YES" every -# client (wireplumber, neswire) then races the socket and a client that -# loses simply exits. -# -# dbus-session already solved exactly this for the bus socket. Same shape -# here. -start_post() { - local i=0 - while [ ! -S "/run/user/${NESTRI_UID}/pipewire-0" ] && [ $i -lt 50 ]; do - sleep 0.1 - i=$((i+1)) - done - [ -S "/run/user/${NESTRI_UID}/pipewire-0" ] || { - eerror "PipeWire socket never appeared" - return 1 - } -} diff --git a/build/etc/init.d/wireplumber b/build/etc/init.d/wireplumber deleted file mode 100644 index 1c71732a..00000000 --- a/build/etc/init.d/wireplumber +++ /dev/null @@ -1,20 +0,0 @@ -#!/sbin/openrc-run - -description="WirePlumber session manager for PipeWire" - -nestri_export_env - -: "${NESTRI_UID:=1000}" -: "${NESTRI_USER:=nestri}" - -command="/usr/bin/wireplumber" -command_user="${NESTRI_USER}:${NESTRI_USER}" -command_background="yes" -pidfile="/run/pipewire/wireplumber.pid" - -export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}" - -depend() { - need pipewire - after pipewire -} diff --git a/build/etc/init.d/xdg-runtime b/build/etc/init.d/xdg-runtime deleted file mode 100644 index 8d271047..00000000 --- a/build/etc/init.d/xdg-runtime +++ /dev/null @@ -1,44 +0,0 @@ -#!/sbin/openrc-run - -description="Create runtime directories (XDG_RUNTIME_DIR, X11 socket dir..)" - -NESTRI_UID="${NESTRI_UID:-1000}" -NESTRI_USER="${NESTRI_USER:-nestri}" - -depend() { - need localmount - before dbus pipewire -} - -start() { - ebegin "Preparing runtime directories" - - if ! getent passwd "${NESTRI_USER}" >/dev/null 2>&1; then - eerror "User ${NESTRI_USER} does not exist" - return 1 - fi - - # Per-user XDG runtime dir - mkdir -p "/run/user/${NESTRI_UID}" - chown "${NESTRI_USER}:${NESTRI_USER}" "/run/user/${NESTRI_UID}" - chmod 0700 "/run/user/${NESTRI_UID}" - - # X11 socket dir (Xwayland + any X clients expect this) - mkdir -p /tmp/.X11-unix - chown root:root /tmp/.X11-unix - chmod 1777 /tmp/.X11-unix - - # ICE socket dir - some toolkits look for it - mkdir -p /tmp/.ICE-unix - chown root:root /tmp/.ICE-unix - chmod 1777 /tmp/.ICE-unix - - eend $? -} - -stop() { - ebegin "Removing runtime directories" - rm -rf "/run/user/${NESTRI_UID}" - # Don't rm /tmp/.X11-unix on stop - other things may be using it - eend 0 -} diff --git a/build/etc/profile.d/nestri-env.sh b/build/etc/profile.d/nestri-env.sh index fa0a8590..61f69adc 100644 --- a/build/etc/profile.d/nestri-env.sh +++ b/build/etc/profile.d/nestri-env.sh @@ -14,14 +14,22 @@ if [ "$(id -u)" = "1000" ]; then export XDG_SESSION_TYPE="${XDG_SESSION_TYPE:-wayland}" export XDG_SESSION_DESKTOP="${XDG_SESSION_DESKTOP:-nestri}" - # Ensure proper VAAPI driver is used - #export LIBVA_DRIVER_NAME="radeonsi" - - # Force zink usage for OpenGL -> Vulkan translation - #export __GLX_VENDOR_LIBRARY_NAME=mesa - #export MESA_LOADER_DRIVER_OVERRIDE=zink - #export GALLIUM_DRIVER=zink # Ensure standard XDG dirs exist mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_CACHE_HOME}" "${XDG_STATE_HOME}" 2>/dev/null || true fi + +# OpenGL -> Vulkan, for anybody who reaches a shell in here. +# +# Outside the uid test above on purpose, and duplicated from the init on +# purpose. This file is only ever read by a person who got a shell in a box -- +# nothing a box runs is started from a login, so every service and every +# workload is exec'd with a cleared environment and never sees this. The init +# sets the same three for what it starts. +# +# It is here so that a debug shell renders the way a session does. A shell that +# silently has no GL driver is how somebody concludes the image is broken while +# the image is fine. +export __GLX_VENDOR_LIBRARY_NAME=mesa +export MESA_LOADER_DRIVER_OVERRIDE=zink +export GALLIUM_DRIVER=zink diff --git a/build/etc/rc.conf b/build/etc/rc.conf deleted file mode 100644 index 5fc4c575..00000000 --- a/build/etc/rc.conf +++ /dev/null @@ -1,27 +0,0 @@ -# Nestri rootfs OpenRC config - -rc_parallel="YES" -rc_depend_strict="NO" -rc_interactive="NO" -rc_shell_timeout="0" -# Every service's start/fail line, on the host directory that outlives the VM. -# -# Without this a guest nobody can log into reports nothing about its own boot: -# a service that was skipped because a dependency failed looks exactly like a -# service that was never registered. /nestri/logs is a virtiofs share, so the -# record survives the guest — see the fstab entry below. -rc_logger="YES" -rc_log_path="/nestri/logs/rc.log" -rc_verbose="NO" - -# cgroup v2 -rc_cgroup_mode="unified" - -# Don't bother trying to set hostname twice -rc_hotplug="!net.*" - -# Faster sulogin behavior on emergency -rc_shell="/bin/sh" - -# Default umask -umask 022 diff --git a/build/scripts/mkimage.sh b/build/scripts/mkimage.sh index 8baf62d5..2819d439 100644 --- a/build/scripts/mkimage.sh +++ b/build/scripts/mkimage.sh @@ -15,14 +15,14 @@ set -euo pipefail IMAGE="${1:?usage: mkimage.sh [size]}" OUT="${2:?usage: mkimage.sh [size]}" -SIZE="${3:-5G}" +SIZE="${3:-3G}" if [[ "$(id -u)" -eq 0 ]]; then echo "mkimage.sh should run as yourself, not root/sudo — see the comment at the top of this script" >&2 exit 1 fi -CONTAINER_RT="$(command -v docker || command -v podman || true)" +CONTAINER_RT="$(command -v podman || command -v docker || true)" [[ -n "$CONTAINER_RT" ]] || { echo "Neither docker nor podman found in PATH" >&2; exit 1; } sudo -v # cache credentials once, rather than prompting mid-pipeline diff --git a/build/scripts/proton-build.sh b/build/scripts/proton-build.sh new file mode 100755 index 00000000..2e576157 --- /dev/null +++ b/build/scripts/proton-build.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Builds proton-cachyos from the tree proton-fetch.sh laid down. Container-only. +# +# The one thing that matters here is --enable-wow64: it builds wine so that +# 32-bit Windows code runs inside a 64-bit unix process, thunking down to the +# 64-bit host libraries. Without it, Proton needs a complete 32-bit host stack — +# lib32 glibc, a second Mesa built for i686, and a second nescapture layer, +# because a 32-bit game would load the 32-bit Vulkan loader and our 64-bit +# capture layer would be invisible to it. With it, none of that exists. +# +# The cost is that the distro package cannot be used: proton-cachyos-native is +# packaged without the flag, which is exactly why it depends on lib32-*. +set -euo pipefail + +: "${GECKO_VER:?}" +: "${MONO_VER:?}" + +JOBS="${JOBS:-$(nproc)}" +BUILD_NAME="proton-cachyos" +SRC_DIR="/build/proton-cachyos" +BUILD_DIR="/build/build" +OUT_DIR="/artifacts/proton/usr/share/steam/compatibilitytools.d/${BUILD_NAME}" + +[[ -d "${SRC_DIR}" ]] || { echo "no source tree — proton-fetch.sh did not run"; exit 1; } + +# ── Toolchain wrappers ────────────────────────────────── +# Proton's build calls the compiler by GNU triplet. Arch's gcc does not install +# under those names, so stand in for them. The i686 set is generated too: with +# wow64 nothing should reach for it, and if something does, failing on a missing +# 32-bit header beats silently building a 32-bit unix library we then have to +# ship libraries for. +WRAP=/build/wrappers +rm -rf "$WRAP" && mkdir -p "$WRAP" +_wrappers() { + local arch="$1" gccflag="$2" ldflag="$3" asflag="$4" stripfmt="$5" + local l t + for l in ar ranlib nm; do + ln -sf "/usr/bin/gcc-${l}" "${WRAP}/${arch}-pc-linux-gnu-${l}" + done + for t in gcc g++; do + printf '#!/usr/bin/bash\n/usr/bin/%s %s "$@"\n' "$t" "$gccflag" \ + > "${WRAP}/${arch}-pc-linux-gnu-${t}" + chmod 755 "${WRAP}/${arch}-pc-linux-gnu-${t}" + done + printf '#!/usr/bin/bash\n/usr/bin/ld %s "$@"\n' "$ldflag" > "${WRAP}/${arch}-pc-linux-gnu-ld" + printf '#!/usr/bin/bash\n/usr/bin/as %s "$@"\n' "$asflag" > "${WRAP}/${arch}-pc-linux-gnu-as" + printf '#!/usr/bin/bash\n/usr/bin/strip -F %s "$@"\n' "$stripfmt" > "${WRAP}/${arch}-pc-linux-gnu-strip" + chmod 755 "${WRAP}/${arch}-pc-linux-gnu-"{ld,as,strip} +} +_wrappers x86_64 "-m64" "-melf_x86_64" "--64" "elf64-x86-64" +_wrappers i686 "-m32" "-melf_i386" "--32" "elf32-i386" +export PATH="${WRAP}:${PATH}" + +# ── Build ─────────────────────────────────────────────── +# -march=nocona matches the distro packaging: Proton has to run on whatever CPU +# the guest is given, and the VMM does not promise a feature level. +export CFLAGS="-O3 -march=nocona -mtune=core-avx2" +export CXXFLAGS="${CFLAGS}" +export RUSTFLAGS="-C opt-level=3 -C target-cpu=nocona" +export LDFLAGS="-Wl,-O1,--sort-common,--as-needed" +export RUSTUP_TOOLCHAIN=stable + +mkdir -p "${BUILD_DIR}" +cd "${BUILD_DIR}" + +ROOTLESS_CONTAINER="" \ +"${SRC_DIR}/configure.sh" \ + --container-engine="none" \ + --proton-sdk-image="" \ + --build-name="${BUILD_NAME}" \ + --without-extras=all \ + --without-vklayers=all \ + --without-steamrt-depends \ + --without-tts \ + --without-nvidia-libs \ + --enable-wow64 + +# The top-level make is serial by design; SUBJOBS is what it hands to each +# component's build. +SUBJOBS="${JOBS}" make -j1 dist + +# ── Install ───────────────────────────────────────────── +mkdir -p "${OUT_DIR}" +cp -a "${BUILD_DIR}/dist/." "${OUT_DIR}/" + +# Debug symbols in the bundled PE runtimes are dead weight in a guest image. +cd "${OUT_DIR}/files" +find "share/wine/gecko/wine-gecko-${GECKO_VER}-x86" -name '*.dll' -o -name '*.exe' 2>/dev/null \ + | xargs -r i686-w64-mingw32-strip --strip-debug 2>/dev/null || true +find "share/wine/gecko/wine-gecko-${GECKO_VER}-x86_64" -name '*.dll' -o -name '*.exe' 2>/dev/null \ + | xargs -r x86_64-w64-mingw32-strip --strip-debug 2>/dev/null || true +find "share/wine/mono/wine-mono-${MONO_VER}" -name '*.dll' -o -name '*.exe' 2>/dev/null \ + | xargs -r i686-w64-mingw32-strip --strip-debug 2>/dev/null || true + +rm -rf "${BUILD_DIR}" +echo "proton: installed to ${OUT_DIR}" diff --git a/build/scripts/proton-fetch.sh b/build/scripts/proton-fetch.sh new file mode 100755 index 00000000..3b0cfad4 --- /dev/null +++ b/build/scripts/proton-fetch.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Fetches proton-cachyos' source and its bundled runtimes. Container-only. +# +# Deliberately its own script, and its own layer: the submodule checkout runs +# well past ten minutes, and it must not be redone every time a build flag or a +# missing dependency changes. Keep everything that can fail *fast* in +# proton-build.sh instead. +set -euo pipefail + +: "${PROTON_GIT:?}" +: "${PROTON_TAG:?}" +: "${GECKO_VER:?}" +: "${MONO_VER:?}" +: "${XALIA_VER:?}" + +SRC_DIR="/build/proton-cachyos" + +git clone --branch "${PROTON_TAG}" --depth=1 "${PROTON_GIT}" "${SRC_DIR}" +cd "${SRC_DIR}" +# Relative submodule paths resolve against origin, so it has to be the real URL +# even though we cloned by tag. +git remote set-url origin "${PROTON_GIT}" +# No --depth here: submodules are pinned to commits that are often not a branch +# tip. --filter=tree:0 keeps the download down instead. +git submodule update --init --filter=tree:0 --recursive + +# Still needed with wow64: these are PE-side, and a 32-bit Windows program wants +# the 32-bit gecko and mono regardless of how wine is built. +mkdir -p contrib +for url in \ + "https://dl.winehq.org/wine/wine-gecko/${GECKO_VER}/wine-gecko-${GECKO_VER}-x86.tar.xz" \ + "https://dl.winehq.org/wine/wine-gecko/${GECKO_VER}/wine-gecko-${GECKO_VER}-x86_64.tar.xz" \ + "https://github.com/madewokherd/wine-mono/releases/download/wine-mono-${MONO_VER}/wine-mono-${MONO_VER}-x86.tar.xz" \ + "https://github.com/madewokherd/xalia/releases/download/xalia-${XALIA_VER}/xalia-${XALIA_VER}-net48-mono.zip" \ +; do + curl -fL --retry 3 -o "contrib/$(basename "$url")" "$url" +done + +# Proton's cargo rule runs `cargo build --locked --offline`, so every crate has +# to be in CARGO_HOME before the build starts — including the git dependencies, +# which is what the "you are in the offline mode" failure is really saying. The +# error names a URL that is perfectly reachable; the build simply refuses to go +# out and get it. +# +# gst-plugins-rs is the only cargo component in the tree. Both targets are +# fetched: wow64 should mean nothing builds the i386 unix side, but a fetch is +# metadata only and costs almost nothing next to being wrong about that. +# +# CARGO_HOME is left at its default so it lands in this layer and the build +# layer inherits it. +export CARGO_NET_GIT_FETCH_WITH_CLI=true +export RUSTUP_TOOLCHAIN=stable +cd "${SRC_DIR}/gst-plugins-rs" +cargo fetch --locked --target x86_64-unknown-linux-gnu +cargo fetch --locked --target i686-unknown-linux-gnu + +echo "proton: source at ${SRC_DIR}" diff --git a/crates/nesprotocol/src/lifecycle.rs b/crates/nesprotocol/src/lifecycle.rs index 8589eb8b..f00c2915 100644 --- a/crates/nesprotocol/src/lifecycle.rs +++ b/crates/nesprotocol/src/lifecycle.rs @@ -1,14 +1,34 @@ // The lifecycle layer of the control channel between a box and whatever runs -// it: the boot descriptor the guest is handed, and what the guest says back -// about carrying it out. +// it: the document describing the box, the commands that run things inside it, +// and what the guest says back about carrying either out. // // It lives beside the media types for the same reason they live here — one // definition, so the two ends cannot drift from each other silently. // -// Nothing in this module describes *what* the guest runs. A command line, a -// set of share tags, an output geometry, and what an exit means: that is the -// whole vocabulary, and a field that only makes sense for one kind of workload -// does not belong in it. ref(d-0033) +// Nothing in this module describes *what* the guest runs. A set of share tags, +// a command line, and what an exit means: that is the whole vocabulary, and a +// field that only makes sense for one kind of workload does not belong in it. +// ref(d-0033) +// +// # The box outlives what runs in it +// +// The guest init is resident: it mounts what the descriptor names, brings up +// the box's service stack, says so, and then takes commands for as long as the +// box lives. So the descriptor describes the *box* — which shares are mounted +// where — and a command describes an occupant. A box may be launched into many +// times. ref(d-0064) +// +// That is why every launch carries an id and every event about a launch carries +// it back. Without one, a second launch's exit is indistinguishable from the +// first's, which reads as an ended session that keeps billing or a running one +// reported as stopped. +// +// # What is deliberately absent +// +// **Output geometry.** The compositor wraps the workload rather than running as +// a service, so it is started by a launch with that launch's geometry in its own +// argv, and the numbers appear nowhere else. Two sources of truth for one number +// is a worse failure than either choice, because the wrong one is used silently. // // The channel also carries a second layer, which the guest relays as opaque // bytes and never parses. Those types land with the relay that needs them. @@ -34,7 +54,12 @@ pub const CONTROL_PORT: u32 = 7000; /// /// Adding a variant or a field does not need a bump; removing or renaming one /// does. -pub const CONTROL_VERSION: u32 = 2; +/// +/// Version 3 took three fields off the descriptor and put an id on three +/// messages, so a version-2 peer and a version-3 peer do not talk at all. +/// There is deliberately no shim: nothing is deployed, and a shim would be the +/// second definition of this wire that one shared crate exists to prevent. +pub const CONTROL_VERSION: u32 = 3; /// The command to run, and who runs it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -76,14 +101,32 @@ pub struct Mount { pub ro: bool, } -/// The output the compositor should produce. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct Geometry { - pub width: u32, - pub height: u32, - pub fps: u32, - #[serde(default)] - pub hdr: bool, +/// Names one launch, for as long as anything has something to say about it. +/// +/// Minted by the caller and only ever echoed by the guest. A guest that +/// generated these would be naming things the caller then has to correlate +/// against something else. +/// +/// Opaque on purpose: nothing in this layer parses it, and a caller that wants +/// meaning in it can put meaning in it without this crate having an opinion. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct LaunchId(pub String); + +impl LaunchId { + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for LaunchId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } } /// What the workload exiting means for the session. @@ -95,21 +138,33 @@ pub struct OnExit { /// reports the exit and stops, and starting something again is a new /// command from the caller — the only end that can see whether restarting /// is repair or a loop. ref(d-0033) + /// + /// It rides on the launch rather than on the descriptor, because a box that + /// can be launched into repeatedly cannot have one answer to this fixed at + /// boot. ref(d-0064) pub terminal: bool, } -/// Everything the guest is told at boot, in one document. +/// What the box *is*, in one document: the shares it has and where they land. /// /// Sent once, immediately after the handshake, and read once. Deliberately not -/// a conversation: boot configuration is a document, and a document cannot -/// half-arrive. +/// a conversation: this much is a document, and a document cannot half-arrive. +/// What runs *in* the box is a conversation, and a separate one — see +/// [`HostToGuest::Launch`]. ref(d-0064) +/// +/// An empty `mounts` is legitimate. A box with nothing mounted still boots and +/// still brings up its services. +/// +/// `deny_unknown_fields` is load-bearing rather than strictness for its own +/// sake. A descriptor still carrying a command line is a caller that has not +/// been updated, and the default behaviour — ignore what it does not recognise — +/// would mount the shares, silently drop the command, and leave a box that came +/// up correctly and runs nothing. Refusing it says so instead. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BootDescriptor { - pub exec: Exec, #[serde(default)] pub mounts: Vec, - pub geometry: Geometry, - pub on_exit: OnExit, } /// How a workload ended. @@ -145,6 +200,12 @@ impl Exit { #[serde(tag = "type", rename_all = "snake_case")] pub enum GuestToHost { /// First line on the connection, before anything else is read or written. + /// + /// **This is the handshake, not readiness.** It says a connection exists + /// and both ends speak the same version. Whether anything in the box works + /// is [`GuestToHost::Initialized`], which is a different fact and must not + /// be merged with this one — a caller that treats this as readiness has a + /// wait stage that succeeds before the guest has started anything. Ready { protocol_version: u32 }, /// Every share the descriptor named is where it said to put it. Mounted, @@ -154,13 +215,45 @@ pub enum GuestToHost { /// looked at: a share that did not appear and a command that did not run /// are not the same incident. MountFailed { reason: String }, - /// The command the descriptor named is running. - Started, - /// The command could not be run, in the words the operating system used. - StartFailed { reason: String }, - /// The workload the descriptor named has ended. Terminal or not is the - /// descriptor's answer, not this message's. + /// The box's own services are up and it will accept launches. + /// + /// The one fact a caller waits on before it may launch anything. It names + /// what came up, so a log says which — an empty list is a box with no + /// service stack, which is legitimate and worth being able to see. + Initialized { + #[serde(default)] + services: Vec, + }, + /// The box's services could not be brought up, in the words of whatever + /// refused. + /// + /// Kept separate from [`GuestToHost::MountFailed`] and + /// [`GuestToHost::StartFailed`] for the reason those two are separate from + /// each other: a share that did not appear, a box that could not be made, + /// and a command that did not run are three incidents that want three + /// different things looked at. A box in this state cannot be launched into + /// at all, which is what distinguishes it from a refused launch. + InitFailed { reason: String }, + /// A service in the box's own stack exited. + /// + /// **Reported, never repaired.** Nothing else in the guest is watching + /// these, so a death that is not said here is a box that looks healthy and + /// cannot work. Restarting one is a decision for whoever can see whether + /// restarting is repair or a loop, and that is not this end. ref(d-0064) + ServiceDied { + name: String, + #[serde(flatten)] + exit: Exit, + }, + /// The launch with this id is running. + Started { id: LaunchId }, + /// The launch with this id could not be run, in the words the operating + /// system used. + StartFailed { id: LaunchId, reason: String }, + /// The launch with this id has ended. Terminal or not is the launch's own + /// answer, not this message's. WorkloadExited { + id: LaunchId, #[serde(flatten)] exit: Exit, }, @@ -182,8 +275,26 @@ pub enum HostToGuest { #[serde(flatten)] descriptor: Box, }, - /// Stop the workload. Idempotent, and does not end the session. - Stop, + /// Run something in the box. Any number of times, after `initialized`. + /// + /// The caller mints `id` and every message about this launch carries it + /// back. `on_exit` belongs here rather than on the descriptor because a box + /// that can be launched into repeatedly has one answer per launch, not one + /// per boot. ref(d-0064) + Launch { + id: LaunchId, + exec: Exec, + on_exit: OnExit, + }, + /// Stop one launch. Idempotent, and does not end the session. + Stop { id: LaunchId }, + /// Stop one launch and start it again with the same command. + /// + /// **Defined as a kill followed by a launch of the same `Exec`, and nothing + /// more.** No retry, no backoff, no policy of any kind in the guest — it + /// exists as one message only because a caller sending two has the same + /// effect with a worse race in it. The relaunch keeps the id. ref(d-0064) + Restart { id: LaunchId }, /// Shut the guest down. Shutdown, /// Bytes for the workload, relayed. See [`Payload`]. @@ -260,25 +371,21 @@ mod tests { fn descriptor() -> BootDescriptor { BootDescriptor { - exec: Exec { - argv: vec!["/usr/bin/true".into()], - env: BTreeMap::from([("HOME".to_string(), "/mnt/user".to_string())]), - cwd: Some("/mnt/user".into()), - uid: 1000, - gid: 1000, - }, mounts: vec![Mount { tag: "install".into(), at: "/mnt/install".into(), ro: true, }], - geometry: Geometry { - width: 1920, - height: 1080, - fps: 60, - hdr: false, - }, - on_exit: OnExit { terminal: true }, + } + } + + fn exec() -> Exec { + Exec { + argv: vec!["/usr/bin/true".into()], + env: BTreeMap::from([("HOME".to_string(), "/mnt/user".to_string())]), + cwd: Some("/mnt/user".into()), + uid: 1000, + gid: 1000, } } @@ -300,9 +407,122 @@ mod tests { ); } + #[test] + fn a_launch_round_trips_with_its_id() { + let launch = HostToGuest::Launch { + id: LaunchId::new("l-1"), + exec: exec(), + on_exit: OnExit { terminal: true }, + }; + let back: HostToGuest = from_line(&to_line(&launch).unwrap()).unwrap(); + assert_eq!(back, launch); + } + + /// The reason ids exist: a caller has to be able to tell which launch it is + /// being told about, or the second one's exit overwrites the first's record. + #[test] + fn every_event_about_a_launch_carries_the_launch_it_is_about() { + let first = LaunchId::new("l-1"); + let second = LaunchId::new("l-2"); + + let events = [ + GuestToHost::Started { id: first.clone() }, + GuestToHost::StartFailed { + id: first.clone(), + reason: "ENOENT".into(), + }, + GuestToHost::WorkloadExited { + id: first.clone(), + exit: Exit::code(0), + }, + ]; + + for event in events { + let line = to_line(&event).unwrap(); + assert!( + line.contains(first.as_str()) && !line.contains(second.as_str()), + "an event does not say which launch it is about: {line}" + ); + let back: GuestToHost = from_line(&line).unwrap(); + assert_eq!(back, event); + } + } + + /// The descriptor describes the box. A caller still sending a command in it + /// has not been updated, and the cost of accepting one quietly is a box that + /// mounts, comes up, and runs nothing. + #[test] + fn a_descriptor_carrying_a_command_is_refused_rather_than_ignored() { + let stale = r#"{"exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000}, + "mounts":[], + "geometry":{"width":1280,"height":720,"fps":60}, + "on_exit":{"terminal":true}}"#; + let parsed: Result = from_line(stale); + assert!( + parsed.is_err(), + "a descriptor with a command in it parsed: {parsed:?}" + ); + } + + /// Geometry is in the launched argv and nowhere else, so there is no field + /// here for it to disagree with. + #[test] + fn geometry_is_not_on_this_layer() { + let line = to_line(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .unwrap(); + for named in ["width", "height", "fps", "hdr", "geometry"] { + assert!( + !line.contains(named), + "the descriptor names {named}, which belongs in the launch: {line}" + ); + } + } + + /// A dead service is not a dead workload: they are different incidents and + /// want different things looked at. + #[test] + fn a_dead_service_says_which_one_and_how() { + let died = GuestToHost::ServiceDied { + name: "pipewire".into(), + exit: Exit::signal(9), + }; + let line = to_line(&died).unwrap(); + assert!(line.contains("pipewire"), "no service name: {line}"); + assert!( + !line.contains("exit_code"), + "a signalled service has no exit code: {line}" + ); + let back: GuestToHost = from_line(&line).unwrap(); + assert_eq!(back, died); + } + + /// `ready` is the handshake and `initialized` is the box working. A caller + /// waiting on the wrong one succeeds before anything has started. + #[test] + fn readiness_and_initialisation_are_two_messages() { + let ready = to_line(&GuestToHost::Ready { + protocol_version: CONTROL_VERSION, + }) + .unwrap(); + let initialized = to_line(&GuestToHost::Initialized { + services: vec!["dbus".into(), "pipewire".into()], + }) + .unwrap(); + assert_ne!(ready, initialized); + + // An empty stack is legitimate and has to survive the round trip, or a + // box with no services looks like a box that never came up. + let empty = GuestToHost::Initialized { services: vec![] }; + let back: GuestToHost = from_line(&to_line(&empty).unwrap()).unwrap(); + assert_eq!(back, empty); + } + #[test] fn a_signalled_exit_is_not_a_zero_exit() { let signalled = to_line(&GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), exit: Exit::signal(9), }) .unwrap(); @@ -312,6 +532,7 @@ mod tests { ); let clean = to_line(&GuestToHost::WorkloadExited { + id: LaunchId::new("l-1"), exit: Exit::code(0), }) .unwrap(); @@ -381,13 +602,19 @@ mod tests { #[test] fn defaults_cover_what_a_caller_may_leave_out() { - let json = r#"{"exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000}, - "geometry":{"width":1280,"height":720,"fps":30}, - "on_exit":{"terminal":false}}"#; - let parsed: BootDescriptor = from_line(json).unwrap(); + // A box with nothing mounted is a legitimate box. + let parsed: BootDescriptor = from_line("{}").unwrap(); assert!(parsed.mounts.is_empty()); - assert!(parsed.exec.env.is_empty()); - assert_eq!(parsed.exec.cwd, None); - assert!(!parsed.geometry.hdr); + + let launch = r#"{"type":"launch","id":"l-1", + "exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000}, + "on_exit":{"terminal":false}}"#; + let HostToGuest::Launch { id, exec, on_exit } = from_line(launch).unwrap() else { + panic!("not a launch: {launch}") + }; + assert_eq!(id, LaunchId::new("l-1")); + assert!(exec.env.is_empty()); + assert_eq!(exec.cwd, None); + assert!(!on_exit.terminal); } }