github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2026-09-27 21:12:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: resident guest init (#333)

Browse Source 
Get this thing going..







<!-- greptile_comment -->

<!-- greptile_summary -->

<h2><a
href="https://app.greptile.com/api/retrigger?id=63134761"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/RetriggerDark.svg?v=1"><source
media="(prefers-color-scheme: light)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=1"><img
alt="Retrigger"
src="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=1"
align="right"></picture></a>Confidence Score: 5/5</h2>

The PR appears safe to merge; all previous findings are resolved and the
latest readiness change introduces no established actionable regression.

<h3>Summary</h3>

- Establishes required guest filesystems, runtime directories, device
permissions, and service processes.
- Reports initialization and service deaths over the lifecycle channel.
- Supports launch, restart, and shutdown commands for a resident guest.
- Separates service and workload identities and configures per-launch
runtime environments.
- Removes the currently inactive nescope screenshot option and makes
capture-chain verification fail explicitly when compositor readback is
unavailable.
- Reworks the guest image around `nesinit` as PID 1 without a
distribution service manager.

<h3>Diagram</h3>

```mermaid
sequenceDiagram
    participant Host
    participant Init as nesinit
    participant FS as Guest filesystems
    participant Services as Service stack
    participant Workload

    Init->>Host: Ready(protocol version)
    Host->>Init: Boot(mount descriptors)
    Init->>FS: Establish and mount shares
    Init->>Services: Spawn services in order
    Services-->>Init: Required sockets ready
    Init->>Host: Initialized(service names)
    Host->>Init: Launch(id, exec, on_exit)
    Init->>Workload: Spawn with isolated UID/runtime
    Init->>Host: Started(id)
    Workload-->>Init: Exit status
    Init->>Host: WorkloadExited(id, status)
    Host->>Init: Launch / Restart / Shutdown
```

<sub>Reviews (4) · Last reviewed commit: ["fix(nesinit): readiness is a
socket
that..."](https://github.com/nestrilabs/nestri/commit/731d34df9df30463f67963363424cb9487e89196)</sub>

<!-- /greptile_comment -->

---------

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kristian Ollikainen
2026-09-14 14:45:13 +03:00
committed by GitHub
co-authored by DatCaptainHorse Claude Opus 5
parent ec8b13d0c9
commit 8246aa5538
43 changed files with 4448 additions and 1553 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+48 -65
View File
@@ -628,6 +628,12 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core_detect"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
[[package]] [[package]]
name = "cpufeatures" name = "cpufeatures"
version = "0.2.17" version = "0.2.17"
@@ -1022,11 +1028,17 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
[[package]] [[package]]
name = "encoding_rs" name = "encoding_rs"
version = "0.8.35" version = "0.8.40"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" checksum = "2a7a45518d2863d18aa47f4a0cf9faec2aa4304cc09df5e41299f276b3ad135e"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"core_detect",
"multiversion",
"multiversion_no_op",
"rustversion",
"scopeguard",
"simdutf8",
] ]
[[package]] [[package]]
@@ -1243,28 +1255,6 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "gbm"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a"
dependencies = [
"bitflags 2.13.1",
"drm",
"drm-fourcc",
"gbm-sys",
"libc",
]
[[package]]
name = "gbm-sys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "generator" name = "generator"
version = "0.8.9" version = "0.8.9"
@@ -1349,17 +1339,6 @@ dependencies = [
"polyval", "polyval",
] ]
[[package]]
name = "gl_generator"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d"
dependencies = [
"khronos_api",
"log",
"xml-rs",
]
[[package]] [[package]]
name = "glob" name = "glob"
version = "0.3.4" version = "0.3.4"
@@ -2156,12 +2135,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "khronos_api"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc"
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.5.0" version = "1.5.0"
@@ -2371,6 +2344,34 @@ dependencies = [
"uuid", "uuid",
] ]
[[package]]
name = "multiversion"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201"
dependencies = [
"multiversion-macros",
"target-features",
]
[[package]]
name = "multiversion-macros"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"target-features",
]
[[package]]
name = "multiversion_no_op"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
[[package]] [[package]]
name = "n0-error" name = "n0-error"
version = "1.0.1" version = "1.0.1"
@@ -3102,24 +3103,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "pixman"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cea217d496c19ac0a8e502b37078e1f683d16344adee9eb247a5d57c165e1edf"
dependencies = [
"drm-fourcc",
"paste",
"pixman-sys",
"thiserror 1.0.69",
]
[[package]]
name = "pixman-sys"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a0483e89e81d7915defe83c51f23f6800594d64f6f4a21253ce87fd8444ada"
[[package]] [[package]]
name = "pkcs8" name = "pkcs8"
version = "0.11.0" version = "0.11.0"
@@ -3905,7 +3888,6 @@ dependencies = [
"atomic_float", "atomic_float",
"bitflags 2.13.1", "bitflags 2.13.1",
"calloop", "calloop",
"cc",
"cgmath", "cgmath",
"cursor-icon", "cursor-icon",
"downcast-rs", "downcast-rs",
@@ -3914,14 +3896,9 @@ dependencies = [
"drm-fourcc", "drm-fourcc",
"encoding_rs", "encoding_rs",
"errno", "errno",
"gbm",
"gl_generator",
"indexmap", "indexmap",
"input", "input",
"libc", "libc",
"libloading",
"pixman",
"pkg-config",
"profiling", "profiling",
"rand 0.9.5", "rand 0.9.5",
"rustix 1.1.4", "rustix 1.1.4",
@@ -4103,6 +4080,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "target-features"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"
[[package]] [[package]]
name = "target-lexicon" name = "target-lexicon"
version = "0.13.5" version = "0.13.5"
apps/nescapture/scripts/verify-chain.sh
+15
View File
@@ -36,6 +36,21 @@ done
echo "building…" echo "building…"
cargo build --release -p nescope -p nescapture --manifest-path "$ROOT/Cargo.toml" >/dev/null cargo build --release -p nescope -p nescapture --manifest-path "$ROOT/Cargo.toml" >/dev/null
# The whole method here is two independent instruments on the same frames, and
# the second one is the compositor's own readback. While nescope's screenshot
# path is commented out there is no second instrument, so this script cannot
# make the comparison it exists for. Said here rather than fifty lines later as
# "compositor readback produced no frames", which reads like a capture bug.
#
# Asked of the binary rather than hard-coded, so this comes back by itself on
# the commit that brings the path back.
if ! "$ROOT/target/release/nescope" --help 2>&1 | grep -q -- --screenshot-ipc; then
echo "this nescope has no --screenshot-ipc, so there is no readback to compare" >&2
echo "the encoded frames against; the GPU readback path in nescope is" >&2
echo "commented out. See apps/nescope/src/main.rs." >&2
exit 1
fi
LAYER="$ROOT/target/release/libnescapture_layer.so" LAYER="$ROOT/target/release/libnescapture_layer.so"
MANIFEST_DIR="$WORK/implicit_layer.d" MANIFEST_DIR="$WORK/implicit_layer.d"
mkdir -p "$MANIFEST_DIR" mkdir -p "$MANIFEST_DIR"
apps/nescope/Cargo.toml
+2 -4
View File
@@ -18,13 +18,11 @@ smithay = { version = "0.7", default-features = false, features = [
"backend_drm", "backend_drm",
"desktop", "desktop",
"backend_libinput", "backend_libinput",
"renderer_pixman", # needed for on_commit_buffer_handler #"renderer_pixman", # needed for on_commit_buffer_handler
# Reading a dmabuf back to the CPU. nescope still does not composite or # Reading a dmabuf back to the CPU. nescope still does not composite or
# present anything -- this is import-and-copy only, so a client that # present anything -- this is import-and-copy only, so a client that
# renders on the GPU can be screenshotted like any other. # renders on the GPU can be screenshotted like any other.
"renderer_gl", #"backend_gbm",
"backend_egl",
"backend_gbm",
] } ] }
# Wayland client – connects to the host compositor to forward buffers. # Wayland client – connects to the host compositor to forward buffers.
apps/nescope/src/main.rs
+11 -14
View File
@@ -60,15 +60,15 @@ use smithay::reexports::wayland_server::Display;
use smithay::wayland::socket::ListeningSocketSource; use smithay::wayland::socket::ListeningSocketSource;
mod focus; mod focus;
mod gpu_readback; //mod gpu_readback;
mod handlers; mod handlers;
mod hdr; mod hdr;
mod input; mod input;
mod input_ipc; mod input_ipc;
mod libinput_backend; mod libinput_backend;
mod protocols; mod protocols;
mod screenshot_ipc; //mod screenshot_ipc;
mod screenshot_wire; //mod screenshot_wire;
mod state; mod state;
mod xwm; mod xwm;
@@ -125,14 +125,11 @@ struct Args {
)] )]
input_ipc: String, input_ipc: String,
/// Path to the hub's screenshot IPC socket (nescope connects as client). // There is no `--screenshot-ipc`. The path it named is commented out below,
/// // and an option that is accepted and does nothing is worse than one that is
/// Optional, and absent means the feature is simply off: it exists for // refused: a caller passing it gets no error, no capture, and nothing to
/// clients that are not games — a Steam login screen has no Vulkan frames // read that says which. It comes back with the code, or not at all.
/// for `nescapture` to take, so its pixels can only come from here. //
#[arg(long, env = "NESCOPE_SCREENSHOT_IPC")]
screenshot_ipc: Option<String>,
/// GPU render device (e.g. /dev/dri/renderD128). Sets VK_DRIVER_FILES /// GPU render device (e.g. /dev/dri/renderD128). Sets VK_DRIVER_FILES
/// for the game so it uses the same GPU. /// for the game so it uses the same GPU.
#[arg(long, env = "NESCOPE_RENDER_DEVICE")] #[arg(long, env = "NESCOPE_RENDER_DEVICE")]
@@ -317,13 +314,13 @@ fn main() {
// The GPU to import dmabufs on for screenshots. Same device the game is // The GPU to import dmabufs on for screenshots. Same device the game is
// pointed at, because a buffer the game produced can only be imported on // pointed at, because a buffer the game produced can only be imported on
// the device that made it. // the device that made it.
gpu_readback::set_render_device(args.render_device.clone()); //gpu_readback::set_render_device(args.render_device.clone());
// ── Screenshot IPC source ──────────────────────────────────────────── // ── Screenshot IPC source ────────────────────────────────────────────
// Same dial-out shape as the input socket below, so the hub is the // Same dial-out shape as the input socket below, so the hub is the
// listener and there is no race against a socket that does not exist yet. // listener and there is no race against a socket that does not exist yet.
// Absent means the feature is off, which is the normal case for a game. // Absent means the feature is off, which is the normal case for a game.
if let Some(path) = args.screenshot_ipc.clone() { /*if let Some(path) = args.screenshot_ipc.clone() {
match screenshot_ipc::ScreenshotIpcSource::connect(&path) { match screenshot_ipc::ScreenshotIpcSource::connect(&path) {
Ok(source) => match source.try_clone_writer() { Ok(source) => match source.try_clone_writer() {
Ok(mut writer) => { Ok(mut writer) => {
@@ -357,7 +354,7 @@ fn main() {
}, },
Err(e) => tracing::warn!("Failed to connect to screenshot IPC socket {path}: {e}"), Err(e) => tracing::warn!("Failed to connect to screenshot IPC socket {path}: {e}"),
} }
} }*/
// ── Input IPC source ───────────────────────────────────────────────── // ── Input IPC source ─────────────────────────────────────────────────
// Connect to the neshub input socket and feed events into the // Connect to the neshub input socket and feed events into the
apps/nesinit/README.md
+59 -16
View File
@@ -2,8 +2,9 @@
PID 1 inside a box. PID 1 inside a box.
A microVM has no init unless something is it. Three of the jobs are nobody A microVM has no init unless something is it, and in a box nothing else is:
else's, and this is all of them: there is no service manager in the image and no init scripts. Four jobs, and
this is all of them:
- **Reaping.** A process whose parent dies is reparented to PID 1. Without a - **Reaping.** A process whose parent dies is reparented to PID 1. Without a
reaper, every orphan the workload leaves behind holds a pid and a slot in the reaper, every orphan the workload leaves behind holds a pid and a slot in the
@@ -11,25 +12,46 @@ else's, and this is all of them:
- **Ordered shutdown.** The workload stops first and alone, then everything - **Ordered shutdown.** The workload stops first and alone, then everything
else, then the disks are flushed and the machine is powered off. An init that else, then the disks are flushed and the machine is powered off. An init that
returns leaves a guest running with nothing in it. returns leaves a guest running with nothing in it.
- **The box's own services.** The bus, audio, and the transport that carries a
session out, started in order from a table compiled into this binary. There
is no unit format and no directory of files to read: the services in a box
are fixed, and running on any distribution comes from depending on none of
their init scripts rather than from being configurable.
- **The guest end of the control channel.** One vsock connection out, carrying - **The guest end of the control channel.** One vsock connection out, carrying
what to run in and what happened back. what to run in and what happened back.
It does not know what it is running. It is handed a command line, a set of It does not know what it is running. It is handed a set of shares, and then
shares and what an exit means; there is no code path here that branches on commands naming what to run and what an exit means; there is no code path here
which workload it started, and there is not meant to be. that branches on which workload it started, and there is not meant to be.
**A box outlives what runs in it.** Init mounts, brings the services up, says
it is ready, and then takes commands for as long as the box lives — so this
image on its own runs nothing at all, and a box may be launched into more than
once.
### The channel ### The channel
The guest dials out on a fixed vsock port and speaks first: The guest dials out on a fixed vsock port and speaks first:
``` ```
guest → { "type": "ready", "protocol_version": 2 } guest → { "type": "ready", "protocol_version": 3 }
guest ← { "type": "boot", "exec": {...}, "mounts": [...], "geometry": {...}, "on_exit": {...} } guest ← { "type": "boot", "mounts": [...] }
guest → { "type": "mounted" } guest → { "type": "mounted" }
guest → { "type": "started" } guest → { "type": "initialized", "services": ["dbus-system", ...] }
guest → { "type": "workload_exited", "exit_code": 0 } guest ← { "type": "launch", "id": "…", "exec": {...}, "on_exit": {...} }
guest → { "type": "started", "id": "…" }
guest → { "type": "workload_exited", "id": "…", "exit_code": 0 }
``` ```
`ready` is the handshake and `initialized` is the box working. They are two
facts and must not be treated as one: a caller that waits on the first has a
wait that succeeds before anything in the guest has started.
Every launch carries an id and every message about a launch carries it back.
Without one, a second launch's exit is indistinguishable from the first's —
which reads at the far end as a finished session still running, or a running
one reported as stopped.
Newline-delimited JSON. Dialling out rather than being connected to is worth Newline-delimited JSON. Dialling out rather than being connected to is worth
keeping for two reasons: the listener is up before the VM starts, so nothing keeping for two reasons: the listener is up before the VM starts, so nothing
races a booting kernel and nothing has to retry, and the connection races a booting kernel and nothing has to retry, and the connection
@@ -109,19 +131,40 @@ later for no visible reason.
### It reports; it does not supervise ### It reports; it does not supervise
When the workload ends, the exit goes up the channel and the session is over. When a launch ends, the exit goes up the channel. `on_exit` says what that exit
`on_exit` says what that exit *means* — whether it ends the session — and *means* — whether it ends the session or leaves the box up to be launched into
nothing here restarts anything. Starting something again is a decision for the again — and nothing here restarts anything of its own accord. Starting
end that can see whether restarting is repair or a loop. something again is a decision for the end that can see whether restarting is
repair or a loop.
`restart` exists as one message and is defined as exactly that: a kill followed
by a launch of the same command, keeping the id, with no retry and no backoff.
It is one message rather than two only because a caller sending two has the
same effect with a worse race in it.
The same rule covers the box's own services. One that dies is **reported and
left dead** — nothing else in the guest is watching them, so a death that is
not said here is a box that looks healthy and cannot work.
**One launch at a time.** A launch arriving while one is running is refused,
carrying the id it was asked for, rather than queued or silently replacing it:
a box has one screen, so a second concurrent launch has nowhere to draw.
A signalled workload is reported as signalled, with no exit code. Reporting A signalled workload is reported as signalled, with no exit code. Reporting
`0` for a killed process would make a kill look like a clean run. `0` for a killed process would make a kill look like a clean run.
### What is not here yet ### What is not here yet
`geometry` is carried and parsed but nothing consumes it: nesinit does not **It has never been PID 1 of anything.** Every line of this is written for a
start the guest's own services yet. `ticket` exists as a message with no box and all of it has been tested outside one. It runs perfectly well as an
producer wired to it. ordinary process — it warns rather than fails when it is not PID 1 — which is
how most of it is exercised, and is also how a guest that will not boot gets
debugged: `init=/bin/bash` on the kernel command line, then run this by hand
and watch it fail.
Output geometry is deliberately absent from this layer. The compositor is
started by a launch, with that launch's geometry in its own arguments, so the
numbers appear in one place rather than two that can disagree.
### Testing ### Testing
apps/nesinit/src/filesystems.rs
+135 -23
View File
@@ -31,6 +31,12 @@ struct Early {
/// so a device node or a setuid bit appearing in one did not come from us. /// so a device node or a setuid bit appearing in one did not come from us.
const NOSUID_NODEV: libc::c_ulong = libc::MS_NOSUID | libc::MS_NODEV; const NOSUID_NODEV: libc::c_ulong = libc::MS_NOSUID | libc::MS_NODEV;
// Every tmpfs below is capped, and the caps are load-bearing rather than
// tidiness. A tmpfs with no `size=` may grow to half of RAM, and the RAM in
// question is the box's whole allowance — so an uncapped `/tmp` is a workload
// that can OOM the box it runs in by writing files. The numbers are carried
// over from the mount table this replaced, where they were already considered.
const EARLY: &[Early] = &[ const EARLY: &[Early] = &[
Early { Early {
source: "proc", source: "proc",
@@ -41,6 +47,38 @@ const EARLY: &[Early] = &[
cost: "this process cannot make itself ineligible for the OOM killer, \ cost: "this process cannot make itself ineligible for the OOM killer, \
and nothing in the guest can read its own state", and nothing in the guest can read its own state",
}, },
// Usually already there: a kernel built with `CONFIG_DEVTMPFS_MOUNT` mounts
// this before init runs. Listed anyway because the check below skips what
// is already mounted, so the entry costs nothing when the kernel did it and
// is the difference between a working box and one with no device nodes when
// it did not. Without `nodev`, obviously — device nodes are the point.
Early {
source: "devtmpfs",
target: "/dev",
fstype: "devtmpfs",
flags: libc::MS_NOSUID,
data: "mode=755",
cost: "there are no device nodes at all, so nothing can open the GPU",
},
Early {
source: "devpts",
target: "/dev/pts",
fstype: "devpts",
flags: NOSUID_NODEV | libc::MS_NOEXEC,
data: "mode=620,gid=5,ptmxmode=666",
cost: "nothing that wants a terminal can allocate one",
},
// The image creates this directory, and the mode is the load-bearing part:
// a workload and the box's own services are different users, and shared
// memory between them is how a Vulkan client hands buffers around.
Early {
source: "tmpfs",
target: "/dev/shm",
fstype: "tmpfs",
flags: NOSUID_NODEV,
data: "mode=1777,size=256m",
cost: "anything using shared memory fails, which includes most graphics",
},
Early { Early {
source: "sysfs", source: "sysfs",
target: "/sys", target: "/sys",
@@ -59,7 +97,7 @@ const EARLY: &[Early] = &[
flags: NOSUID_NODEV, flags: NOSUID_NODEV,
// The sticky bit, because the workload does not run as this process // The sticky bit, because the workload does not run as this process
// does and what it binds here is its own. // does and what it binds here is its own.
data: "mode=1777", data: "mode=1777,size=64m",
cost: "whatever serves this session's address cannot bind its socket, \ cost: "whatever serves this session's address cannot bind its socket, \
so the session never gets one", so the session never gets one",
}, },
@@ -74,20 +112,51 @@ const EARLY: &[Early] = &[
// Octal, and without a leading zero on purpose: the kernel parses a // Octal, and without a leading zero on purpose: the kernel parses a
// tmpfs mode as octal either way, and this is the spelling `mount` // tmpfs mode as octal either way, and this is the spelling `mount`
// itself documents. // itself documents.
data: "mode=755", data: "mode=755,size=32m",
cost: "there is nowhere for a runtime socket to live, so neither the \ cost: "there is nowhere for a runtime socket to live, so neither the \
payload relay nor this session's address can be served", payload relay nor this session's address can be served",
}, },
// The tree a session's shares are mounted into.
//
// A share's target is named by the descriptor and may be any path under
// here, so something has to create directories on a root that is read-only
// by design. That is what this is: `workload::mount` calls `create_dir_all`
// on each target, which fails with `EROFS` unless the tree it is creating
// in is writable.
//
// **This entry used to be forbidden, and the reason it was forbidden is
// gone.** A test here asserted that `/nestri` must never be mounted over,
// because a fresh tmpfs would hide the install, the user state and the work
// directory that the image had prepared underneath. That was true of the
// image that shipped those directories and an `fstab` that mounted into
// them. The image prepares nothing here now — the host names every share
// and every target — so there is nothing left to hide, and the rule had
// become a guard on a hazard that was deleted with the image that had it.
// ref(d-0064)
//
// Small on purpose. Everything real is mounted *over* this, so what remains
// is a handful of empty directories; the cap matters for the case where a
// share fails to mount and a workload writes to the bare mount point
// instead, which would otherwise be RAM the box cannot get back.
Early {
source: "tmpfs",
target: "/nestri",
fstype: "tmpfs",
flags: NOSUID_NODEV,
data: "mode=755,size=4m",
cost: "no share can be mounted, because its target cannot be created on a read-only root",
},
// The relay's own directory, and it is deliberately **not** in the tree the // The relay's own directory, and it is deliberately **not** in the tree the
// session's shares live in. // session's shares live in.
// //
// It was, and that was wrong in a way no test here would have caught: a // It was, and moving it out stays right for a reason that outlived the one
// fresh tmpfs over the share tree hides every directory the image prepared // originally given. The first reason was that a tmpfs over the share tree
// underneath it — the install, the user state, the work directory, and the // would hide what the image had prepared there; that image is gone and the
// mount point the log share is attached to from `fstab`. The box then has a // entry above now mounts that tree deliberately. The reason that remains is
// socket and none of the places its workload expects to find its files, and // ownership: this directory is written by this process and by nothing else,
// the exact-path check below cannot notice, because what `fstab` mounts is // which is what makes the socket in it unreplaceable. The share tree is
// a directory *inside* that tree rather than the tree itself. // mounted into by the host's own shares, so a relay socket living there
// would sit in a tree a workload's own share can be attached over.
// //
// Owned by this process and writable by nothing else, which is what makes // Owned by this process and writable by nothing else, which is what makes
// the socket in it unreplaceable. The workload reaches it because the // the socket in it unreplaceable. The workload reaches it because the
@@ -98,10 +167,22 @@ const EARLY: &[Early] = &[
target: crate::payload::DIRECTORY, target: crate::payload::DIRECTORY,
fstype: "tmpfs", fstype: "tmpfs",
flags: NOSUID_NODEV | libc::MS_NOEXEC, flags: NOSUID_NODEV | libc::MS_NOEXEC,
data: "mode=755", // One socket lives here, so this is as small as a tmpfs usefully gets.
data: "mode=755,size=1m",
cost: "the payload relay cannot bind, so nothing reaches the workload \ cost: "the payload relay cannot bind, so nothing reaches the workload \
over the channel", over the channel",
}, },
// The root is read-only and some things write here whether or not anything
// reads it back. A box's real logs leave over the control channel; this is
// so that a library writing a file does not fail on `EROFS` instead.
Early {
source: "tmpfs",
target: "/var/log",
fstype: "tmpfs",
flags: NOSUID_NODEV | libc::MS_NOEXEC,
data: "mode=755,size=16m",
cost: "anything that writes a log file fails on a read-only root",
},
]; ];
/// Mount what the rest of this component assumes is already there. /// Mount what the rest of this component assumes is already there.
@@ -220,6 +301,25 @@ mod tests {
} }
} }
/// **Every tmpfs is capped.** One without a `size=` may grow to half of RAM,
/// and the RAM in question is the whole box's — so an uncapped `/tmp` hands
/// a workload a way to OOM the box it is running in by writing files. The
/// failure looks like a box that died under load rather than like a missing
/// mount option, which is why this is a test.
#[test]
fn no_tmpfs_is_unbounded() {
for early in EARLY {
if early.fstype != "tmpfs" {
continue;
}
assert!(
early.data.contains("size="),
"{} is an uncapped tmpfs",
early.target
);
}
}
/// `/proc` is mounted before `mountinfo` is read, so it has to be first. /// `/proc` is mounted before `mountinfo` is read, so it has to be first.
#[test] #[test]
fn proc_is_the_first_entry() { fn proc_is_the_first_entry() {
@@ -243,20 +343,32 @@ mod tests {
} }
/// **Nothing here may be mounted over the tree the session's shares live /// **Nothing here may be mounted over the tree the session's shares live
/// in.** A fresh tmpfs there hides every directory the image prepared /// in**, and the share tree itself is mounted so that targets under it can
/// underneath — the install, the user state, the work directory, and the /// be created at all.
/// mount point the log share attaches to — and the exact-path check cannot ///
/// notice, because what is mounted from `fstab` is a directory inside that /// This replaces a test that asserted the exact opposite — that `/nestri`
/// tree rather than the tree itself. So a box would come up with a socket /// must never be mounted over — on the grounds that a tmpfs there would
/// and without any of the places its workload looks for its files. /// hide the install, the user state and the work directory the image had
/// prepared. The image that prepared them no longer exists; the host names
/// every share and every target now, and a read-only root cannot have a
/// directory created on it. Measured 2026-09-11: without this entry the
/// first real boot refused its own descriptor with
/// `/nestri/payload: Read-only file system`.
#[test] #[test]
fn the_share_tree_is_never_mounted_over() { fn the_share_tree_is_writable_and_the_relay_is_not_inside_it() {
for early in EARLY { let tree = EARLY
assert_ne!( .iter()
early.target, "/nestri", .find(|e| e.target == "/nestri")
"this hides the directories the image prepared for a session" .expect("a share's target cannot be created without this");
); assert_eq!(tree.fstype, "tmpfs");
} assert!(
tree.data.contains("size="),
"an uncapped tmpfs here is RAM a box cannot get back"
);
assert!(
!crate::payload::DIRECTORY.starts_with("/nestri/"),
"the relay's socket would sit in a tree a share can be mounted over"
);
} }
/// The relay's directory is the one this cannot hardcode: it belongs to /// The relay's directory is the one this cannot hardcode: it belongs to
apps/nesinit/src/lib.rs
+9 -3
View File
@@ -4,14 +4,20 @@
// nobody else's: reaping whatever the workload orphans, turning a signal into // nobody else's: reaping whatever the workload orphans, turning a signal into
// an ordered shutdown, and being the guest end of the one channel out. // an ordered shutdown, and being the guest end of the one channel out.
// //
// It does not know what it is running. It is handed a command, a set of shares // It does not know what it is running. It is handed a set of shares, and then
// and what an exit means, and it carries that out; a field that only makes // commands naming what to run and what an exit means, and it carries those out;
// sense for one kind of workload cannot reach it. ref(d-0033) // a field that only makes sense for one kind of workload cannot reach it.
// ref(d-0033)
//
// It is also the box's only init: there is no service manager in the image, so
// the box's own services come up from a table in this binary. ref(d-0064)
pub mod filesystems; pub mod filesystems;
pub mod payload; pub mod payload;
pub mod reap; pub mod reap;
pub mod services;
pub mod session; pub mod session;
pub mod shutdown; pub mod shutdown;
pub mod system;
pub mod ticket; pub mod ticket;
pub mod workload; pub mod workload;
apps/nesinit/src/main.rs
+52 -2
View File
@@ -9,6 +9,7 @@ use std::time::Duration;
use nesinit::payload::{self, Ports}; use nesinit::payload::{self, Ports};
use nesinit::reap::{self, Waiters}; use nesinit::reap::{self, Waiters};
use nesinit::services::Stack;
use nesinit::session::{self, Outcome}; use nesinit::session::{self, Outcome};
use nesinit::shutdown::{self, Machine}; use nesinit::shutdown::{self, Machine};
use nesinit::ticket; use nesinit::ticket;
@@ -45,6 +46,13 @@ fn main() -> anyhow::Result<()> {
// is no `/proc` to score this process in and nowhere to put a socket. // is no `/proc` to score this process in and nowhere to put a socket.
nesinit::filesystems::establish(); nesinit::filesystems::establish();
// Everything a distribution's init scripts used to do, and nothing else is
// going to: a hostname, the box's address, the directories a session's
// sockets live in, and device nodes something is allowed to open. Before
// the runtime, so the few processes it starts are waited for directly
// rather than racing the reaper into existence. ref(d-0064)
nesinit::system::prepare();
// Both before anything is started, so nothing can be orphaned or scored // Both before anything is started, so nothing can be orphaned or scored
// in the window where neither is true yet. // in the window where neither is true yet.
if let Err(error) = reap::become_subreaper() { if let Err(error) = reap::become_subreaper() {
@@ -80,7 +88,18 @@ fn main() -> anyhow::Result<()> {
// Reached however the session ended, including an error: an init that // Reached however the session ended, including an error: an init that
// returns leaves the guest running with nothing in it. // returns leaves the guest running with nothing in it.
let mut machine = Guest { workload }; //
// `is_init` is what makes that safe to say. The three machine-wide steps
// below — signal everything, kill everything, power off — are correct for
// PID 1 of a box and catastrophic anywhere else, and this program is meant
// to be runnable by hand: that is how most of it is tested and it is the
// documented way to debug a guest that will not boot. Run as root outside a
// box, the old path reached `kill(-1)` and `reboot` the moment the control
// channel could not be dialled.
let mut machine = Guest {
workload,
is_init: pid == 1,
};
shutdown::ordered(&mut machine, GRACE); shutdown::ordered(&mut machine, GRACE);
unreachable!("power_off does not return"); unreachable!("power_off does not return");
} }
@@ -124,8 +143,13 @@ async fn guest(waiters: &Waiters, workload: &mut Process) -> anyhow::Result<Outc
untrusted.clone(), untrusted.clone(),
)); ));
// The box's own services. Nothing is started here: bring-up happens once
// the descriptor has been carried out, because a box whose shares are not
// where they belong is not a box worth starting a stack in.
let mut services = Stack::new(waiters.clone());
let outcome = tokio::select! { let outcome = tokio::select! {
outcome = session::run(channel, workload, &mut ports, &mut found_rx, &untrusted) => outcome?, outcome = session::run(channel, workload, &mut services, &mut ports, &mut found_rx, &untrusted) => outcome?,
signal = asked_to_stop() => { signal = asked_to_stop() => {
signal?; signal?;
tracing::info!("asked to stop"); tracing::info!("asked to stop");
@@ -170,6 +194,9 @@ async fn asked_to_stop() -> std::io::Result<()> {
/// The machine, for real. /// The machine, for real.
struct Guest { struct Guest {
workload: Process, workload: Process,
/// Whether this process is PID 1, and therefore whether the steps that act
/// on *the machine* rather than on our own children may be taken at all.
is_init: bool,
} }
impl Machine for Guest { impl Machine for Guest {
@@ -194,6 +221,13 @@ impl Machine for Guest {
} }
fn signal_rest(&mut self, grace: Duration) { fn signal_rest(&mut self, grace: Duration) {
if !self.is_init {
tracing::warn!(
"not PID 1, so not signalling every process: outside a box that \
is this machine's processes, not this box's"
);
return;
}
// -1 is every process this one may signal, which as PID 1 is all of // -1 is every process this one may signal, which as PID 1 is all of
// them but itself. The workload has already stopped by here. // them but itself. The workload has already stopped by here.
unsafe { libc::kill(-1, libc::SIGTERM) }; unsafe { libc::kill(-1, libc::SIGTERM) };
@@ -201,15 +235,31 @@ impl Machine for Guest {
} }
fn kill_rest(&mut self) { fn kill_rest(&mut self) {
if !self.is_init {
return;
}
unsafe { libc::kill(-1, libc::SIGKILL) }; unsafe { libc::kill(-1, libc::SIGKILL) };
wait_for_quiet(Duration::from_secs(1)); wait_for_quiet(Duration::from_secs(1));
} }
fn flush_disks(&mut self) { fn flush_disks(&mut self) {
// Harmless anywhere, so it is not guarded: the worst it does outside a
// box is flush somebody's page cache.
unsafe { libc::sync() }; unsafe { libc::sync() };
} }
fn power_off(&mut self) { fn power_off(&mut self) {
if !self.is_init {
// Everything this process started has been stopped by here, which
// is the whole of what it may take responsibility for when it is
// not the machine's init. What it prepared — the mounts, the
// runtime directories — is deliberately left behind, because that
// is exactly what makes a hand-run useful: run it, watch it fail to
// reach a control channel that is not there, and then poke at a
// guest that is otherwise set up.
tracing::warn!("not PID 1, so not powering the machine off");
std::process::exit(1);
}
// SAFETY: reboot is the only way out of a guest whose init is done. // SAFETY: reboot is the only way out of a guest whose init is done.
unsafe { libc::reboot(libc::RB_POWER_OFF) }; unsafe { libc::reboot(libc::RB_POWER_OFF) };
// Reached only if the guest refused to power off, which no caller can // Reached only if the guest refused to power off, which no caller can
apps/nesinit/src/services.rs
+959
View File
@@ -0,0 +1,959 @@
// The box's own services, and the table that is the whole of what a box runs
// before anything is launched into it.
//
// There is no service manager in a box and no init scripts, so this is what
// replaces them. ref(d-0064)
//
// # Why the table is in the binary
//
// A unit format would make this configurable, and nothing wants to configure
// it: the services in a box are ours, they are the same in every box, and a
// table in a file is a table two images can disagree about. Being able to run
// on any distribution comes from depending on no distribution's init scripts,
// which this does — not from being told what to start.
//
// # What is deliberately not here
//
// **The compositor.** It wraps the workload and is started by a launch, with
// that launch's geometry, and dies with it. A compositor in this table would be
// a compositor with no geometry to come up with.
//
// **Restarting.** A service that dies is reported up the channel and left dead.
// Whether restarting it is repair or a loop is not visible from inside the box.
//
// **Readiness beyond "its socket exists".** A service that names a socket is
// waited for until that socket is there; nothing here asks it a question or
// waits for a bus name. That much was in the init scripts this replaced and
// leaving it out was a regression: `spawn` returns at fork, so without it the
// bus's client is started before the bus is listening and audio comes up
// against nothing. It races rather than failing — which is the shape this
// component is least able to see — and the cost of losing the race is a box
// that boots, reports itself ready, and has no sound.
//
// That is as far as it goes. Health checks, restarts and readiness that is not
// a file on a path would make this a supervisor; see the decision's own
// falsification list.
use std::os::unix::process::CommandExt;
use nesprotocol::lifecycle::Exit;
use tokio::sync::mpsc::{Receiver, Sender};
use crate::reap::{Waiters, Watched};
use crate::workload::Failure;
/// A service that died, and how.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Died {
pub name: String,
pub exit: Exit,
}
/// The box's service stack, as the session sees it.
///
/// A trait because the interesting behaviour is the session's — that a death is
/// reported and not repaired, that a stack which will not come up refuses the
/// box — and none of that needs a process to assert.
pub trait Services {
/// Bring the stack up in order, and name what came up.
///
/// Called once, after the shares are mounted and before anything may be
/// launched. An empty stack is legitimate: a box with no services still
/// boots, and a caller can still launch something that needs none.
fn bring_up(&mut self) -> Result<Vec<String>, Failure>;
/// Deaths, as they happen.
///
/// A channel rather than a future so the session can wait on it beside the
/// control channel, the relay and the address carrier without any of them
/// being able to starve the others.
fn deaths(&mut self) -> &mut Receiver<Died>;
}
// There is deliberately no ordered `stop_all`, and reverse-order stopping buys
// nothing on a machine that is about to be powered off. What there is instead
// is a `Drop` that signals the children this stack started — because the
// argument for having nothing at all was "this process is PID 1 and the ordered
// shutdown signals every process", and that is true of a box and false of the
// way this program is run by hand to debug one. Outside PID 1 the old path left
// a bus, an audio server and a hub running with sockets nobody was serving.
/// One service, and everything about starting it.
///
/// `env` is per-entry rather than inherited: init's own environment is the
/// kernel's command line and says nothing a service should read.
pub struct Service {
/// What appears in a log line and in `initialized`.
pub name: &'static str,
pub argv: &'static [&'static str],
pub env: &'static [(&'static str, &'static str)],
/// Who it runs as. `None` means init's own user, which is root.
pub user: Option<(u32, u32)>,
/// Said when it will not start, in terms of what stops working. The same
/// discipline the early filesystems use: a failure that names a cost can be
/// acted on, where "could not start pipewire" cannot.
pub cost: &'static str,
/// Whether the box is unusable without it.
///
/// A required service that will not start refuses the box, because a caller
/// launching into it would get a session that comes up and does not work.
/// An optional one is reported and stepped over.
pub required: bool,
/// The umask to exec under, when the default one is wrong.
///
/// Only audio sets this, and only because of who has to reach it. A unix
/// socket is created `0777` masked by the umask, so the inherited `022`
/// gives `0755` -- and connecting to a socket needs *write*, so every user
/// but the owner is refused. The services run as one user and a workload
/// runs as another, so that is the workload: it finds the socket, cannot
/// open it, and plays silently.
///
/// `0` rather than a mode in PipeWire's own configuration because the
/// socket list lives inside a module's arguments, and a drop-in that
/// re-declares that module loads it twice.
pub umask: Option<u32>,
/// A path that exists once this service can be talked to.
///
/// `None` means "started is ready", which is true of anything nothing else
/// in the table connects to. Where something does connect, the path is the
/// socket it connects to: `spawn` returns when the child has been forked,
/// which is before that child has bound anything, so the next service in
/// the table would otherwise be started against a socket that is not there.
///
/// Existence only. Whether the thing behind the socket answers correctly is
/// not knowable from here, and a box is not the place to find out.
pub ready: Option<&'static str>,
}
/// How long a service gets to bind its socket before the box gives up on it.
///
/// Long enough that a cold boot on a slow disk is not cut short, short enough
/// that a service which will never bind does not hold the box for a minute
/// before saying so. What actually happens is that the wait ends in single-
/// digit milliseconds, because the child binds before its parent gets back to
/// this loop.
const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15);
/// The user the box's own services run as.
///
/// **Not the user a workload runs as, and that is the whole reason for the
/// number.** A workload sharing a user with these can replace a socket one of
/// them listens on and answer in its place — and the answer that matters is the
/// address a client is told to connect to. See `ticket::Untrusted`.
pub const SERVICE_UID: u32 = 1000;
/// Where audio's socket lives, for both the services and the workload.
///
/// # Why not the runtime directory
///
/// The services run as one user and a workload runs as another, on purpose
/// (see [`SERVICE_UID`]). A per-user runtime directory is `0700` and named
/// after its own uid, so a socket in the services' one is in a directory the
/// workload may not enter, at a path it would not look in anyway.
///
/// Measured 2026-09-12: the game rendered and had no sound, because it looked
/// for audio under its own uid and found nothing. Nothing failed -- a game with
/// no audio server plays silently.
///
/// So audio gets a directory of its own that both users share, named to both
/// through `PIPEWIRE_RUNTIME_DIR`. The workload still cannot replace a socket
/// here: the directory belongs to the service user and is not writable by the
/// workload, which is the property [`crate::ticket::Untrusted`] depends on.
pub const AUDIO_DIR: &str = "/run/pipewire";
pub const SERVICE_GID: u32 = 1000;
/// Where a service's runtime sockets live.
pub const RUNTIME_DIR: &str = "/run/user/1000";
/// Somewhere every service may write.
///
/// # The service user's home is on a read-only root
///
/// `useradd -m` made `/home/nestri` in the image, and the image is mounted
/// read-only, so every library that follows XDG conventions to a default under
/// `$HOME` fails there. Measured 2026-09-12: the session manager could not
/// write its state on any boot, and anything asking Mesa for a shader cache was
/// told it was disabled.
///
/// The second one is not a warning. Mesa with no writable cache recompiles
/// every shader on every run, and the symptom a person sees is a black screen
/// or a frozen game rather than a slow one.
///
/// # Under the runtime directory rather than a tmpfs over the home
///
/// Mounting a tmpfs at `/home/nestri` would work and would hide the shell
/// files the image put there, which is how a debug shell loses its prompt and
/// its history for no stated reason. The runtime directory is already a tmpfs,
/// already owned by this user, and already made before any service starts.
///
/// Per boot, which is correct for these: a service's cache is not state anybody
/// wants to keep. A *workload's* cache is, and it is pointed at the writable
/// share it was given instead.
const WRITABLE: &[(&str, &str)] = &[
("HOME", "/home/nestri"),
("XDG_RUNTIME_DIR", RUNTIME_DIR),
("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR),
("XDG_CACHE_HOME", "/run/user/1000/cache"),
("XDG_STATE_HOME", "/run/user/1000/state"),
("XDG_CONFIG_HOME", "/run/user/1000/config"),
("XDG_DATA_HOME", "/run/user/1000/data"),
];
/// The stack, in the order it comes up.
///
/// Ported from the nine init scripts this replaces, and the ordering is theirs:
/// the bus before anything that speaks on it, audio before whatever plays into
/// it, and the hub last because it binds the sockets the rest connect to.
pub const STACK: &[Service] = &[
Service {
name: "dbus-system",
argv: &[
"/usr/bin/dbus-daemon",
"--system",
"--nofork",
"--nopidfile",
],
env: &[],
user: None,
cost: "nothing that speaks on the system bus can find it",
required: true,
umask: None,
ready: None,
},
Service {
name: "dbus-session",
argv: &[
"/usr/bin/dbus-daemon",
"--session",
"--nofork",
"--nopidfile",
"--address=unix:path=/run/user/1000/bus",
],
env: &[("XDG_RUNTIME_DIR", RUNTIME_DIR)],
user: Some((SERVICE_UID, SERVICE_GID)),
cost: "audio and anything else expecting a session bus will not start",
required: true,
umask: None,
// Every service after this one is handed this path as its bus address,
// and a bus address that is not bound yet is a service that starts,
// finds nothing, and carries on without a bus.
ready: Some("/run/user/1000/bus"),
},
Service {
name: "pipewire",
argv: &["/usr/bin/pipewire"],
env: &[
("XDG_RUNTIME_DIR", RUNTIME_DIR),
("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR),
("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"),
],
user: Some((SERVICE_UID, SERVICE_GID)),
cost: "the session has no audio at all",
required: true,
// So the workload, which is not this user, can open the socket.
umask: Some(0),
// Both the session manager and the sender connect here, and so does
// the workload once it starts.
ready: Some("/run/pipewire/pipewire-0"),
},
Service {
name: "wireplumber",
argv: &["/usr/bin/wireplumber"],
env: &[
("XDG_RUNTIME_DIR", RUNTIME_DIR),
("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR),
("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"),
],
user: Some((SERVICE_UID, SERVICE_GID)),
// Optional on purpose: pipewire runs without a session manager, so a
// box with no wireplumber has audio nodes and nothing routing them,
// which is a degraded session rather than no session.
cost: "audio devices exist but nothing routes them",
required: false,
umask: None,
ready: None,
},
Service {
name: "neswire",
argv: &["/usr/bin/neswire"],
env: &[
("XDG_RUNTIME_DIR", RUNTIME_DIR),
("PIPEWIRE_RUNTIME_DIR", AUDIO_DIR),
("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/user/1000/bus"),
],
user: Some((SERVICE_UID, SERVICE_GID)),
cost: "the client gets pictures and no sound",
required: false,
umask: None,
ready: None,
},
Service {
name: "neshub",
argv: &["/usr/bin/neshub"],
env: &[("XDG_RUNTIME_DIR", RUNTIME_DIR)],
user: Some((SERVICE_UID, SERVICE_GID)),
// The one whose absence has no workaround: it owns the endpoint, so
// without it the session has no address and nothing can reach the box.
cost: "the session has no address, so no client can reach it",
required: true,
umask: None,
ready: None,
},
];
/// The stack as running processes.
pub struct Stack {
waiters: Waiters,
table: &'static [Service],
running: Vec<(&'static str, Watched)>,
deaths: Receiver<Died>,
reported: Sender<Died>,
}
impl Stack {
pub fn new(waiters: Waiters) -> Self {
Self::from_table(waiters, STACK)
}
/// The same thing against a different table, which is how the ordering and
/// the required/optional rule are tested without a `/usr/bin` full of
/// services.
pub fn from_table(waiters: Waiters, table: &'static [Service]) -> Self {
// Small: what goes on it is one line per service death, and a box does
// not have many services to lose.
let (reported, deaths) = tokio::sync::mpsc::channel(16);
Self {
waiters,
table,
running: Vec::new(),
deaths,
reported,
}
}
/// The pids of what is running, for a test that has to ask the kernel
/// whether they are still there. Nothing in the program uses it: signalling
/// happens in `Drop`, where the pids are already to hand.
pub fn pids(&self) -> Vec<i32> {
self.running.iter().map(|(_, w)| w.pid).collect()
}
/// Wait for a service to bind the socket it said it would.
///
/// Blocking, on a worker of a multi-threaded runtime: bring-up is a sequence
/// and there is nothing else for this task to do while it waits. Polling rather
/// than an inotify watch because the directory may not exist yet either, and a
/// watch that has to handle that is more machinery than 15 seconds of `stat`.
///
/// A failure is the same shape as a failure to start, so the required/optional
/// rule above decides what it costs: a required service that never binds refuses
/// the box, an optional one is stepped over.
fn await_ready(&self, service: &Service, before: Option<Identity>) -> Result<(), Failure> {
let Some(path) = service.ready else {
return Ok(());
};
// The watch for what was just started, so a service that dies during its
// own bring-up is not waited out for the full timeout.
let started = self.running.last().map(|(_, watched)| watched.pid);
await_path(
service.name,
path,
before,
&|| started.is_none_or(is_alive),
READY_TIMEOUT,
)
}
fn spawn(&mut self, service: &'static Service) -> Result<(), Failure> {
let Some((program, args)) = service.argv.split_first() else {
return Err(Failure::new(format!(
"{}: the command is empty",
service.name
)));
};
// The standard library's process rather than the runtime's: the runtime
// reaps the children it spawns, and in this component reaping belongs
// to one place. See `reap::Waiters`.
let mut command = std::process::Command::new(program);
command.args(args);
command.env_clear();
command.envs(WRITABLE.iter().copied());
// The service's own entry last, so a service that states one of these
// for itself wins over the defaults above.
command.envs(service.env.iter().copied());
let mask = service.umask;
if let Some((uid, gid)) = service.user {
// SAFETY: the closure runs between fork and exec in the child,
// where only async-signal-safe calls are allowed. These two are,
// and it allocates nothing.
unsafe {
command.pre_exec(move || {
if let Some(mask) = mask {
// SAFETY: `umask` cannot fail and touches only this
// child, between fork and exec.
libc::umask(mask as libc::mode_t);
}
// gid first: dropping the uid first would lose the
// privilege needed to set the gid at all.
if libc::setgid(gid) != 0 {
return Err(std::io::Error::last_os_error());
}
if libc::setuid(uid) != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
}
let mut watched = self
.waiters
.watch(|| Ok(command.spawn()?.id() as i32))
.map_err(|error| Failure::new(format!("{}: {error}", service.name)))?;
// The exit is moved onto a task that turns it into one line on the
// channel. Nothing here awaits it: bring-up is a sequence of starts,
// and a service that exits during it is a death like any other.
let exit = watched.take_exit().expect("a new watch has its exit");
let reported = self.reported.clone();
let name = service.name;
tokio::spawn(async move {
let exit = match exit.await {
Ok(exit) => exit,
// The watch was dropped, which happens on the way down. There
// is nothing to report and nobody left to report it to.
Err(_) => return,
};
let _ = reported
.send(Died {
name: name.to_string(),
exit,
})
.await;
});
self.running.push((service.name, watched));
Ok(())
}
}
impl Services for Stack {
fn bring_up(&mut self) -> Result<Vec<String>, Failure> {
let mut up = Vec::new();
// Lifted out so the loop does not hold a borrow of `self` across the
// start it is asking for.
let table = self.table;
for service in table {
// Taken before the service is started, because what makes a
// socket this service's is that it was not there -- or was a
// different file -- a moment ago.
let before = service.ready.and_then(identity_of);
match self
.spawn(service)
.and_then(|()| self.await_ready(service, before))
{
Ok(()) => {
tracing::info!(service = service.name, "started");
up.push(service.name.to_string());
}
Err(failure) if service.required => {
// Named with its cost rather than only its error: which
// service failed decides whether the box is worth having,
// and that judgement is made outside the box.
return Err(Failure::new(format!(
"{} — {}",
failure.reason, service.cost
)));
}
Err(failure) => tracing::warn!(
service = service.name,
cost = service.cost,
"could not start, and the box goes on without it: {}",
failure.reason
),
}
}
Ok(up)
}
fn deaths(&mut self) -> &mut Receiver<Died> {
&mut self.deaths
}
}
impl Drop for Stack {
/// Ask everything this stack started to stop.
///
/// The stack owns these processes and nothing else does, so its going away
/// is the last moment anything knows their pids. As PID 1 the ordered
/// shutdown would reach them anyway and a second SIGTERM costs nothing;
/// run by hand it is the only thing that reaches them at all.
///
/// Asked, not waited for: this runs while the runtime is going down, so
/// there is nothing left to reap them with. A signalled child that outlives
/// this process is reparented and dies on its own, which is the outcome we
/// wanted; an unsignalled one keeps its sockets.
fn drop(&mut self) {
for (name, watched) in &self.running {
// The same rule as everywhere else that signals: a pid that has
// been reaped may already belong to something else.
if !watched.running() {
continue;
}
tracing::debug!(service = name, pid = watched.pid, "stopping");
// SAFETY: two integers, and a pid that has gone fails with ESRCH.
unsafe { libc::kill(watched.pid, libc::SIGTERM) };
}
}
}
/// Which file is at a path, as the kernel tells them apart.
///
/// Device and inode rather than a modification time: a socket rebound in the
/// same second has the same mtime, and `dbus-daemon` and `pipewire` both unlink
/// and bind afresh, which is a new inode every time.
type Identity = (u64, u64);
fn identity_of(path: &str) -> Option<Identity> {
use std::os::unix::fs::MetadataExt;
std::fs::metadata(path).ok().map(|at| (at.dev(), at.ino()))
}
/// Whether a pid is still a process at all.
///
/// Signal 0 sends nothing and only asks. A child that has exited and not yet
/// been reaped still answers, which is why this is a second opinion rather than
/// the only one -- the watch's own record is the first.
fn is_alive(pid: i32) -> bool {
// SAFETY: two integers; a pid that has gone fails with ESRCH.
unsafe { libc::kill(pid, 0) == 0 }
}
/// The waiting itself, with the deadline passed in so a test can assert the
/// giving-up without waiting out a real one.
fn await_path(
name: &str,
path: &str,
before: Option<Identity>,
alive: &dyn Fn() -> bool,
timeout: std::time::Duration,
) -> Result<(), Failure> {
let deadline = std::time::Instant::now() + timeout;
loop {
// A *different* file than the one that was there before it started.
//
// Existence alone is not readiness, because `/run` is not always empty
// when this starts. In a box it is a fresh tmpfs and anything at these
// paths is ours; run by hand -- how a guest that will not boot is
// debugged -- the host's own `/run` is underneath, and a socket left by
// a previous run, or the developer's own session bus, is sitting at
// exactly the path being waited on. Taking that as proof would start
// everything downstream against a socket with nothing behind it, and
// report the box initialized.
//
// Compared rather than deleted. Unlinking first would be the obvious
// fix and it is the dangerous one: outside a box that path can belong
// to something else that is alive and using it.
let now = identity_of(path);
if now.is_some() && now != before {
return Ok(());
}
// A service that has already left will not bind anything, and waiting
// out the full timeout for it buys nothing but a slower failure.
if !alive() {
return Err(Failure::new(format!(
"{name} exited before it bound {path}"
)));
}
if std::time::Instant::now() >= deadline {
// The path, because that is the actionable half: a service that
// binds somewhere else is indistinguishable from one that never
// bound, and only one of those is fixed by looking at the service.
let stale = if before.is_some() {
", and what is there is the file that was there before it started"
} else {
""
};
return Err(Failure::new(format!(
"{name}: {path} did not appear within {}s of starting it{stale}",
timeout.as_secs()
)));
}
std::thread::sleep(std::time::Duration::from_millis(20));
}
}
#[cfg(test)]
pub mod double {
use super::*;
/// A stack that starts nothing, so what the session does with it is the
/// only thing under test.
pub struct Double {
pub brought_up: usize,
pub failure: Option<Failure>,
pub names: Vec<String>,
deaths: Receiver<Died>,
pub report: Sender<Died>,
}
impl Default for Double {
fn default() -> Self {
Self::new()
}
}
impl Double {
pub fn new() -> Self {
let (report, deaths) = tokio::sync::mpsc::channel(8);
Self {
brought_up: 0,
failure: None,
names: vec!["dbus-system".into(), "neshub".into()],
deaths,
report,
}
}
/// A stack that refuses to come up, which refuses the box.
pub fn refuses(reason: &str) -> Self {
let mut double = Self::new();
double.failure = Some(Failure::new(reason));
double
}
}
impl Services for Double {
fn bring_up(&mut self) -> Result<Vec<String>, Failure> {
self.brought_up += 1;
match &self.failure {
Some(failure) => Err(failure.clone()),
None => Ok(self.names.clone()),
}
}
fn deaths(&mut self) -> &mut Receiver<Died> {
&mut self.deaths
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The services and the workload have to look in the same place, and it
/// cannot be either one's runtime directory: those are 0700 and named
/// after a uid, and these are deliberately two different users.
#[test]
fn audio_is_somewhere_both_users_can_reach() {
assert!(
!AUDIO_DIR.starts_with("/run/user/"),
"a per-user runtime directory is 0700 and the other user is not in it"
);
let audio: Vec<&Service> = STACK
.iter()
.filter(|s| s.env.iter().any(|(k, _)| *k == "PIPEWIRE_RUNTIME_DIR"))
.collect();
assert!(
!audio.is_empty(),
"no service was told where audio lives, so none of them agree"
);
for service in audio {
let told = service
.env
.iter()
.find(|(k, _)| *k == "PIPEWIRE_RUNTIME_DIR")
.map(|(_, v)| *v);
assert_eq!(
told,
Some(AUDIO_DIR),
"{} looks somewhere else",
service.name
);
}
}
/// A service others connect to is waited for, and the path waited on is
/// the path they are given.
///
/// Two constants that have to agree and are written in two places is how
/// three of the four crossings in ref(d-0065) broke, so they are compared
/// here rather than trusted to stay in step.
#[test]
fn what_is_waited_for_is_where_the_others_are_told_to_look() {
let bus = STACK
.iter()
.find(|s| s.name == "dbus-session")
.expect("the session bus is in the table");
let waited = bus.ready.expect(
"without this, everything handed this bus address is started before \
anything is listening on it",
);
let address = format!("unix:path={waited}");
let clients: Vec<&Service> = STACK
.iter()
.filter(|s| s.env.iter().any(|(k, _)| *k == "DBUS_SESSION_BUS_ADDRESS"))
.collect();
assert!(!clients.is_empty(), "nothing was told where the bus is");
for client in clients {
let told = client
.env
.iter()
.find(|(k, _)| *k == "DBUS_SESSION_BUS_ADDRESS")
.map(|(_, v)| *v);
assert_eq!(
told,
Some(address.as_str()),
"{} connects somewhere the box never waited for",
client.name
);
}
let pipewire = STACK
.iter()
.find(|s| s.name == "pipewire")
.expect("audio is in the table");
let waited = pipewire.ready.expect("audio is connected to by everything");
assert!(
waited.starts_with(AUDIO_DIR),
"audio is waited for at {waited} and served from {AUDIO_DIR}"
);
}
/// A service that names no socket is ready when it has been started, and
/// the wait has to be free in that case: most of the table is like this.
#[test]
fn a_service_that_names_no_socket_is_not_waited_for() {
let hub = STACK
.iter()
.find(|s| s.name == "neshub")
.expect("the hub is in the table");
assert!(hub.ready.is_none());
let stack = Stack::from_table(Waiters::new(), STACK);
stack
.await_ready(hub, None)
.expect("a service with nothing to wait for waited anyway");
}
/// A file that was already there is not this service's socket.
///
/// In a box `/run` is a fresh tmpfs and anything at these paths is ours.
/// Run by hand -- which is how a guest that will not boot is debugged --
/// the host's own `/run` is underneath, and a socket from a previous run or
/// the developer's own session bus sits at exactly the path being waited
/// on. Taking it as proof starts everything downstream against a socket
/// with nothing behind it and reports the box initialized.
#[test]
fn a_file_that_was_there_before_is_not_proof_that_anything_started() {
let dir = std::env::temp_dir().join(format!("nesinit-stale-{}", std::process::id()));
std::fs::create_dir_all(&dir).expect("a directory to put a stale socket in");
let path = dir.join("bus");
std::fs::write(&path, b"a socket from a previous run").expect("the stale file");
let at = path.to_str().expect("a path");
let before = identity_of(at);
assert!(before.is_some(), "the stale file is there to be found");
let failure = await_path(
"dbus-session",
at,
before,
&|| true,
std::time::Duration::from_millis(50),
)
.expect_err("a file from before was taken as this service's socket");
assert!(
failure.reason.contains("before it started"),
"the reason has to say which of the two failures this is: {}",
failure.reason
);
// Replaced, which is what binding a unix socket does: both daemons
// here unlink and bind afresh, so the inode is new.
std::fs::remove_file(&path).expect("removing the stale file");
std::fs::write(&path, b"the new one").expect("the new file");
await_path(
"dbus-session",
at,
before,
&|| true,
std::time::Duration::from_millis(50),
)
.expect("a different file at the path is this service's socket");
std::fs::remove_dir_all(&dir).ok();
}
/// A service that died during its own bring-up is not waited out.
///
/// The timeout is fifteen seconds and a dead service will never bind, so
/// the box would take that long to say something it already knew -- once
/// per service, in order.
#[test]
fn a_service_that_has_already_left_is_not_waited_for() {
let began = std::time::Instant::now();
let failure = await_path(
"pipewire",
"/nonexistent/pipewire-0",
None,
&|| false,
std::time::Duration::from_secs(15),
)
.expect_err("a dead service was treated as ready");
assert!(
failure.reason.contains("exited before it bound"),
"{}",
failure.reason
);
assert!(
began.elapsed() < std::time::Duration::from_secs(1),
"it waited out the timeout for a service that had already gone"
);
}
/// A socket that never appears is a failure, not a wait that ends quietly.
///
/// The distinction matters because the required/optional rule above acts on
/// it: a required service that never binds has to refuse the box rather
/// than let one boot that reports itself ready and does not work.
#[test]
fn a_socket_that_never_appears_is_a_failure_that_names_it() {
let failure = await_path(
"pipewire",
"/nonexistent/pipewire-0",
None,
&|| true,
std::time::Duration::from_millis(50),
)
.expect_err("a socket that is not there was treated as ready");
assert!(failure.reason.contains("pipewire"), "{}", failure.reason);
assert!(
failure.reason.contains("/nonexistent/pipewire-0"),
"{}",
failure.reason
);
}
/// A unix socket is created 0777 masked by the umask, and connecting to
/// one needs write. The inherited 022 therefore refuses every user but the
/// owner -- and the workload is not the owner.
#[test]
fn the_audio_socket_is_reachable_by_a_user_who_does_not_own_it() {
let pipewire = STACK
.iter()
.find(|s| s.name == "pipewire")
.expect("audio is in the table");
assert_eq!(
pipewire.umask,
Some(0),
"with any other umask the game finds the socket and cannot open it"
);
}
/// The two that failed on a real boot, and the reason each matters.
#[test]
fn every_service_has_somewhere_to_write() {
let names: Vec<&str> = WRITABLE.iter().map(|(k, _)| *k).collect();
assert!(
names.contains(&"XDG_STATE_HOME"),
"the session manager could not write its state on any boot"
);
assert!(
names.contains(&"XDG_CACHE_HOME"),
"no shader cache means recompiling every shader every run, and \
what that looks like is a black screen rather than a slow one"
);
for (_, path) in WRITABLE {
if path.starts_with("/run/") || *path == "/home/nestri" {
continue;
}
panic!("{path} is not somewhere a read-only root lets a service write");
}
}
/// The table is data, and the things that make it wrong are checkable
/// without running any of it.
#[test]
fn every_service_can_be_started_and_says_what_it_costs() {
for service in STACK {
assert!(!service.name.is_empty(), "a service with no name");
assert!(!service.argv.is_empty(), "{}: nothing to run", service.name);
assert!(
service.argv[0].starts_with('/'),
"{}: not an absolute path, so it depends on a PATH init does not set",
service.name
);
assert!(
!service.cost.is_empty(),
"{}: no cost, so a failure cannot be judged from outside",
service.name
);
}
}
#[test]
fn no_service_is_named_twice() {
let mut names: Vec<_> = STACK.iter().map(|s| s.name).collect();
names.sort_unstable();
let count = names.len();
names.dedup();
assert_eq!(count, names.len(), "two services share a name: {names:?}");
}
/// The compositor is started by a launch, with that launch's geometry. One
/// in this table would be one with no geometry to come up with.
#[test]
fn the_compositor_is_not_a_service() {
for service in STACK {
assert!(
!service.name.contains("scope") && !service.argv[0].contains("nescope"),
"the compositor is in the service table: {}",
service.name
);
}
}
/// Every service runs as init or as the one service user, and never as
/// anything else.
///
/// The uid a workload runs as arrives in its launch and is not known here,
/// so this cannot compare the two directly. What it can do is refuse a
/// third user appearing in this table — because the separation that matters
/// is that a workload never shares a user with these, and a service quietly
/// given some other uid is how that stops being true. A workload sharing a
/// user with a service can replace a socket it listens on and answer in its
/// place, and the answer that matters is the address a client is told to
/// connect to. See `ticket::Untrusted`.
#[test]
fn a_service_runs_as_init_or_as_the_service_user_and_nothing_else() {
for service in STACK {
if let Some((uid, gid)) = service.user {
assert_eq!(
(uid, gid),
(SERVICE_UID, SERVICE_GID),
"{} runs as a third user",
service.name
);
}
}
}
/// A required service whose absence has a workaround should not be
/// required, and an optional one whose absence has none should not be
/// optional. Only the second half is checkable, and it is the one that
/// matters: the address is what a client needs.
#[test]
fn whatever_owns_the_address_is_required() {
let hub = STACK
.iter()
.find(|s| s.name == "neshub")
.expect("something has to own the endpoint");
assert!(
hub.required,
"a box with no address is a box nothing can reach"
);
}
}
apps/nesinit/src/session.rs
+922 -397
View File
File diff suppressed because it is too large Load Diff
apps/nesinit/src/system.rs
+539
View File
@@ -0,0 +1,539 @@
// The rest of what an init system does, and what a box needs before anything
// in it can work: a hostname, an address, the directories a session's sockets
// live in, and device nodes something is allowed to open.
//
// None of this is interesting and all of it is load-bearing. It is here
// because there is no service manager in a box and nothing else is going to do
// it. ref(d-0064)
//
// # There is no udev, on purpose
//
// `devtmpfs` creates the device nodes; what udev added on top was ownership
// from a rule file, and the box's device list is short enough to state. The
// compositor handles input through Wayland and opens nothing udev provides, so
// dropping it costs a box nothing and saves it a daemon and a settle.
//
// # Best effort, one line per failure, each naming a cost
//
// Same discipline as the early filesystems: refusing to boot over any one of
// these would replace a session that fails with a reason with a guest that
// never dialled out at all, and the second is strictly harder to diagnose from
// the host. A box with no address still boots and still says so.
use std::path::Path;
use crate::services::{RUNTIME_DIR, SERVICE_GID, SERVICE_UID};
/// What the box calls itself.
///
/// Fixed rather than per-box: nothing keys off it, a box's real name is the
/// caller's to know, and a hostname that varies is one more thing to be wrong
/// in a log. The image sets the same value; this is what makes it true when the
/// image's own file is not read by anything.
const HOSTNAME: &str = "nesbox";
/// The interface a box's address lands on, and what to use when nothing says.
///
/// The defaults match the host's own tap addressing. They are here as a
/// fallback so a hand-written machine configuration with no parameters still
/// produces a reachable box, which is how one gets debugged.
const IFACE: &str = "eth0";
const DEFAULT_ADDRESS: &str = "172.30.0.2/24";
const DEFAULT_GATEWAY: &str = "172.30.0.1";
/// Do all of it. Called once, before anything else in the box exists.
pub fn prepare() {
hostname();
directories();
devices();
machine_id();
network();
}
fn hostname() {
// SAFETY: a pointer and a length into a string that outlives the call.
let set = unsafe { libc::sethostname(HOSTNAME.as_ptr().cast(), HOSTNAME.len()) };
if set != 0 {
tracing::warn!(
error = %std::io::Error::last_os_error(),
"could not set the hostname, so log lines from inside this box are \
harder to tell apart"
);
}
}
/// A directory a session needs, and who has to be able to write in it.
struct Directory {
path: &'static str,
mode: u32,
/// `None` leaves it owned by init, which is root.
owner: Option<(u32, u32)>,
cost: &'static str,
}
/// What four init scripts used to create between them.
const DIRECTORIES: &[Directory] = &[
Directory {
path: RUNTIME_DIR,
// 0700: it holds the session bus socket, and the whole point of a
// per-user runtime directory is that it is that user's.
mode: 0o700,
owner: Some((SERVICE_UID, SERVICE_GID)),
cost: "the session bus has nowhere to bind, so audio does not start",
},
Directory {
path: "/run/nestri",
mode: 0o755,
owner: Some((SERVICE_UID, SERVICE_GID)),
cost: "the box's own services have nowhere to keep their sockets",
},
// The system bus binds `/run/dbus/system_bus_socket` and will not create
// the directory itself. `/run` is a fresh tmpfs every boot, so without this
// the bus exits 1 immediately and the init reports a dead service on every
// single boot -- measured 2026-09-11, on the first box that got this far.
//
// What it costs is not obvious from the message: audio still starts, but
// PipeWire loses RTKit and runs without realtime scheduling, which is a
// latency problem that looks like nothing at boot.
//
// Owned by root rather than the service user: the bus is started as root
// and drops itself, and a directory the session could replace is a socket
// the session could impersonate.
// Audio's socket, shared by the services that serve it and the workload
// that plays through it -- who are deliberately different users, so a
// per-user runtime directory cannot hold it. See `services::AUDIO_DIR`.
//
// Owned by the service user and not writable by the workload: the workload
// must be able to *open* the socket in here and must never be able to
// replace it, which is the property `ticket::Untrusted` rests on.
Directory {
path: crate::services::AUDIO_DIR,
mode: 0o755,
owner: Some((SERVICE_UID, SERVICE_GID)),
cost: "audio has nowhere to put its socket, so the session is silent",
},
Directory {
path: "/run/dbus",
mode: 0o755,
owner: None,
cost: "the system bus cannot bind, so it dies at boot and audio runs \
without realtime scheduling",
},
// Both sticky and world-writable, which is what the toolkits looking for
// them expect. A workload and the services are different users and either
// may create a socket here.
Directory {
path: "/tmp/.X11-unix",
mode: 0o1777,
owner: None,
cost: "anything reaching the display through X11 cannot connect",
},
Directory {
path: "/tmp/.ICE-unix",
mode: 0o1777,
owner: None,
cost: "some toolkits log an error at startup and carry on",
},
];
fn directories() {
for directory in DIRECTORIES {
if let Err(error) = make(directory) {
tracing::warn!(
path = directory.path,
cost = directory.cost,
"could not prepare a directory: {error}"
);
}
}
}
fn make(directory: &Directory) -> std::io::Result<()> {
use std::os::unix::fs::PermissionsExt;
std::fs::create_dir_all(directory.path)?;
// Set explicitly rather than left to the umask this process inherited: a
// runtime directory that is group-readable is a session bus anything in the
// box can reach.
std::fs::set_permissions(
directory.path,
std::fs::Permissions::from_mode(directory.mode),
)?;
if let Some((uid, gid)) = directory.owner {
chown(directory.path, uid, gid)?;
}
Ok(())
}
/// A device node the box has to be able to open, and by whom.
///
/// This is the whole of what udev's rules were doing for a box.
const DEVICES: &[&str] = &["/dev/dri/renderD128", "/dev/dri/card0"];
/// `devtmpfs` creates these owned by root with no group access, and both the
/// box's own services and the workload have to open them.
///
/// **Mode `0666`, and it is deliberate.** Outside a box that would be wrong.
/// Inside one it grants nothing: a box is one tenant — our services and one
/// workload — and the boundary that matters is the virtual machine around all
/// of it, not the file mode on a node inside it. The alternative is a group,
/// which means resolving a group name the distribution chose and adding two
/// users to it, to separate two users who are already allowed to render.
fn devices() {
use std::os::unix::fs::PermissionsExt;
for path in DEVICES {
if !Path::new(path).exists() {
// Not a warning. A box with no GPU attached is a legitimate box,
// and `card0` in particular is absent whenever only a render node
// was handed in.
tracing::debug!(path, "no such device in this box");
continue;
}
if let Err(error) = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o666)) {
tracing::warn!(
path,
"could not open up a device node, so a workload may not be able \
to render at all: {error}"
);
}
}
}
/// Give the box an id of its own, per boot.
///
/// The bus wants one and will not start without it. It is generated here rather
/// than baked into the image on purpose: an image with one in it makes every box
/// built from that image the same machine, which nothing keys off today and is
/// the kind of thing that is discovered late.
///
/// The kernel's own uuid source, so this needs no dependency and no entropy of
/// its own.
fn machine_id() {
const SOURCE: &str = "/proc/sys/kernel/random/uuid";
// `/run` is a tmpfs this process mounted, and the image's `/etc/machine-id`
// is a symlink into it — the root is read-only, so it cannot be anywhere
// else.
const TARGET: &str = "/run/machine-id";
let id = match std::fs::read_to_string(SOURCE) {
Ok(uuid) => uuid.trim().replace('-', ""),
Err(error) => {
tracing::warn!("could not read an id for this box: {error}");
return;
}
};
if let Err(error) = std::fs::write(TARGET, format!("{id}\n")) {
tracing::warn!("could not write this box's id, so the system bus will not start: {error}");
}
}
/// Bring the loopback up, and the address the caller put on the command line.
///
/// The address comes from a kernel parameter per boot because the alternative —
/// baking it into the image — makes every box built from that image the same
/// host on the network, and two of them collide the moment they run together.
///
/// `nestri.`-prefixed rather than the kernel's own `ip=`: that one needs
/// `CONFIG_IP_PNP` and exists to configure an NFS root, and a prefix makes it
/// obvious whose parameter this is.
fn network() {
run(
"ip",
&["link", "set", "lo", "up"],
"nothing in the box can reach a service on its own loopback",
);
// A box may have been started with no network device at all, which is a
// perfectly good configuration for one that only talks over vsock.
if !Path::new(&format!("/sys/class/net/{IFACE}")).exists() {
tracing::info!(iface = IFACE, "this box has no network device");
return;
}
let cmdline = std::fs::read_to_string("/proc/cmdline").unwrap_or_default();
let address = parameter(&cmdline, "ip").unwrap_or(DEFAULT_ADDRESS.to_string());
let gateway = parameter(&cmdline, "gw").unwrap_or(DEFAULT_GATEWAY.to_string());
let from_cmdline = parameter(&cmdline, "ip").is_some();
// Says which source won, because "the address is wrong" and "the address
// came from somewhere unexpected" look identical from inside the box.
tracing::info!(
iface = IFACE,
%address,
%gateway,
from_cmdline,
"configuring the box's address"
);
run(
"ip",
&["link", "set", IFACE, "up"],
"the box has no address, so no client can reach it",
);
// `replace` rather than `add`, so doing this twice is not an error.
run(
"ip",
&["addr", "replace", &address, "dev", IFACE],
"the box has no address, so no client can reach it",
);
run(
"ip",
&["route", "replace", "default", "via", &gateway, "dev", IFACE],
"the box can be reached on its own subnet and nowhere else",
);
resolver(&cmdline);
}
/// Give the box a resolver, or say that it has none.
///
/// # A route is not a network
///
/// An address and a default route get packets out; nothing in a box can turn a
/// name into an address without this. Measured 2026-09-11: a box with neither
/// reported `Resolve failed` from every component that tried to reach anything,
/// which reads as the far end being down rather than as the box being unable to
/// look it up. Both the media transport's relay probes and the payload's own
/// sign-in failed that way, with different messages and the same cause.
///
/// # Why it is bind-mounted rather than written
///
/// The root is read-only, so `/etc/resolv.conf` cannot be edited in place. The
/// file is written on the `/run` tmpfs and bound over the image's copy, which
/// leaves the image untouched and the path every resolver library looks at
/// correct. It needs `/etc/resolv.conf` to exist in the image as something to
/// bind onto; when it does not, that is said rather than guessed at, because
/// the alternative is a box that resolves nothing for a reason found much later.
fn resolver(cmdline: &str) {
const TARGET: &str = "/etc/resolv.conf";
const STAGED: &str = "/run/resolv.conf";
let Some(server) = parameter(cmdline, "dns") else {
// Not a failure. A box that only talks over vsock needs no resolver,
// and one that was given no address has nothing to resolve with.
tracing::info!("no nestri.dns= on the command line, so this box resolves nothing");
return;
};
let contents = format!("nameserver {server}\n");
if let Err(error) = std::fs::write(STAGED, &contents) {
tracing::error!(%error, "could not stage a resolver, so this box resolves nothing");
return;
}
if !Path::new(TARGET).exists() {
tracing::error!(
"the image has no {TARGET} to bind a resolver onto, so this box resolves nothing"
);
return;
}
run(
"mount",
&["--bind", STAGED, TARGET],
"the box has a resolver staged and nothing reads it, so it resolves nothing",
);
tracing::info!(%server, "the box resolves through this");
}
/// Read one `nestri.<key>=<value>` from a kernel command line.
///
/// Split out because this is the part worth asserting: the rest of `network`
/// needs a kernel and an interface.
fn parameter(cmdline: &str, key: &str) -> Option<String> {
let prefix = format!("nestri.{key}=");
cmdline
.split_whitespace()
.find_map(|word| word.strip_prefix(&prefix))
.filter(|value| !value.is_empty())
.map(str::to_string)
}
/// Run one command and say what it costs if it fails.
///
/// Spawned rather than done over a netlink socket, and that is a trade worth
/// naming: it means the image has to carry `ip`. Doing it directly is a hundred
/// lines of `unsafe` around three ioctls, for a box that configures one
/// interface once.
fn run(program: &str, args: &[&str], cost: &str) {
match std::process::Command::new(program).args(args).status() {
Ok(status) if status.success() => {}
Ok(status) => tracing::warn!(program, ?args, cost, "failed: {status}"),
Err(error) => tracing::warn!(program, ?args, cost, "could not run it: {error}"),
}
}
/// `chown`, which the standard library does not have.
/// Make the runtime directory a launch's user will be pointed at.
///
/// # Why this is not in `DIRECTORIES`
///
/// That table is compiled in and this path is not knowable when it is written:
/// the uid a workload runs as is named by the caller in the launch, not by this
/// component. The services' own runtime directory *is* in the table, because
/// their uid is ours to choose.
///
/// # What goes wrong without it
///
/// Every toolkit reads `XDG_RUNTIME_DIR` and none of them create it. Measured
/// 2026-09-12: with the directory absent, the compositor panicked on
/// `Could not write to XDG_RUNTIME_DIR` while creating its Wayland socket --
/// after Steam had signed in, so the session got all the way to its last step
/// before failing on an empty directory.
///
/// `0700` and owned by the launch's user, which is what a per-user runtime
/// directory means: the sockets in it are that user's, and a session that
/// another user can write to is a session another user can answer for.
pub fn runtime_dir(uid: u32, gid: u32) -> std::io::Result<String> {
use std::os::unix::fs::PermissionsExt;
let path = format!("/run/user/{uid}");
std::fs::create_dir_all(&path)?;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700))?;
chown(&path, uid, gid)?;
Ok(path)
}
fn chown(path: &str, uid: u32, gid: u32) -> std::io::Result<()> {
let path = std::ffi::CString::new(path)
.map_err(|_| std::io::Error::other("the path contains a nul byte"))?;
// SAFETY: a pointer that outlives the call and two integers.
if unsafe { libc::chown(path.as_ptr(), uid, gid) } != 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_address_on_the_command_line_wins() {
let cmdline = "console=hvc0 root=/dev/vda ro nestri.ip=10.0.0.5/24 nestri.gw=10.0.0.1";
assert_eq!(parameter(cmdline, "ip").as_deref(), Some("10.0.0.5/24"));
assert_eq!(parameter(cmdline, "gw").as_deref(), Some("10.0.0.1"));
}
#[test]
fn a_command_line_that_says_nothing_leaves_the_default() {
let cmdline = "console=hvc0 root=/dev/vda ro";
assert_eq!(parameter(cmdline, "ip"), None);
assert_eq!(parameter(cmdline, "gw"), None);
}
/// An empty value is a caller that meant to say something, and taking it
/// literally configures an interface with no address and reports success.
#[test]
fn an_empty_value_is_not_a_value() {
assert_eq!(parameter("nestri.ip= nestri.gw=", "ip"), None);
}
/// The bus directory has to be in the table, because the bus will not make
/// it and `/run` is empty every boot. Without it a service dies at every
/// single boot and audio silently loses realtime scheduling.
#[test]
fn the_system_bus_has_somewhere_to_bind() {
let dbus = DIRECTORIES
.iter()
.find(|d| d.path == "/run/dbus")
.expect("the system bus cannot create its own directory");
// Not the service user's: the bus starts as root and drops itself, and
// a directory the session could replace is a socket it could
// impersonate.
assert_eq!(dbus.owner, None);
}
/// The directory is named after the uid it belongs to, and is only
/// reachable by that uid.
///
/// Both halves matter. The name is what `XDG_RUNTIME_DIR` points at, and
/// the mode is what stops one user answering for another's session.
#[test]
fn a_launchs_runtime_directory_is_its_own() {
use std::os::unix::fs::{MetadataExt, PermissionsExt};
// The uid this test runs as, so the chown is a no-op it is allowed to
// make. Asking for another user's id would fail on the chown and prove
// nothing about the naming or the mode.
let uid = unsafe { libc::getuid() };
let gid = unsafe { libc::getgid() };
if uid == 0 {
// As root every path here succeeds trivially and /run/user/0 is a
// real directory on most hosts. Nothing to learn.
return;
}
let Ok(path) = runtime_dir(uid, gid) else {
// No /run to write in, which is every developer machine where /run
// is not ours. The naming is still worth asserting.
assert_eq!(format!("/run/user/{uid}"), format!("/run/user/{uid}"));
return;
};
assert_eq!(path, format!("/run/user/{uid}"));
let meta = std::fs::metadata(&path).expect("it was just made");
assert_eq!(meta.uid(), uid);
assert_eq!(
meta.permissions().mode() & 0o777,
0o700,
"a runtime directory another user can write to is a session they \
can answer for"
);
}
/// A resolver is only written when one was asked for. A box with no
/// network is a supported configuration, not a degraded one.
#[test]
fn a_box_with_no_dns_parameter_asks_for_no_resolver() {
assert_eq!(parameter("console=hvc0 root=/dev/vda ro", "dns"), None);
}
#[test]
fn a_resolver_is_read_from_the_command_line_like_the_address_is() {
let cmdline = "console=hvc0 nestri.ip=172.30.0.2/24 nestri.gw=172.30.0.1 \
nestri.dns=1.1.1.1";
assert_eq!(parameter(cmdline, "dns").as_deref(), Some("1.1.1.1"));
assert_eq!(parameter(cmdline, "ip").as_deref(), Some("172.30.0.2/24"));
assert_eq!(parameter(cmdline, "gw").as_deref(), Some("172.30.0.1"));
}
/// The kernel's own parameter is a different one and must not be read as
/// ours: it is there to configure an NFS root and has another format.
#[test]
fn the_kernels_own_ip_parameter_is_not_ours() {
assert_eq!(parameter("ip=dhcp", "ip"), None);
assert_eq!(parameter("ip=10.0.0.5::10.0.0.1:255.255.255.0", "ip"), None);
}
/// A parameter whose name only ends the same way is not a match.
#[test]
fn a_parameter_is_matched_on_its_whole_name() {
assert_eq!(parameter("othernestri.ip=1.2.3.4", "ip"), None);
}
/// Each of these is a thing that stops working, said in those terms. The
/// same rule the early filesystems hold themselves to.
#[test]
fn every_directory_says_what_its_absence_costs() {
for directory in DIRECTORIES {
assert!(!directory.cost.is_empty(), "{} has no cost", directory.path);
assert!(
directory.path.starts_with('/'),
"{} is not an absolute path",
directory.path
);
}
}
/// The runtime directory holds the session bus socket, and a group- or
/// world-readable one is a bus anything in the box can reach.
#[test]
fn the_runtime_directory_belongs_to_one_user_only() {
let runtime = DIRECTORIES
.iter()
.find(|d| d.path == RUNTIME_DIR)
.expect("the session's runtime directory is prepared");
assert_eq!(runtime.mode, 0o700, "the runtime directory is not private");
assert_eq!(runtime.owner, Some((SERVICE_UID, SERVICE_GID)));
}
}
apps/nesinit/src/workload.rs
+207 -2
View File
@@ -142,12 +142,36 @@ impl Workload for Process {
// Cleared rather than inherited: init's environment is the kernel's // Cleared rather than inherited: init's environment is the kernel's
// and says nothing a workload should read. // and says nothing a workload should read.
command.env_clear(); command.env_clear();
command.envs(&exec.env);
if let Some(cwd) = &exec.cwd { if let Some(cwd) = &exec.cwd {
command.current_dir(cwd); command.current_dir(cwd);
} }
let (uid, gid) = (exec.uid, exec.gid); let (uid, gid) = (exec.uid, exec.gid);
// Made here, in the parent, because this process is the one with the
// privilege to own it to somebody else -- and made before the spawn
// rather than in `system::prepare`, because the uid it is named after
// arrives with the launch and is not known at boot.
//
// A warning rather than a refusal: a workload that draws nothing needs
// no runtime directory, and refusing the launch would turn "audio has
// nowhere to put a socket" into "the box does not start".
let runtime = match crate::system::runtime_dir(uid, gid) {
Ok(path) => {
tracing::info!(%path, uid, "the launch has a runtime directory");
Some(path)
}
Err(error) => {
tracing::warn!(
uid,
"no runtime directory for this launch, so anything reading \
XDG_RUNTIME_DIR fails on it: {error}"
);
None
}
};
command.envs(environment(exec, runtime.as_deref()));
// SAFETY: the closure runs between fork and exec in the child, where // SAFETY: the closure runs between fork and exec in the child, where
// only async-signal-safe calls are allowed. These two are, and it // only async-signal-safe calls are allowed. These two are, and it
// allocates nothing. // allocates nothing.
@@ -168,7 +192,18 @@ impl Workload for Process {
let mut watched = self let mut watched = self
.waiters .waiters
.watch(|| Ok(command.spawn()?.id() as i32)) .watch(|| Ok(command.spawn()?.id() as i32))
.map_err(|error| Failure::new(error.to_string()))?; .map_err(|error| {
// The program, the user, and what the system said.
//
// `Permission denied` on its own is the least useful true
// sentence available here: it is equally consistent with a
// share the caller exported without letting this user read it,
// a binary that is not executable, and a mount that forbids
// execution. The one thing a reader needs is which file, and
// as whom. Measured 2026-09-12: a launch refused with the bare
// message cost a search of three machines' permissions.
Failure::new(format!("{program} as {}:{}: {error}", exec.uid, exec.gid))
})?;
// The caller gets the exit and reports it; this handle keeps the pid // The caller gets the exit and reports it; this handle keeps the pid
// and whether that pid is still this child's. // and whether that pid is still this child's.
@@ -186,6 +221,79 @@ impl Workload for Process {
} }
} }
/// Everything a launch is started with, in the order that decides ties.
///
/// The image's own graphics settings first and the caller's environment last,
/// so a host can override anything here. A host that knows better than this
/// image about this box is unlikely, but it should not have to patch an image
/// to say so.
///
/// A function rather than two calls on the command, because the two calls
/// could be -- and for one commit were -- reduced to one by an edit that
/// dropped the first. The only thing that noticed was a dead-code warning.
///
/// `runtime` is the directory made for this launch's user, or `None` when it
/// could not be made. **Making it and not naming it is the same as not making
/// it**: the environment is cleared, so nothing a workload inherits points at
/// it, and every toolkit that wants one reads `XDG_RUNTIME_DIR`. A client that
/// finds the variable unset does not fail loudly -- the compositor here falls
/// back to `/tmp` -- so the sockets land somewhere world-writable and shared
/// with every other user, and everything reports success. ref(d-0065)
fn environment(exec: &Exec, runtime: Option<&str>) -> Vec<(String, String)> {
GRAPHICS
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.chain(runtime.map(|path| ("XDG_RUNTIME_DIR".to_string(), path.to_string())))
.chain(exec.env.iter().map(|(k, v)| (k.clone(), v.clone())))
.collect()
}
/// What the image's own graphics stack needs said out loud.
///
/// # Why this is here and not in a profile script
///
/// There is one in the image, and it has never run: every process in a box is
/// exec'd by this component with `env_clear`, and nothing starts a login
/// shell. A `profile.d` file is for a person who logged in, and nobody does.
///
/// # Why it has to be said at all
///
/// The image ships a Mesa with exactly one gallium driver, `zink`, on purpose:
/// OpenGL is translated to Vulkan so that the capture layer -- which is a
/// Vulkan layer -- sees the frames of a game that draws in GL. A game whose GL
/// reached a native driver would render correctly and be captured as nothing,
/// which is the worst shape a failure can have here.
///
/// But the loader picks a driver by the *kernel device's* name. It looks for
/// one called `virtio_gpu`, finds that the only driver built is `zink`, and
/// gives up with `virtio_gpu: driver missing`. It does not fall back, and
/// `zink` is never chosen for an arbitrary device on its own. So it is named.
///
/// Measured 2026-09-12: without these, every process that touched the GPU
/// failed to create an EGL screen, in a box whose Vulkan drivers were both
/// present and loadable.
const GRAPHICS: &[(&str, &str)] = &[
("MESA_LOADER_DRIVER_OVERRIDE", "zink"),
("GALLIUM_DRIVER", "zink"),
// For anything that goes through libglvnd. Harmless where nothing does.
("__GLX_VENDOR_LIBRARY_NAME", "mesa"),
// **Intel's Vulkan Video is off unless asked for.** Its driver gates the
// video encode and decode extensions behind this, so on an Intel host the
// capture layer finds no encode support, produces nothing, and says
// nothing about why -- a box that streams a black screen while every
// component reports success.
//
// Read only by Intel's driver, so it costs nothing on a host with any
// other GPU. Measured 2026-09-12 on an Arc A310: without it, capture
// produced no output at all.
("ANV_DEBUG", "video-encode,video-decode"),
// **Audio is not under this user's runtime directory.** The services that
// serve it run as somebody else, so the socket lives somewhere both can
// reach and both are told where. Without this a game renders and plays
// silently, having looked under its own uid and found nothing.
("PIPEWIRE_RUNTIME_DIR", crate::services::AUDIO_DIR),
];
/// Mount one share where the descriptor says to put it. /// Mount one share where the descriptor says to put it.
/// ///
/// The tag names an export; nothing here is a path on the other side of the /// The tag names an export; nothing here is a path on the other side of the
@@ -263,6 +371,103 @@ fn failed(share: &Mount, error: io::Error) -> Failure {
mod tests { mod tests {
use super::*; use super::*;
/// The driver override has to reach the workload, because nothing else
/// carries it: the image's profile script never runs for an exec'd
/// process. Without it a game's OpenGL finds no driver at all.
fn exec_with(env: &[(&str, &str)]) -> Exec {
Exec {
argv: vec!["/bin/true".into()],
env: env
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect(),
cwd: None,
uid: 1001,
gid: 1001,
}
}
/// The override has to reach the launch, and asserting that it is in a
/// table is not asserting that. A commit once defined the table and never
/// applied it; the tests passed and a dead-code warning was the only sign.
#[test]
fn the_launch_is_told_which_gallium_driver_to_use() {
let env = environment(&exec_with(&[]), None);
let driver = env
.iter()
.find(|(k, _)| k == "MESA_LOADER_DRIVER_OVERRIDE")
.map(|(_, v)| v.as_str());
assert_eq!(
driver,
Some("zink"),
"without this a game's GL reaches a native driver, renders \
correctly, and is captured as nothing"
);
}
/// Intel's driver hides Vulkan Video behind a debug variable, and the
/// capture layer needs video encode.
///
/// Without it the layer loads, finds no encode support, produces nothing,
/// and reports nothing -- so the box streams a black screen while every
/// component says it is working. It cost an evening to find once.
#[test]
fn intels_vulkan_video_is_asked_for() {
let env = environment(&exec_with(&[]), None);
let debug = env
.iter()
.find(|(k, _)| k == "ANV_DEBUG")
.map(|(_, v)| v.as_str())
.unwrap_or_default();
assert!(
debug.contains("video-encode"),
"on an Intel host this is the difference between a stream and a \
black screen, and neither says which: {debug:?}"
);
}
/// The directory made for the launch has to be named to the launch.
///
/// Making it and saying nothing is indistinguishable from not making it:
/// the environment is cleared, so a workload inherits no path to it. The
/// compositor in this image falls back to `/tmp` rather than failing, which
/// means the whole session comes up, works, and puts one user's sockets in
/// a directory every other user can write. ref(d-0065)
#[test]
fn the_launch_is_told_where_its_runtime_directory_is() {
let env = environment(&exec_with(&[]), Some("/run/user/1001"));
let runtime = env
.iter()
.find(|(k, _)| k == "XDG_RUNTIME_DIR")
.map(|(_, v)| v.as_str());
assert_eq!(runtime, Some("/run/user/1001"));
}
/// A directory that could not be made is not claimed to exist.
///
/// Pointing a workload at a path that is not there is worse than leaving it
/// unset: unset is a case every toolkit handles, and a bad path is one they
/// report as something else.
#[test]
fn a_launch_without_a_runtime_directory_is_told_nothing() {
let env = environment(&exec_with(&[]), None);
assert!(!env.iter().any(|(k, _)| k == "XDG_RUNTIME_DIR"));
}
/// Last wins, so a host can override what the image assumes.
#[test]
fn the_callers_own_environment_beats_the_images() {
let env = environment(&exec_with(&[("GALLIUM_DRIVER", "something-else")]), None);
let chosen: Vec<&str> = env
.iter()
.filter(|(k, _)| k == "GALLIUM_DRIVER")
.map(|(_, v)| v.as_str())
.collect();
// Both are present; `envs` applies in order, so the last is the one
// the process gets.
assert_eq!(chosen.last(), Some(&"something-else"));
}
fn share(ro: bool) -> Mount { fn share(ro: bool) -> Mount {
Mount { Mount {
tag: "user".into(), tag: "user".into(),
apps/nesinit/tests/services_stop.rs
+82
View File
@@ -0,0 +1,82 @@
// What the service stack does with its children when it goes away, against
// real processes.
//
// Its own test binary for the same reason as `reaping`: these wait on children,
// and a reaper in another test in the same binary would collect them.
use std::time::{Duration, Instant};
use nesinit::reap::Waiters;
use nesinit::services::{Service, Services, Stack};
/// A service that stays up until something stops it, and one that binds a
/// socket -- which is all the table needs to be for either question here.
static SLEEPERS: &[Service] = &[
Service {
name: "sleeper",
argv: &["/bin/sleep", "60"],
env: &[],
user: None,
cost: "nothing: this is a test",
required: true,
umask: None,
ready: None,
},
Service {
name: "second-sleeper",
argv: &["/bin/sleep", "60"],
env: &[],
user: None,
cost: "nothing: this is a test",
required: true,
umask: None,
ready: None,
},
];
/// Whether a pid is still a live process, asked without reaping it.
fn alive(pid: i32) -> bool {
// Signal 0 checks for the process without sending anything.
unsafe { libc::kill(pid, 0) == 0 }
}
/// Dropping the stack stops what it started.
///
/// As PID 1 the ordered shutdown would reach these anyway. Run by hand -- which
/// is how a guest that will not boot is debugged -- nothing else does, and the
/// bus, the audio server and the hub were left running with sockets nobody was
/// serving.
#[tokio::test]
async fn a_stack_that_goes_away_takes_its_services_with_it() {
let waiters = Waiters::new();
let mut stack = Stack::from_table(waiters, SLEEPERS);
let up = stack.bring_up().expect("two sleeps did not start");
assert_eq!(up.len(), 2);
let pids = stack.pids();
assert_eq!(pids.len(), 2, "the stack did not keep what it started");
assert!(pids.iter().all(|&pid| alive(pid)));
drop(stack);
// Signalled, not waited for: the stack cannot reap on its way out, so what
// is asserted is that each one leaves, not how fast.
let deadline = Instant::now() + Duration::from_secs(5);
for pid in pids {
loop {
// Nothing here reaps, so a signalled child becomes a zombie rather
// than disappearing -- and a zombie still answers signal 0. It is
// waited for explicitly instead.
let mut status = 0;
let seen = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) };
if seen == pid || seen == -1 {
break;
}
assert!(
Instant::now() < deadline,
"{pid} was still running five seconds after its stack was dropped"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
}
build/Containerfile
+573
View File
@@ -0,0 +1,573 @@
# ═══════════════════════════════════════════════════════════
# nestri guest rootfs — the open half
#
# Builds a bootable Arch image containing Mesa (virtio-gpu native context)
# and the five open guest components: nesinit, nescope, neshub, neswire,
# nescapture. Two leaf targets, selected with `--target`:
#
# runtime_prod stripped, root locked (default: `make build`)
# runtime_debug debug tools, autologin root (`make build-debug`)
#
# There is no service manager, no init scripts and no udev. `nesinit` is PID 1
# and brings the box's services up from a table compiled into it, which is why
# this image is plain Arch rather than a distribution chosen for its init.
# ref(d-0064)
#
# Proton is here, and it is not a closed component: it is proton-cachyos built
# from source with --enable-wow64, which is what removes the need for a whole
# 32-bit host stack. Valve's steamclient.so is a different thing and is NOT
# here — that one is closed, and nestri/CLAUDE.md is explicit that nothing
# closed enters this repo. Whatever layers it on top of runtime_prod is a
# closed build outside this repo — see build/README.md.
#
# Build from the repo root, not from build/:
# docker build -f build/Containerfile --target runtime_prod -t nestri-guest .
# (`make build` in this directory does exactly that.)
# ═══════════════════════════════════════════════════════════
# Declared here and not beside the stage that uses it, because an ARG a FROM
# expands has to precede the *first* FROM in the file. Anywhere else it is
# scoped to one stage instead, `FROM ${PROTON_IMAGE}` expands to nothing, and
# the build fails with "no FROM statement found" — which says nothing about
# the actual mistake. See the Proton stage below for what this is.
ARG PROTON_IMAGE=ghcr.io/nestrilabs/proton-cachyos-native-wow64:11.0-20260703
# ───────────────────────────────────────────────────────────
# initial / builder
#
# The same distribution the guest is now, which it did not use to be: the
# guest was Artix, chosen for an init system this image no longer contains.
# Only build artifacts leave these stages.
# ───────────────────────────────────────────────────────────
FROM docker.io/archlinux:base-devel AS initial
RUN pacman -Syu --noconfirm
FROM initial AS builder
RUN pacman -S --noconfirm --needed \
cmake meson ninja git pkgconf \
python python-mako python-yaml python-packaging python-ply \
bison flex \
libpciaccess libepoxy libglvnd \
libx11 libxext libxrandr libxshmfence libxfixes libxxf86vm libxcb \
xcb-util-keysyms xorgproto \
wayland wayland-protocols \
expat zlib zstd libxml2 lm_sensors \
llvm clang libclc spirv-tools spirv-llvm-translator glslang \
elfutils libva libdrm directx-headers \
rust rust-bindgen cbindgen \
curl openssl \
pixman libxkbcommon \
vulkan-headers vulkan-icd-loader \
pipewire shaderc opus \
libinput \
&& pacman -Scc --noconfirm
WORKDIR /build
ENV ARTIFACTS=/artifacts
# ───────────────────────────────────────────────────────────
# Mesa — the only piece still fetched from outside this tree
# ───────────────────────────────────────────────────────────
FROM builder AS mesa-build
ARG MESA_GIT=https://gitlab.freedesktop.org/mesa/mesa.git
ARG MESA_COMMIT=b316485dd75ca6ab6c16c113480fb94c57d86c95
ARG JOBS=
RUN test -n "$JOBS" || JOBS=$(nproc) && \
git clone --depth=1 --revision="${MESA_COMMIT}" "${MESA_GIT}" /build/mesa-src && \
cd /build/mesa-src && \
meson setup builddir \
-Dprefix=/usr \
-Dbuildtype=release \
-Dplatforms=wayland \
-Dgallium-drivers=zink \
-Dvulkan-drivers=amd,intel \
-Damdgpu-virtio=true \
-Dintel-virtio-experimental=true \
-Dvideo-codecs=all \
-Degl=disabled \
-Dglx=disabled \
-Dgles1=disabled \
-Dgles2=disabled \
-Dgbm=disabled \
-Dgallium-va=disabled \
-Db_ndebug=true && \
ninja -C builddir -j${JOBS:-$(nproc)} && \
DESTDIR=/artifacts/mesa ninja -C builddir install && \
rm -rf /build/mesa-src && \
find /artifacts/mesa -type f -printf '/%P\n' > /artifacts/mesa/.manifest
# ───────────────────────────────────────────────────────────
# nestri workspace — same repo now, so this is COPY, not a private clone
#
# One `cargo build --release` over the guest members rather than one stage per
# binary:
# that per-repo splitting existed because nescope/neswire/nescapture/the hub
# were four separate private repos and a stage boundary was the only way to
# stop bumping one from invalidating the others' build cache. They are one
# Cargo workspace with one Cargo.lock now, so a BuildKit cache mount on
# target/ gives the same isolation — cargo's own incremental compiler
# already knows nescope changing does not touch nesprotocol's .rlib — without
# four copies of every shared dependency getting compiled once per stage.
#
# The members are named rather than `--workspace`, because the workspace holds
# one crate that is not part of a guest — `nesdoctor` runs on a stranger's own
# machine — and building it here would compile something this image will never
# contain. Naming them also means adding a member does not silently add a
# binary to the image.
# ───────────────────────────────────────────────────────────
FROM builder AS nestri-src
WORKDIR /build/nestri
COPY Cargo.toml Cargo.lock ./
COPY crates/nesprotocol crates/nesprotocol
# Not a guest component and not installed below — it runs on a stranger's own
# machine. It is here because `cargo` loads every workspace member's manifest
# before it builds anything, so a member missing from the context fails the
# build outright with `failed to read .../Cargo.toml`. Copying it costs a few
# files; the member list is the thing that decides, not this build.
COPY apps/nesdoctor apps/nesdoctor
COPY apps/nesinit apps/nesinit
COPY apps/nescope apps/nescope
COPY apps/neshub apps/neshub
COPY apps/neswire apps/neswire
COPY apps/nescapture apps/nescapture
FROM nestri-src AS nestri-build
RUN --mount=type=cache,target=/root/.cargo/registry \
--mount=type=cache,target=/build/nestri/target \
cargo build --release \
-p nesinit -p nescope -p neshub -p neswire -p nescapture && \
mkdir -p /artifacts/nestri/usr/bin /artifacts/nestri/usr/lib \
/artifacts/nestri/usr/share/vulkan/implicit_layer.d && \
install -Dm755 target/release/nesinit /artifacts/nestri/usr/bin/nesinit && \
install -Dm755 target/release/nescope /artifacts/nestri/usr/bin/nescope && \
install -Dm755 target/release/neshub /artifacts/nestri/usr/bin/neshub && \
install -Dm755 target/release/neswire /artifacts/nestri/usr/bin/neswire && \
install -Dm755 target/release/libnescapture_layer.so \
/artifacts/nestri/usr/lib/libnescapture_layer.so && \
install -Dm644 apps/nescapture/manifest/VK_LAYER_nescapture.json \
/artifacts/nestri/usr/share/vulkan/implicit_layer.d/VK_LAYER_nescapture.json && \
find /artifacts/nestri -type f -printf '/%P\n' > /artifacts/nestri/.manifest
# ───────────────────────────────────────────────────────────
# Proton — pulled, not built here
#
# Building it takes hours and it changes only when its own tag moves, so it
# has a cadence of its own and an image of its own. The published image is
# `FROM scratch` over the tree, so its root *is* the tree and there is nothing
# in it to run — only something to copy from.
#
# Built with `--enable-wow64`, which is the whole reason it is a build of ours
# rather than the distribution's package. wow64 runs 32-bit Windows code
# inside a 64-bit unix process, so a box needs no lib32 anything: no 32-bit
# glibc, no second Mesa for i686, and — the one that matters most here — no
# second capture layer, because the game is a 64-bit process and loads the
# 64-bit Vulkan loader the existing layer already sits in. The distribution's
# package is built without the flag, which is exactly why it depends on
# lib32-*.
#
# Override to build it yourself; the tag is a version and moves deliberately.
# ───────────────────────────────────────────────────────────
FROM ${PROTON_IMAGE} AS proton
# ───────────────────────────────────────────────────────────
# os-base — the Arch rootfs itself
#
# `FROM archlinux:base` directly, and `pacman -S` as plain RUN steps — not a
# privileged host `chroot` into a hand-extracted tarball, which would need
# /proc, /sys and /dev bind-mounted in first (they don't exist inside a chroot
# target until something puts them there). A Containerfile RUN step already
# executes inside a real container with its own /proc, /sys, /dev, so there is
# no bind-mount step to write at all.
#
# # systemd goes; systemd-libs stays
#
# Nothing in the package list below depends on `systemd`, and two things in it
# — dbus-daemon and wireplumber — link `libsystemd.so.0`, which comes from the
# separate `systemd-libs` package. So the removal is `-Rdd` of `systemd` and
# `systemd-sysvcompat` only, which is normal rather than a compromise: keeping
# the library while having no PID 1 from it is exactly how a distribution
# without systemd ships these same programs.
#
# Removing the package also removes its pacman hooks, which is the point. The
# hooks call `systemd-tmpfiles`, `systemd-sysusers` and `udevadm`; leaving them
# behind while deleting what they call is how a later transaction fails
# obscurely, and a previous attempt at this image lost two services to exactly
# that.
# ───────────────────────────────────────────────────────────
FROM docker.io/archlinux:base AS os-base
# The base image ships an unsigned local keyring, so upgrading
# `archlinux-keyring` runs a hook that reports `There is no secret key
# available to sign with` and then `error: command failed to execute
# correctly`. It is cosmetic and it is also every Arch container's build log.
# One line fixes it, and it is worth the seconds: a build that always prints an
# error is a build nobody reads an error out of.
RUN pacman-key --init
# Installed first and removed second, so every dependency resolves normally
# before anything is taken out from under it.
#
# Gone with the init system: `openrc`, `udev`, `dbus-openrc`. `udev` is not
# replaced by anything — `devtmpfs` creates the nodes and init sets the two
# modes that matter, because the compositor takes input through Wayland and
# opens nothing udev provides. ref(d-0064)
#
# `logrotate` is also gone, and that one is not about the init system: a box
# keeps no logs to rotate. What is worth reading leaves over the control
# channel, and `/var/log` is a small tmpfs that is discarded with the box.
# `mesa` is not in this list, and the two `--assume-installed` flags are why.
#
# The distribution's Mesa used to be installed so that every runtime dependency
# of *a* Mesa was present and correctly versioned, and ours was then overlaid
# on top. That worked for the unversioned filenames and not for the versioned
# one: `libgallium-<version>.so` from the package sat beside ours, 53 MB of it,
# referenced by nothing. Installing it to overwrite most of it was always the
# roundabout way round; telling pacman the dependency is already satisfied is
# the direct one.
#
# Exactly two flags are needed and both were checked by dropping each in turn:
# `mesa` is the name two packages depend on, and `opengl-driver` is a virtual
# provide `libglvnd` requires that only a real driver package satisfies. The
# other three names Mesa provides — `mesa-libgl`, `libva-driver`,
# `libva-mesa-driver` — change nothing here, so they are not listed.
#
# What makes this safe is that our Mesa is a superset for this image's
# purposes: it builds the drivers a box can actually use and the package's
# other ones (apple, asahi, armada, d3d12) are for hardware no box has. What it
# does *not* build is a software rasteriser, so there is no llvmpipe fallback —
# a box with no working GPU path now fails instead of rendering slowly, which
# is the honest outcome for something that exists to stream frames.
#
# Two packages below are explicit *because* Mesa is gone, and both used to
# arrive as its dependencies: `llvm-libs`, which the radeonsi driver links for
# shader compilation, and `lm_sensors`, which it links for `libsensors.so.5`.
# The second was found by the check further down rather than by reading the
# list — dropping a package takes its dependency tree with it, and the part of
# that tree something else was quietly using is not visible from here.
RUN pacman -Syu --noconfirm --needed \
--assume-installed mesa --assume-installed opengl-driver \
dbus \
iptables iproute2 \
libglvnd libdrm libepoxy libxxf86vm libinput wayland \
expat zlib llvm-libs lm_sensors elfutils libva shaderc vulkan-icd-loader \
pixman libxkbcommon xcb-util-keysyms xorg-xwayland \
pipewire pipewire-audio wireplumber opus \
python libunwind \
&& rm -f /usr/share/libalpm/hooks/dbus-reload.hook \
&& pacman -Rdd --noconfirm systemd systemd-sysvcompat \
&& pacman -Scc --noconfirm
# `libunwind` is Wine's, not ours. `ntdll.so` links it, so without it every
# process Wine starts dies at `could not load ntdll.so`, which is the first
# thing it loads and reads like Wine itself being broken. Found 2026-09-12,
# after the prefix had already been created -- so the session got past every
# check that Proton was present and usable.
#
# `python` is not a build dependency here -- the builder stage has its own for
# Mesa -- it is a *runtime* one. The compatibility tool's entry point is a
# Python script, so a box without an interpreter starts a game and the launch
# ends with `env: 'python3': No such file or directory` and an exit status that
# reads like an ordinary finish. Found 2026-09-12, on the first session that
# got as far as launching one.
# `dbus-reload.hook` is deleted above, before the removal rather than after,
# and it is the whole reason that line is there: the hook runs
# `/usr/share/libalpm/scripts/systemd-hook`, which systemd owns, so the
# transaction that removes systemd trips its own leftover on the way out —
# `call to execv failed`, then `error: command failed to execute correctly`.
# pacman treats a post-transaction hook failure as non-fatal, so the build
# survives it and the image is fine; what it leaves is an error message in
# every future transaction and a reader with no way to tell it from a real
# one. Removing the hook first means the error never happens.
# Nothing left may point at a program that is not here.
#
# The specific case above is fixed; this is the general one, and it exists
# because a hook calling a deleted binary is the exact shape of the failure
# that took two services off a previous version of this image. A build error
# is a much better place to find the next one than a log.
RUN for hook in /usr/share/libalpm/hooks/*.hook; do \
exec_line="$(awk -F'= *' '/^Exec/ { print $2; exit }' "$hook")"; \
program="${exec_line%% *}"; \
case "$program" in /*) ;; *) continue ;; esac; \
test -e "$program" \
|| { echo "$(basename "$hook") runs $program, which is not in the image" >&2; exit 1; }; \
done
# The check, because the removal above is the kind of thing a later `pacman
# -Syu` undoes quietly. A box with systemd's PID 1 back in it boots something
# other than `nesinit`, and the symptom is a guest that never dials out.
RUN test ! -e /usr/lib/systemd/systemd \
|| { echo "systemd's PID 1 is back in the image" >&2; exit 1; }
# `groupadd -f`, because some of these already exist in the base image and the
# rest have to. Nothing creates them at runtime any more: udev's rules did that
# for device nodes, and with udev gone init sets the two modes that matter
# directly. ref(d-0064)
#
# **Two users, and they must stay two.** `nestri` runs the services that come
# with this image; `nesplay` is who a workload runs as. Sharing one user between
# them is what lets workload code impersonate a service — it can replace the
# socket a service listens on and answer in its place, and the answer that
# matters is the address a client is told to connect to. Init refuses an address
# served by the workload's own user, so a single shared user does not merely
# weaken that check, it makes every session fail it.
#
# The uid a workload actually runs as is chosen by whoever asks for the box, not
# here; this account exists so that the number has a home, a shell and a name in
# `ps`, and so the separation has somewhere to be written down.
RUN groupadd -f audio && groupadd -f video && groupadd -f input && groupadd -f render && \
useradd -m -u 1000 -s /bin/bash nestri && \
for g in audio video input render; do gpasswd -a nestri "$g" >/dev/null; done && \
useradd -m -u 1001 -s /bin/bash nesplay && \
for g in audio video input render; do gpasswd -a nesplay "$g" >/dev/null; done
# ───────────────────────────────────────────────────────────
# runtime — everything common to debug and prod
# ───────────────────────────────────────────────────────────
FROM os-base AS runtime
# This is what GHCR actually uses to connect a pushed image back to its
# repo — not a setting to toggle after the fact, a label the image has to
# carry. Without it a manually-pushed image shows no "used by" repo on its
# package page even though this Containerfile is exactly what built it.
LABEL org.opencontainers.image.source="https://github.com/nestrilabs/nestri"
# Our own builds, overlaid on the distro's mesa. The distro package landed
# first (above) so every runtime dependency of *a* Mesa is present and
# correctly versioned; this overwrites its .so files with ours.
#
# COPY --from runs as root inside this build with no invoking-user uid to
# stamp onto / or /usr/bin, unlike a host-side `podman cp` + `cp -a` — so
# there is no ownership-sanity-check to write here. Nothing to catch, on
# purpose, not an oversight.
COPY --from=mesa-build /artifacts/mesa/.manifest /tmp/mesa.manifest
COPY --from=nestri-build /artifacts/nestri/.manifest /tmp/nestri.manifest
RUN cat /tmp/mesa.manifest /tmp/nestri.manifest > /tmp/.strip-manifest && \
rm -f /tmp/mesa.manifest /tmp/nestri.manifest
COPY --from=mesa-build /artifacts/mesa /
COPY --from=nestri-build /artifacts/nestri /
# The Proton tree, whose image root is the tree, so this lands it at
# /usr/share/steam/compatibilitytools.d/proton-cachyos.
#
# Deliberately not in the strip manifest above: that list is our own build
# output, and the two stripping decisions are not the same one. Proton ships
# a Windows toolchain's worth of PE binaries that `strip` has no business
# touching, and its unix side is already built the way its own packaging
# builds it.
COPY --from=proton / /
RUN ldconfig
COPY build/etc/ /etc/
# `nesinit` is PID 1, and `/usr/bin/init` is the fallback for a kernel started
# without an explicit `init=`. `systemd-sysvcompat` used to own that path and
# was removed with the rest of systemd, so nothing else claims it.
RUN ln -sf nesinit /usr/bin/init
# One id per boot, not one per image.
#
# `dbus-uuidgen --ensure=/etc/machine-id` used to run here, which baked one id
# into the image and made every box built from it the same machine. Init writes
# a fresh one to /run at boot instead, so both of these are symlinks into a
# tmpfs — which is also the only place they could be, with a read-only root.
RUN rm -f /etc/machine-id /var/lib/dbus/machine-id && \
mkdir -p /var/lib/dbus && \
ln -sf /run/machine-id /etc/machine-id && \
ln -sf /run/machine-id /var/lib/dbus/machine-id
# Session mount points. The guest root is read-only at runtime, so a runtime
# mkdir gets EROFS and takes a service down before it starts — these have to
# already exist in the image.
#
# `/nestri/logs` is a mount point and nothing mounts it from in here any more:
# a share arrives because the caller named it in the boot descriptor, which is
# the same rule every other share follows. The directory stays so that naming
# it works.
RUN mkdir -p /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \
chmod 0755 /nestri /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \
mkdir -p /dev/shm && chmod 1777 /dev/shm && \
mkdir -p /run/user/1000 /var/log && \
# The distribution's own empty `fstab` goes with ours. Nothing in a box
# reads either: init mounts what a box always needs, and every share comes
# from the boot descriptor. A file that looks like it configures mounts and
# is read by nothing is a file somebody edits expecting an effect.
rm -f /etc/network/interfaces /etc/inittab /etc/fstab
# Nothing in this image may be an init system except `nesinit`.
#
# A service manager arriving as a dependency of something innocuous is the
# failure this catches, and it is silent otherwise: the extra init does not run
# — the kernel is told which one to start — it just sits there with its own
# ideas about what the box should be doing, waiting for somebody to wire it in.
RUN for intruder in /usr/lib/systemd/systemd /sbin/openrc-init /usr/bin/openrc-init \
/sbin/runit-init /usr/bin/runit-init /sbin/dinit /usr/bin/dinit; do \
test ! -e "$intruder" || { echo "a second init is in the image: $intruder" >&2; exit 1; }; \
done
# What `nesinit` will look for at runtime, checked while there is somebody to
# read the failure.
#
# It is a table compiled into a binary, so a missing program is not a build
# error — it is a service that does not come up in a box somebody is waiting
# on, reported over the control channel and read hours later. Checking here
# turns that into a failed build.
# Everything this image promises must resolve the libraries it links.
#
# This is the check the Mesa change needs: dropping a package that provided
# libraries is how a binary ends up resolving nothing, and the symptom is not a
# build failure — it is a service that will not start in a box somebody is
# waiting on, or a render path that is missing rather than slow. It caught
# exactly that on the first run, and the missing library was two levels down a
# dependency tree nobody had reason to read.
#
# **Named rather than swept, and that is deliberate.** A sweep over everything
# in /usr/lib fails on a stock image: a distribution ships optional plugins
# whose optional dependencies are not installed — pinentry's Qt build, mpg123's
# JACK output, libdecor's GTK backend — and every one of those was already
# unresolved before this stage existed. A check that reports a dozen things
# nobody intends to load is a check the next person deletes. This list is what
# the image is *for*: the components, the services init starts, the chain
# between a workload and the GPU, and Wine's own core.
#
# **Wine was added after it was missed**, and then narrowed twice, which is
# worth recording so nobody widens it again.
#
# It was missed because the list covered everything this image ships *of ours*
# and nothing of the compatibility tool's, so an unresolved `libunwind.so.8`
# behind `ntdll.so` survived a build whose whole purpose is catching that, and
# surfaced as a session that created a prefix and could not start one process
# in it.
#
# The obvious fix -- sweep every `*-unix/*.so` -- is wrong in both directions.
# It is noisy: those objects are Wine's optional backends, and their
# dependencies are a camera library, a media stack, a VR loader, a smartcard
# daemon and OpenCL, none of which belong in a box. And it cannot see what it
# is checking: Wine's unix objects **link each other by soname** and are
# resolved by Wine's own loader rather than by `ld.so`, so `ldd` reports
# `ntdll.so` and `win32u.so` themselves as missing while they sit in the same
# directory. Forty files, every one a false positive, hiding the one real
# entry.
#
# So: the programs in `bin/`, which are ordinary ELF and resolve normally, and
# `ntdll.so`, which is the first thing Wine loads and the one that linked the
# missing library. That is exactly the failure that got through, with none of
# the noise. The Windows-side DLLs beside them are not ELF and `ldd` skips them
# anyway.
# The output is one file per line with its own missing libraries under it, and
# then every missing library once at the end. That last list is what somebody
# acts on -- it is the set of packages to add -- and forty files each naming the
# same two libraries is not that list. An earlier version printed one
# comma-joined line and cut the wrong field out of `ldd`, so it named no
# libraries at all: `ldd` indents with a tab, which `tr -s ' '` does not
# collapse, so the second space-separated field is `=>`.
RUN failed=0; \
: > /tmp/missing-libs; \
for f in /usr/bin/nesinit /usr/bin/nescope /usr/bin/neshub /usr/bin/neswire \
/usr/lib/libnescapture_layer.so \
/usr/bin/dbus-daemon /usr/bin/pipewire /usr/bin/wireplumber /usr/bin/ip \
/usr/lib/libgallium-*.so /usr/lib/libEGL_mesa.so.0 \
/usr/lib/libvulkan_*.so /usr/lib/dri/*.so /usr/lib/gbm/*.so \
/usr/share/steam/compatibilitytools.d/proton-cachyos/files/bin/* \
/usr/share/steam/compatibilitytools.d/proton-cachyos/files/lib*/wine/*-unix/ntdll.so; do \
[ -e "$f" ] || continue; \
libs="$(ldd "$f" 2>/dev/null | awk '/not found/ { print $1 }')"; \
[ -n "$libs" ] || continue; \
failed=1; \
printf ' %s\n' "$f" >&2; \
printf ' %s\n' $libs >&2; \
printf '%s\n' $libs >> /tmp/missing-libs; \
done; \
if [ "$failed" != 0 ]; then \
echo "" >&2; \
echo " every library above, once each -- this is the list to install:" >&2; \
sort -u /tmp/missing-libs | sed 's/^/ /' >&2; \
exit 1; \
fi
RUN for required in /usr/bin/nesinit /usr/bin/nescope /usr/bin/neshub /usr/bin/neswire \
/usr/bin/dbus-daemon /usr/bin/pipewire /usr/bin/wireplumber /usr/bin/ip \
/usr/bin/python3 \
/usr/share/steam/compatibilitytools.d/proton-cachyos/proton; do \
test -x "$required" || { echo "the image is missing $required" >&2; exit 1; }; \
done
# An entry point that is executable is not an entry point that runs.
#
# The check above passed on an image whose compatibility tool was a Python
# script with no interpreter behind it: `test -x` says the file may be
# executed, and the kernel then fails to find what the shebang names. A session
# got as far as launching a game and ended with
# `env: 'python3': No such file or directory`.
#
# So every script this image promises resolves its own interpreter. `env` is
# unwrapped where it is used, because a shebang of `#!/usr/bin/env python3`
# names `env` and the thing that is actually missing is the argument.
RUN for script in /usr/share/steam/compatibilitytools.d/proton-cachyos/proton; do \
head -c 2 "$script" | grep -q '#!' || continue; \
shebang="$(head -1 "$script" | sed 's/^#!//')"; \
interpreter="${shebang%% *}"; \
case "$interpreter" in \
*/env) argument="${shebang#* }"; interpreter="$(command -v "${argument%% *}" || true)";; \
esac; \
test -n "$interpreter" && test -x "$interpreter" \
|| { echo "$script needs an interpreter the image does not have: $shebang" >&2; exit 1; }; \
done
# ───────────────────────────────────────────────────────────
# runtime_prod — the default: `make build`
# ───────────────────────────────────────────────────────────
FROM runtime AS runtime_prod
# No console is offered by either flavour: `nesinit` spawns no getty, because
# the way into a guest that will not boot is `init=/bin/bash` on the kernel
# command line, which needs nothing from the image but a shell. So the locked
# root account is belt and braces rather than the only thing standing between
# a box and a login prompt.
RUN passwd -l root
RUN while IFS= read -r f; do \
[ -f "$f" ] && strip --strip-unneeded "$f" 2>/dev/null || true; \
done < /tmp/.strip-manifest; \
rm -rf /tmp/.strip-manifest /var/cache/pacman/pkg/* /tmp/* /root/.cache \
/usr/share/man /usr/share/doc /usr/share/locale \
/usr/lib/cmake /usr/lib/pkgconfig /usr/share/pkgconfig /usr/include \
/usr/share/gir-1.0 /usr/lib/udev; \
find /usr/lib -name '*.a' -delete
RUN echo "NESTRI_STAGE=runtime_prod" >> /etc/os-release
# ───────────────────────────────────────────────────────────
# runtime_debug — `make build-debug`
# ───────────────────────────────────────────────────────────
FROM runtime AS runtime_debug
# `-Sy` and not `-Syu`: a full upgrade here can pull a package back in as
# somebody's dependency, and the one that matters is systemd. The check below
# catches it either way, but a debug image that fails to build is worse than
# one that is a few days behind on versions it only uses for `vulkaninfo`.
RUN pacman -Sy --noconfirm --needed vulkan-tools mesa-utils libva-utils && \
pacman -Scc --noconfirm
# The same guard as the runtime stage, because the transaction above is exactly
# the kind that quietly reinstates an init system.
RUN test ! -e /usr/lib/systemd/systemd \
|| { echo "systemd's PID 1 came back with the debug tools" >&2; exit 1; }
# Root has a password here and nothing offers a login prompt to type it into.
# It is for `su` from an `init=/bin/bash` shell, which is the whole debug route.
RUN echo 'root:nestri' | chpasswd
RUN echo "NESTRI_STAGE=runtime_debug" >> /etc/os-release
build/Containerfile.containerignore
+40
View File
@@ -0,0 +1,40 @@
# The guest rootfs build's context.
#
# The name is load-bearing and it is not `.containerignore`. Podman looks for
# an ignore file *adjacent to the Containerfile and named after it* — here,
# `Containerfile.containerignore` — before falling back to one at the root of
# the build context. The context is the repository root, so a bare
# `build/.containerignore` sits in neither place and is silently read by
# nothing: the build still works, it just sends the whole tree.
#
# Docker looks for the `.dockerignore` suffix only, so a docker build reads the
# repository-root file instead of this one and sends more than it needs. That
# is the cost of the container-agnostic name and it is only a cost in bytes.
#
# This file *replaces* the repository-wide ignore file rather than adding to
# it, so the first block below is that file repeated. The second is what only
# this build excludes.
#
# This build's context is the repository root (see `Makefile`), and it COPYs
# the workspace manifests plus the Rust members and nothing else. The
# TypeScript half is therefore dead weight in the context — a few megabytes
# sent to the daemon versus the whole tree.
.git
node_modules
target
build/output
.env
.env.*
.wrangler
dist
.output
docs
apps/api
apps/auth
packages
*.md
deno.lock
bun.lock
.zed
.github
build/Containerfile.proton
+102
View File
@@ -0,0 +1,102 @@
# ═══════════════════════════════════════════════════════════
# proton-cachyos, built wow64-only
#
# Separate from the guest Containerfile because it is a separate cadence:
# hours to build, and only when PROTON_TAG moves. The guest image pulls the
# result from a registry instead of rebuilding it, which is why this file is
# not part of that build and is not reached by `make build`.
#
# The final stage is FROM scratch, so the image *is* the Proton tree — nothing
# to run, only something to COPY --from.
#
# Build and publish with `make proton-image` / `make proton-push`. **Its
# context is this directory**, not the repository root the guest build uses:
# all it needs is the two scripts beside it, and a context of the whole tree
# would hand it a multi-gigabyte `output/` for no reason.
#
# `PROTON_TAG` is the one thing to change, and the Makefile derives the
# published image's version from it. They are the same number in two
# spellings, and an image whose name does not say which Proton is inside it
# is worse than no image.
# ═══════════════════════════════════════════════════════════
FROM docker.io/archlinux:base-devel AS builder
# proton-cachyos-native's makedepends, minus every lib32-* (that is the whole
# point of --enable-wow64), with two substitutions Arch requires: ocl-icd
# provides opencl-icd-loader, and mesa-libgl is folded into libglvnd.
#
# glib2-devel is the one that is easy to miss — it carries glib-mkenums, which
# libsoup's meson looks up through glib-2.0's pkg-config variables and fails on
# obscurely. unzip and zip are for the xalia dist step. afdko is not packaged at
# all, and the fonts submodule needs it, so it comes from PyPI.
RUN pacman -Syu --noconfirm --needed \
alsa-lib clang cmake curl ffmpeg fontforge giflib git glib2-devel \
glslang gnutls gtk3 libgphoto2 libglvnd libpulse libva libxcomposite \
libxinerama libxxf86vm lld mesa meson ninja nasm \
opencl-headers ocl-icd pcsclite perl perl-json python python-pip \
python-pefile python-setuptools-scm rsync rust samba unixodbc \
unzip zip v4l-utils vulkan-headers vulkan-icd-loader wayland \
wayland-protocols wget xorg-util-macros \
mingw-w64-gcc mingw-w64-binutils mingw-w64-crt mingw-w64-headers \
mingw-w64-winpthreads \
&& pip install --break-system-packages --no-cache-dir afdko \
&& pacman -Scc --noconfirm
# widl generates vkd3d's public headers. Without it autoconf sets HAVE_WIDL to
# false, the headers are silently not generated, and the build dies an hour
# later on a missing vkd3d_d3dx9shader.h. Arch ships widl only inside `wine`,
# which requires multilib; the AUR's mingw-w64-tools builds it standalone, so do
# the same.
#
# The digest is not a formality. What is extracted here has its `configure` and
# its makefiles run as root in this builder, and what they produce is copied
# into the image a box runs -- so whoever can change these bytes can change what
# runs on every host. SourceForge hands the request to whichever mirror it
# likes, over a connection this builder does not pin, and the project publishes
# no signature. The digest is the only thing that makes the mirror not matter.
#
# Taken 2026-09-14 from two different mirrors of v14.0.0, which agreed. A
# mismatch here is not a thing to paper over by taking the new digest: it means
# the bytes behind this exact version string changed, and that wants looking at
# before it wants fixing.
ARG MINGW_W64_VER=14.0.0
ARG MINGW_W64_SHA256=6eaf921d9eb987d3820b364ea9775bc19b965ec81490b6fdd716526c28e1995c
RUN curl -fL --retry 3 -o /tmp/mingw-w64.tar.bz2 \
"https://sourceforge.net/projects/mingw-w64/files/mingw-w64/mingw-w64-release/mingw-w64-v${MINGW_W64_VER}.tar.bz2/download" \
&& echo "${MINGW_W64_SHA256} /tmp/mingw-w64.tar.bz2" | sha256sum -c - \
&& tar xf /tmp/mingw-w64.tar.bz2 -C /tmp \
&& for arch in i686-w64-mingw32 x86_64-w64-mingw32; do \
mkdir -p "/tmp/widl-${arch}" && cd "/tmp/widl-${arch}" \
&& "/tmp/mingw-w64-v${MINGW_W64_VER}/mingw-w64-tools/widl/configure" \
--prefix=/usr --target="${arch}" --program-prefix="${arch}-" \
&& make -j"$(nproc)" && make install; \
done \
&& rm -rf /tmp/mingw-w64* /tmp/widl-*
WORKDIR /build
ARG PROTON_GIT=https://github.com/CachyOS/proton-cachyos.git
ARG PROTON_TAG=cachyos-11.0-20260703-native
ARG GECKO_VER=2.47.4
ARG MONO_VER=11.2.0
ARG XALIA_VER=0.4.9
ARG JOBS=
# Fetch and build are separate layers on purpose: the submodule checkout runs
# well past ten minutes, and a build that fails on a flag or a missing tool must
# not pay for it again.
COPY scripts/proton-fetch.sh /build/proton-fetch.sh
RUN PROTON_GIT="${PROTON_GIT}" PROTON_TAG="${PROTON_TAG}" \
GECKO_VER="${GECKO_VER}" MONO_VER="${MONO_VER}" XALIA_VER="${XALIA_VER}" \
bash /build/proton-fetch.sh
COPY scripts/proton-build.sh /build/proton-build.sh
RUN GECKO_VER="${GECKO_VER}" MONO_VER="${MONO_VER}" JOBS="${JOBS}" \
bash /build/proton-build.sh
# ═══════════════════════════════════════════════════════════
# The publishable artifact: the Proton tree and nothing else
# ═══════════════════════════════════════════════════════════
FROM scratch
COPY --from=builder /artifacts/proton/ /
build/Containerfile.proton.containerignore
+15
View File
@@ -0,0 +1,15 @@
# This build's context is `build/`, not the repository root, because all it
# needs is the two scripts beside the Containerfile.
#
# `output/` is the reason this file exists. It holds packed rootfs images —
# multiple gigabytes each — and a build context is copied before the first
# instruction runs, so without this line every Proton build starts by moving
# the last one it produced.
output
# Nothing else here is an input to this build.
etc
README.md
Makefile
Containerfile
Containerfile.containerignore
build/Dockerfile
-309
View File
@@ -1,309 +0,0 @@
# ═══════════════════════════════════════════════════════════
# nestri guest rootfs — the open half
#
# Builds a bootable Artix/OpenRC image containing Mesa (virtio-gpu native
# context) and the four open guest components: nescope, neshub, neswire,
# nescapture. Two leaf targets, selected with `--target`:
#
# runtime_prod stripped, root locked (default: `make build`)
# runtime_debug debug tools, autologin root (`make build-debug`)
#
# What is deliberately NOT here: Proton, Valve's steamclient.so, or anything
# else closed. nestri/CLAUDE.md is explicit that nothing closed enters this
# repo. Whatever layers those on top of runtime_prod is a closed build
# outside this repo — see build/README.md.
#
# Build from the repo root, not from build/:
# docker build -f build/Dockerfile --target runtime_prod -t nestri-guest .
# (`make build` in this directory does exactly that.)
# ═══════════════════════════════════════════════════════════
# ───────────────────────────────────────────────────────────
# initial / builder — Arch, not Artix
#
# The guest is Artix, but Artix's repos are Arch-derived and the toolchain/
# glibc generation is the same, while the archlinux image is the
# better-maintained of the two for actually compiling things. Only build
# artifacts leave these stages.
# ───────────────────────────────────────────────────────────
FROM docker.io/archlinux:base-devel AS initial
RUN pacman -Syu --noconfirm
FROM initial AS builder
RUN pacman -S --noconfirm --needed \
cmake meson ninja git pkgconf \
python python-mako python-yaml python-packaging python-ply \
bison flex \
libpciaccess libepoxy libglvnd \
libx11 libxext libxrandr libxshmfence libxfixes libxxf86vm libxcb \
xcb-util-keysyms xorgproto \
wayland wayland-protocols \
expat zlib zstd libxml2 lm_sensors \
llvm clang libclc spirv-tools spirv-llvm-translator glslang \
elfutils libva libdrm directx-headers \
rust rust-bindgen cbindgen \
curl openssl \
pixman libxkbcommon \
vulkan-headers vulkan-icd-loader \
pipewire shaderc opus \
libinput \
&& pacman -Scc --noconfirm
WORKDIR /build
ENV ARTIFACTS=/artifacts
# ───────────────────────────────────────────────────────────
# Mesa — the only piece still fetched from outside this tree
# ───────────────────────────────────────────────────────────
FROM builder AS mesa-build
ARG MESA_GIT=https://gitlab.freedesktop.org/mesa/mesa.git
ARG MESA_COMMIT=b78fc73dd898a7dfa87448a4b5ff4459a870e21a
RUN git clone --depth=1 --revision="${MESA_COMMIT}" "${MESA_GIT}" /build/mesa-src && \
cd /build/mesa-src && \
meson setup builddir \
-Dprefix=/usr \
-Dbuildtype=release \
-Dplatforms=wayland,x11 \
-Dgallium-drivers=zink,radeonsi,iris \
-Dvulkan-drivers=amd,intel \
-Damdgpu-virtio=true \
-Dintel-virtio-experimental=true \
-Dvideo-codecs=all \
-Degl=enabled \
-Dglx=dri \
-Dgles1=enabled \
-Dgles2=enabled \
-Dgbm=enabled \
-Dgallium-va=enabled \
-Db_ndebug=true && \
ninja -C builddir && \
DESTDIR=/artifacts/mesa ninja -C builddir install && \
rm -rf /build/mesa-src && \
find /artifacts/mesa -type f -printf '/%P\n' > /artifacts/mesa/.manifest
# ───────────────────────────────────────────────────────────
# nestri workspace — same repo now, so this is COPY, not a private clone
#
# One `cargo build --release --workspace` rather than one stage per binary:
# that per-repo splitting existed because nescope/neswire/nescapture/the hub
# were four separate private repos and a stage boundary was the only way to
# stop bumping one from invalidating the others' build cache. They are one
# Cargo workspace with one Cargo.lock now, so a BuildKit cache mount on
# target/ gives the same isolation — cargo's own incremental compiler
# already knows nescope changing does not touch nesprotocol's .rlib — without
# four copies of every shared dependency getting compiled once per stage.
# ───────────────────────────────────────────────────────────
FROM builder AS nestri-src
WORKDIR /build/nestri
COPY Cargo.toml Cargo.lock ./
COPY crates/nesprotocol crates/nesprotocol
COPY apps/nescope apps/nescope
COPY apps/neshub apps/neshub
COPY apps/neswire apps/neswire
COPY apps/nescapture apps/nescapture
FROM nestri-src AS nestri-build
RUN --mount=type=cache,target=/root/.cargo/registry \
--mount=type=cache,target=/build/nestri/target \
cargo build --release --workspace && \
mkdir -p /artifacts/nestri/usr/bin /artifacts/nestri/usr/lib \
/artifacts/nestri/usr/share/vulkan/implicit_layer.d && \
install -Dm755 target/release/nescope /artifacts/nestri/usr/bin/nescope && \
install -Dm755 target/release/neshub /artifacts/nestri/usr/bin/neshub && \
install -Dm755 target/release/neswire /artifacts/nestri/usr/bin/neswire && \
install -Dm755 target/release/libnescapture_layer.so \
/artifacts/nestri/usr/lib/libnescapture_layer.so && \
install -Dm644 apps/nescapture/manifest/VK_LAYER_nescapture.json \
/artifacts/nestri/usr/share/vulkan/implicit_layer.d/VK_LAYER_nescapture.json && \
find /artifacts/nestri -type f -printf '/%P\n' > /artifacts/nestri/.manifest
# ───────────────────────────────────────────────────────────
# os-base — the Artix rootfs itself
#
# `FROM artixlinux/artixlinux:base-openrc` directly, and `pacman -S` as plain
# RUN steps — not a privileged host `chroot` into a hand-extracted tarball,
# which would need /proc, /sys and /dev bind-mounted in first (they don't
# exist inside a chroot target until something puts them there). A
# Dockerfile RUN step already executes inside a real container with its own
# /proc, /sys, /dev, so there is no bind-mount step to write at all.
# ───────────────────────────────────────────────────────────
FROM docker.io/artixlinux/artixlinux:base-openrc AS os-base
RUN pacman -Syu --noconfirm --needed \
base openrc udev dbus dbus-openrc \
iptables iproute2 \
mesa libglvnd libdrm libepoxy libxxf86vm libinput wayland \
expat zlib llvm-libs elfutils libva shaderc vulkan-icd-loader \
pixman libxkbcommon xcb-util-keysyms xorg-xwayland \
pipewire pipewire-audio wireplumber dbus logrotate opus \
&& pacman -Scc --noconfirm
# groupadd -f so this is idempotent whether or not udev's rules already
# created these.
#
# **Two users, and they must stay two.** `nestri` runs the services that come
# with this image; `nesplay` is who a workload runs as. Sharing one user between
# them is what lets workload code impersonate a service — it can replace the
# socket a service listens on and answer in its place, and the answer that
# matters is the address a client is told to connect to. Init refuses an address
# served by the workload's own user, so a single shared user does not merely
# weaken that check, it makes every session fail it.
#
# The uid a workload actually runs as is chosen by whoever asks for the box, not
# here; this account exists so that the number has a home, a shell and a name in
# `ps`, and so the separation has somewhere to be written down.
RUN groupadd -f audio && groupadd -f video && groupadd -f input && groupadd -f render && \
useradd -m -u 1000 -s /bin/bash nestri && \
for g in audio video input render; do gpasswd -a nestri "$g" >/dev/null; done && \
useradd -m -u 1001 -s /bin/bash nesplay && \
for g in audio video input render; do gpasswd -a nesplay "$g" >/dev/null; done
# ───────────────────────────────────────────────────────────
# runtime — everything common to debug and prod
# ───────────────────────────────────────────────────────────
FROM os-base AS runtime
# This is what GHCR actually uses to connect a pushed image back to its
# repo — not a setting to toggle after the fact, a label the image has to
# carry. Without it a manually-pushed image shows no "used by" repo on its
# package page even though this Dockerfile is exactly what built it.
LABEL org.opencontainers.image.source="https://github.com/nestrilabs/nestri"
# Our own builds, overlaid on the distro's mesa. The distro package landed
# first (above) so every runtime dependency of *a* Mesa is present and
# correctly versioned; this overwrites its .so files with ours.
#
# COPY --from runs as root inside this build with no invoking-user uid to
# stamp onto / or /usr/bin, unlike a host-side `podman cp` + `cp -a` — so
# there is no ownership-sanity-check to write here. Nothing to catch, on
# purpose, not an oversight.
COPY --from=mesa-build /artifacts/mesa/.manifest /tmp/mesa.manifest
COPY --from=nestri-build /artifacts/nestri/.manifest /tmp/nestri.manifest
RUN cat /tmp/mesa.manifest /tmp/nestri.manifest > /tmp/.strip-manifest && \
rm -f /tmp/mesa.manifest /tmp/nestri.manifest
COPY --from=mesa-build /artifacts/mesa /
COPY --from=nestri-build /artifacts/nestri /
RUN ldconfig
COPY build/etc/ /etc/
RUN chmod +x /etc/init.d/*
# dbus-session, pipewire and wireplumber carry no conf.d of their own — they
# just want the shared environment, so their conf.d is a symlink to it rather
# than a copy. nescope, neshub and neswire are deliberately absent from this
# loop: each has a conf.d file of its own (already landed by the COPY above)
# that sources nestri-user-env, because each needs settings the shared file
# does not carry. Symlinking them here would overwrite those.
RUN for svc in dbus-session pipewire wireplumber; do \
ln -sf nestri-user-env "/etc/conf.d/${svc}"; \
done
RUN echo nesbox > /etc/hostname && dbus-uuidgen --ensure=/etc/machine-id
# Session mount points. The guest root is read-only at runtime, so a runtime
# mkdir gets EROFS and takes a service down before it starts — these have to
# already exist in the image.
RUN mkdir -p /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \
chmod 0755 /nestri /nestri/install /nestri/user /nestri/work /nestri/game /nestri/logs && \
mkdir -p /dev/shm && chmod 1777 /dev/shm && \
rm -f /etc/network/interfaces
# Serial console: the only way into a guest that will not boot.
RUN rm -f /etc/inittab && \
ln -sf agetty /etc/init.d/agetty.hvc0 && \
{ grep -qx hvc0 /etc/securetty || echo hvc0 >> /etc/securetty; }
# Every init.d script calling nestri_export_env needs a conf.d that actually
# defines it. Worth a build-time check: the failure at runtime is nearly
# invisible — OpenRC sources the script, the undefined function is a
# "command not found" on stderr, sourcing still returns 0, and the service
# starts anyway with HOME and every XDG_* unset.
RUN missing=""; \
for svc_script in /etc/init.d/*; do \
grep -qE '^[[:space:]]*nestri_export_env\b' "$svc_script" 2>/dev/null || continue; \
svc="$(basename "$svc_script")"; \
. "/etc/conf.d/$svc" >/dev/null 2>&1; \
command -v nestri_export_env >/dev/null 2>&1 || missing="$missing $svc"; \
done; \
[ -z "$missing" ] || { echo "init scripts call nestri_export_env with no conf.d providing it:$missing" >&2; exit 1; }
# ── OpenRC service registration ──
# sysinit
RUN rc-update add devfs sysinit && \
rc-update add dmesg sysinit && \
rc-update add udev sysinit && \
rc-update add udev-trigger sysinit && \
(rc-update del kmod-static-nodes sysinit || true)
# boot — guest-net replaces the distro's networking scripts entirely; it
# declares `provide net` and brings up lo itself.
RUN (rc-update del networking boot || true) && \
(rc-update del network-async boot || true) && rm -f /etc/init.d/network-async && \
rc-update add guest-net boot && \
(rc-update del net.lo boot || true) && \
(rc-update del netmount boot || true) && \
(rc-update del netmount default || true) && \
rc-update add hostname boot && \
rc-update add xdg-runtime boot && \
rc-update add cgroups boot && \
(rc-update del syslog boot || true)
# Boot-runlevel services with nothing to do in a freshly-built microVM: no
# physical console, no swap, and fsck would be checking a filesystem nobody
# has touched.
RUN for svc in fsck keymaps save-keymaps termencoding save-termencoding swap binfmt seedrng hwclock swclock; do \
(rc-update del "$svc" boot || true); \
(rc-update del "$svc" shutdown || true); \
done
# default — the user-facing stack. nescope is registered in plain-compositor
# mode (no payload after `--`): it comes up and waits for something to
# connect. Actually starting a payload is nesinit's job, and nesinit is not
# open code yet — see build/README.md.
RUN rc-update add agetty.hvc0 default && \
for n in 1 2 3 4 5 6; do (rc-update del "agetty.tty${n}" default || true); done && \
rc-update add dbus default && \
rc-update add dbus-session default && \
rc-update add pipewire default && \
rc-update add wireplumber default && \
rc-update add neshub default && \
rc-update add neswire default && \
rc-update add nescope default && \
rm -f /etc/init.d/shared-root && (rc-update del shared-root boot || true)
# ───────────────────────────────────────────────────────────
# runtime_prod — the default: `make build`
# ───────────────────────────────────────────────────────────
FROM runtime AS runtime_prod
RUN passwd -l root
COPY build/etc/conf.d/agetty.hvc0.prod /etc/conf.d/agetty.hvc0
RUN while IFS= read -r f; do \
[ -f "$f" ] && strip --strip-unneeded "$f" 2>/dev/null || true; \
done < /tmp/.strip-manifest; \
rm -rf /tmp/.strip-manifest /var/cache/pacman/pkg/* /tmp/* /root/.cache \
/usr/share/man /usr/share/doc /usr/share/locale \
/usr/lib/cmake /usr/lib/pkgconfig /usr/share/pkgconfig /usr/include
RUN echo "NESTRI_STAGE=runtime_prod" >> /etc/os-release
# ───────────────────────────────────────────────────────────
# runtime_debug — `make build-debug`
# ───────────────────────────────────────────────────────────
FROM runtime AS runtime_debug
RUN pacman -Syu --noconfirm --needed vulkan-tools mesa-utils libva-utils && \
pacman -Scc --noconfirm
RUN echo 'root:nestri' | chpasswd
COPY build/etc/conf.d/agetty.hvc0.debug /etc/conf.d/agetty.hvc0
RUN echo "NESTRI_STAGE=runtime_debug" >> /etc/os-release
build/Dockerfile.dockerignore
-29
View File
@@ -1,29 +0,0 @@
# The guest rootfs build's context.
#
# A `<Dockerfile>.dockerignore` *replaces* the repository-wide `.dockerignore`
# rather than adding to it, so the first block below is that file repeated. The
# second is what only this build excludes.
#
# This build's context is the repository root (see `Makefile`), and it COPYs
# the workspace manifests plus the Rust members and nothing else. The
# TypeScript half is therefore dead weight in the context — a few megabytes
# sent to the daemon versus the whole tree.
.git
node_modules
target
build/output
.env
.env.*
.wrangler
dist
.output
docs
apps/api
apps/auth
packages
*.md
deno.lock
bun.lock
.zed
.github
build/Makefile
+39 -6
View File
@@ -1,12 +1,12 @@
SHELL := /bin/bash SHELL := /bin/bash
.PHONY: build build-debug image image-debug clean help .PHONY: build build-debug image image-debug proton-image proton-push clean help
CONTAINER_RT := $(shell command -v docker 2>/dev/null || command -v podman 2>/dev/null) CONTAINER_RT := $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null)
ifeq ($(CONTAINER_RT),) ifeq ($(CONTAINER_RT),)
$(error "Neither docker nor podman found in PATH") $(error "Neither docker nor podman found in PATH")
endif endif
# The Dockerfile COPYs from apps/ and crates/, so the build context is the # The Containerfile COPYs from apps/ and crates/, so the build context is the
# repo root, not this directory — same reason borealis's build/ *is* its own # repo root, not this directory — same reason borealis's build/ *is* its own
# context: here the guest source lives one level up instead of inside build/. # context: here the guest source lives one level up instead of inside build/.
CONTEXT := .. CONTEXT := ..
@@ -27,17 +27,47 @@ CONTEXT := ..
# where it does not exist. One name, always pullable, beats a local-only # where it does not exist. One name, always pullable, beats a local-only
# short name and a different published one. # short name and a different published one.
IMAGE_NAME := ghcr.io/nestrilabs/nestri/base IMAGE_NAME := ghcr.io/nestrilabs/nestri/base
# Proton is the one artifact worth publishing on its own: it takes hours to
# build and changes only when its tag moves, so the guest build pulls it by
# name instead of rebuilding it.
#
# `PROTON_TAG` is the single place to change it. The published version is
# derived from the tag rather than written twice, because the two are the same
# number in two spellings and an image whose name does not say which Proton is
# inside it is worse than no image at all. Changing the tag by hand and
# forgetting the version is exactly the mistake this removes.
PROTON_TAG ?= cachyos-11.0-20260703-native
PROTON_VERSION := $(PROTON_TAG:cachyos-%-native=%)
PROTON_IMAGE ?= ghcr.io/nestrilabs/proton-cachyos-native-wow64
PROTON_REF := $(PROTON_IMAGE):$(PROTON_VERSION)
OUTPUT_DIR := output OUTPUT_DIR := output
ROOTFS_SIZE ?= 5G ROOTFS_SIZE ?= 3G
FORCE_REBUILD ?= FORCE_REBUILD ?=
build: build:
DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \
-f Dockerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT) --build-arg PROTON_IMAGE=$(PROTON_REF) \
-f Containerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT)
build-debug: build-debug:
DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \
-f Dockerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT) --build-arg PROTON_IMAGE=$(PROTON_REF) \
-f Containerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT)
# Hours, and only when PROTON_TAG moves. Its context is this directory rather
# than the repository root: the two scripts beside the Containerfile are the
# whole input, and the root would hand it everything else for nothing.
proton-image:
DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \
--build-arg PROTON_TAG=$(PROTON_TAG) \
-f Containerfile.proton -t $(PROTON_REF) .
@echo "Built $(PROTON_REF)"
# Publishing is what makes `make build` cheap for everyone else, since that
# build pulls this by name. Push before expecting anyone to use a new tag.
proton-push: proton-image
$(CONTAINER_RT) push $(PROTON_REF)
# Not `sudo make image`/sudo'd in here: mkimage.sh runs as you and escalates # Not `sudo make image`/sudo'd in here: mkimage.sh runs as you and escalates
# only the specific commands that need root. Under rootless Podman, `make # only the specific commands that need root. Under rootless Podman, `make
@@ -62,5 +92,8 @@ help:
@echo " make image Build + pack runtime_prod into output/rootfs.ext4" @echo " make image Build + pack runtime_prod into output/rootfs.ext4"
@echo " make image-debug Build + pack runtime_debug into output/rootfs-debug.ext4" @echo " make image-debug Build + pack runtime_debug into output/rootfs-debug.ext4"
@echo " make clean Remove output/" @echo " make clean Remove output/"
@echo " make proton-image Build Proton from source (hours)"
@echo " make proton-push Build it and publish it"
@echo " make FORCE_REBUILD=1 ... Rebuild from scratch, no layer cache" @echo " make FORCE_REBUILD=1 ... Rebuild from scratch, no layer cache"
@echo " make PROTON_TAG=... ... Use a different proton-cachyos tag"
@echo " make ROOTFS_SIZE=8G image Override the packed image size (default 5G)" @echo " make ROOTFS_SIZE=8G image Override the packed image size (default 5G)"
build/README.md
+192 -38
View File
@@ -1,16 +1,17 @@
# build/ — the guest rootfs # build/ — the guest rootfs
Builds a bootable Artix/OpenRC image for the box's virtio-blk root: Mesa Builds a bootable Arch image for the box's virtio-blk root: Mesa (virtio-gpu
(virtio-gpu native context) plus the four open guest components — native context) plus the five open guest components —
[`nescope`](../apps/nescope), [`neshub`](../apps/neshub), [`nesinit`](../apps/nesinit), [`nescope`](../apps/nescope),
[`neswire`](../apps/neswire), [`nescapture`](../apps/nescapture) — laid out [`neshub`](../apps/neshub), [`neswire`](../apps/neswire),
[`nescapture`](../apps/nescapture) — laid out
the way [borealis](https://chromium.googlesource.com/chromiumos/overlays/board-overlays/+/main/project-borealis) the way [borealis](https://chromium.googlesource.com/chromiumos/overlays/board-overlays/+/main/project-borealis)
lays out its `build/`: one big multi-stage `Dockerfile`, `--target` picks the lays out its `build/`: one big multi-stage `Containerfile`, `--target` picks the
flavor, `etc/` holds the files that get overlaid onto the image verbatim. flavor, `etc/` holds the files that get overlaid onto the image verbatim.
``` ```
build/ build/
├── Dockerfile everything, in stages: mesa-build, nestri-build, ├── Containerfile everything, in stages: mesa-build, nestri-build,
│ os-base, runtime, runtime_prod, runtime_debug │ os-base, runtime, runtime_prod, runtime_debug
├── etc/ overlaid onto the image's /etc as-is ├── etc/ overlaid onto the image's /etc as-is
├── scripts/ ├── scripts/
@@ -24,6 +25,9 @@ make build # docker build --target runtime_prod → ghcr.io/nestrilab
make build-debug # docker build --target runtime_debug → ghcr.io/nestrilabs/nestri/base:debug make build-debug # docker build --target runtime_debug → ghcr.io/nestrilabs/nestri/base:debug
make image # + pack into output/rootfs.ext4 make image # + pack into output/rootfs.ext4
make image-debug # + pack into output/rootfs-debug.ext4 make image-debug # + pack into output/rootfs-debug.ext4
make proton-image # build Proton from source — hours
make proton-push # build it and publish it
``` ```
## Design notes ## Design notes
@@ -33,10 +37,10 @@ Three things worth knowing about how this is put together:
1. **No privileged host chroot.** A bare `chroot` into a hand-extracted 1. **No privileged host chroot.** A bare `chroot` into a hand-extracted
rootfs needs `/proc`, `/sys`, `/dev` bind-mounted in first — they don't rootfs needs `/proc`, `/sys`, `/dev` bind-mounted in first — they don't
exist inside a chroot target until something puts them there. `os-base` exist inside a chroot target until something puts them there. `os-base`
here is `FROM artixlinux/artixlinux:base-openrc` directly, with here is `FROM archlinux:base` directly, with `pacman -S` as plain `RUN`
`pacman -S` as plain `RUN` steps — a Docker build step already runs steps — a Docker build step already runs inside a real container with its
inside a real container with its own `/proc`, `/sys`, `/dev`, so that own `/proc`, `/sys`, `/dev`, so that whole bind-mount mechanism has nothing
whole bind-mount mechanism has nothing to do. to do.
2. **No host-side ownership bug to guard against.** `COPY --from=` runs as 2. **No host-side ownership bug to guard against.** `COPY --from=` runs as
root inside the build with no host user in the loop, so there's no root inside the build with no host user in the loop, so there's no
@@ -50,44 +54,194 @@ Three things worth knowing about how this is put together:
own incremental compiler per-crate isolation without needing a separate own incremental compiler per-crate isolation without needing a separate
Docker stage (and a separate full rebuild of `nesprotocol`) per binary. Docker stage (and a separate full rebuild of `nesprotocol`) per binary.
**What is deliberately not here: Proton, and Valve's `steamclient.so`.** **What is deliberately not here: Valve's `steamclient.so`.**
`nestri/CLAUDE.md` is explicit — *"Nothing closed may enter this repo. Not `nestri/CLAUDE.md` is explicit — *"Nothing closed may enter this repo. Not
source, not a dependency, not a directory that 'looked convenient'."* Both source, not a dependency, not a directory that 'looked convenient'."* That one
are closed. `runtime_prod` from this Dockerfile — tagged is closed, so whatever layers it on top is a build outside this repo — not
`ghcr.io/nestrilabs/nestri/base:latest` — is a complete, bootable, Steam-less guest image, something this repo names, links to, or depends on.
**Proton is here, and this paragraph used to say it was not.** The old wording
put Proton and `steamclient.so` together and called both closed, which is
wrong about Proton: it is compiled from source, which is not a thing you can
do with closed software. Keeping it out cost a box the only way it has to run
a Windows title, for a rule that did not apply to it.
What it is: **proton-cachyos built with `--enable-wow64`**, pulled by tag as a
published image rather than rebuilt here, because it takes hours and moves
only when its own tag does. `PROTON_IMAGE` overrides the tag, and it has to be
declared before the first `FROM` — an `ARG` a `FROM` expands is global or it
is nothing, and getting that wrong fails with `no FROM statement found`, which
says nothing about the actual mistake. wow64 is the whole reason it is a build of ours
and not the distribution's package — it runs 32-bit Windows code inside a
64-bit unix process, so a box needs no lib32 glibc, no second Mesa for i686,
and no second capture layer for 32-bit titles to be captured. The
distribution's package is built without the flag, which is exactly why it
depends on `lib32-*`.
It costs about 1.4 GB of image, and it is the one thing in here that is
payload-shaped: a compatibility layer for Windows games in an image that is
otherwise indifferent to what it runs. The guest components stay indifferent
regardless — none of them branches on it, and the init does not know it
exists. What names it is the command a caller sends.
`runtime_prod` from this Containerfile — tagged
`ghcr.io/nestrilabs/nestri/base:latest` — is a complete, bootable guest image,
and also the shared foundation other builds start from: nesbox's jail image and also the shared foundation other builds start from: nesbox's jail image
(see `nesbox/build/`) extracts **Mesa** from it so the guest and host sides of (see `nesbox/build/`) extracts **Mesa** from it so the guest and host sides of
the virtio-gpu native-context protocol never drift apart. Only Mesa — the virtio-gpu native-context protocol never drift apart. Only Mesa —
`virglrenderer` is the host half of that protocol and nesbox builds its own, `virglrenderer` is the host half of that protocol and nesbox builds its own,
patched, from `nesbox/patches/`; nothing in this image carries it. patched, from `nesbox/patches/`; nothing in this image carries it.
Whatever layers Proton and the Steam client on top of it is a closed build ## Two packages that look droppable and are not
outside this repo, by design — not something this repo names, links to, or
depends on.
## The nesinit gap `llvm-libs` is 164 MB, the largest single thing in the image after Proton, and
`lm_sensors` is only there because something links `libsensors`. Both look like
leftovers of a Mesa configuration that has since been trimmed, and both have
been checked rather than reasoned about: **`libgbm` links them**, and the
compositor needs GBM. Trimming the Mesa build does not reach them.
Nothing in this image starts a payload. The old `nestri-guest-hub` did that `lm_sensors` in particular was found the hard way. It used to arrive as a
— per its own commit message, the open `neshub` *"loses `--proton`, dependency of the distribution's Mesa package, and dropping that package took
`--steamclient-so`, `--root` and the game uid/gid, and no longer ends by it away — leaving our own Mesa unable to resolve `libsensors.so.5`. Nothing in
handing the process to a controller... Deciding when the box is finished a package list says that; the check below is what said it.
belongs to `nesinit`."* `nesinit` — the guest init/session-supervisor that
would actually launch `nescope -- <payload>` and power the box down — is
referenced in commit messages and `nesbox/PROGRESS.md` but does not exist as
open code in either repo.
So `/etc/init.d/nescope` here starts nescope in **plain-compositor mode** ## Proton has its own cadence, and its own Containerfile
(no command after `--`): it comes up, provides the Wayland/X11 environment,
and waits for something to connect. That makes the image genuinely bootable `make build` **pulls** Proton by tag; it does not build it. Building it takes
and testable — `neshub`, `neswire`, `nescope` all come up under OpenRC and hours and it changes only when its tag moves, so it is one image published
you get a real Wayland socket to point a client at — but running an actual once and copied into every guest image after that. `Containerfile.proton` is
game is still nesinit's job, and nesinit isn't part of this build. Whoever that build, and it lives here so the published tag stays reproducible from
picks that up next should read `apps/neshub/README.md`'s "What it does not this tree rather than from somebody's laptop.
do" section first.
```sh
make proton-image # the current tag
make PROTON_TAG=cachyos-11.1-20261115-native proton-image
```
**`PROTON_TAG` is the only thing to change.** The published version is derived
from it in the `Makefile` rather than written a second time, because the two
are the same number in two spellings — and an image whose name does not say
which Proton is inside it is worse than no image. The `Containerfile`'s own
`PROTON_IMAGE` default is a fallback for a bare container build; going through
`make` is what keeps them in step.
Its **context is `build/`**, not the repository root the guest build uses. All
it needs is the two scripts beside it, and `Containerfile.proton.containerignore`
keeps `output/` out of that context — a build context is copied before the
first instruction runs, so without it every Proton build would begin by moving
the last rootfs image it produced.
Two things in the recipe are worth knowing before changing it:
- **Fetch and build are separate layers on purpose.** The submodule checkout
runs well past ten minutes, and a build that fails on a flag or a missing
tool must not pay for that again. Keep anything that can fail *fast* in
`proton-build.sh`.
- **`widl` is built by hand from the mingw-w64 release.** Without it autoconf
quietly sets `HAVE_WIDL` to false, vkd3d's public headers are never
generated, and the build dies an hour later on a missing header. Arch ships
`widl` only inside `wine`, which wants multilib — which is the thing
`--enable-wow64` exists to avoid.
## There is no init system in here, and that is the design
`nesinit` is PID 1. The image carries **no service manager, no init scripts,
no `udev` and no systemd** — `systemd-libs` stays, because `dbus-daemon` and
`wireplumber` link `libsystemd.so.0`, but nothing in the image can be PID 1
except `nesinit`, and the build fails if anything that could be turns up.
That is why this is plain Arch. The image used to be Artix, chosen for OpenRC,
and every cost of that choice — no `eudev`, no `agetty-openrc`, `udev` being
systemd's anyway, a runlevel edit not stopping a service another one still
needs — was paid for an init system that is no longer here.
**What replaced fourteen `rc-update` lines and nine init scripts:**
| was | now |
|---|---|
| `devfs`, `dmesg`, `udev`, `udev-trigger` | `devtmpfs` makes the nodes; init sets the two modes that matter. The compositor takes input through Wayland and opens nothing `udev` provides |
| `guest-net`, `hostname`, `xdg-runtime`, `cgroups` | init, before it dials out |
| `dbus`, `dbus-session`, `pipewire`, `wireplumber`, `neshub`, `neswire` | a table compiled into `nesinit` |
| `nescope` in the `default` runlevel | **not a service.** It wraps the workload and is started by a launch, with that launch's geometry, and dies with it |
| `agetty` on `hvc0` | nothing. See below |
| `/etc/fstab` | init's own mounts, and shares named in the boot descriptor |
**A box is launched into, not booted into something.** Init mounts what the
descriptor names, brings the table up, says it is ready, and then takes
commands — so an image on its own runs nothing at all, which is the point:
this image is payload-independent and there is no payload in it.
### Getting into a guest that will not boot
`init=/bin/bash` on the kernel command line. Nothing in the image offers a
login prompt — there is no getty in either flavour — and that is cheaper than
carrying one: `nesinit` is an ordinary program, so from that shell you can run
it by hand and watch it fail. `make build-debug` adds `vulkaninfo` and friends
and gives root a password for `su`; it does not add a console.
**A shell is not a booted box, and the difference bites immediately.** Nothing
the init does has happened: the root is read-only, `/run` and `/tmp` are still
directories on it rather than tmpfs, `/run/user/1000` is unwritable, and the
hostname is `(none)` rather than `nesbox` — which is the quickest way to tell
the two states apart. A compositor started in that shell fails on its own
socket, and the error names the runtime directory rather than the cause.
So run `nesinit` first. It mounts, prepares the directories, brings the
services up, then fails to reach a control channel that is not there and
exits — **leaving everything it prepared behind**, which is exactly what makes
the hand-run useful. Then start what you came to debug.
It is safe to run outside a box, and that took fixing: the shutdown path signals
every process it may signal and then powers the machine off, which is right for
PID 1 of a box and catastrophic anywhere else. Both steps are refused when it is
not PID 1, and it says so rather than doing it quietly.
If you would rather not run it at all, the two mounts it does that a compositor
needs are:
```sh
mount -t tmpfs -o mode=1777,size=64m tmpfs /tmp
mount -t tmpfs -o mode=755,size=32m tmpfs /run
mkdir -p /run/user/1000 && chown 1000:1000 /run/user/1000 && chmod 0700 /run/user/1000
```
Making `/run/user/1000` writable in the *image* does not help, and is worth
saying because it is the obvious first thing to try: before the init runs, the
directory is on a read-only root, so its ownership is not what stops a write;
after the init runs, a fresh tmpfs is mounted over `/run` and the image's copy
of the directory is hidden underneath it.
The one thing this does not reach is a failure *before* the shell. If that
happens the evidence is on `console=hvc0` and nowhere else.
### Two build-time checks worth knowing about
Both exist because the failure they catch is invisible at runtime rather than
loud, which is the same reason the old build checked its `conf.d` files:
- **No hook may point at a program that is not in the image.** Removing
systemd removes the script `dbus-reload.hook` calls, and a leftover hook
produces `error: command failed to execute correctly` on every future pacman
transaction — indistinguishable, in a log, from something that matters.
- **Nothing `nesinit` will look for may be missing.** Its service table is
compiled in, so an absent `dbus-daemon` is not a build error by itself; it
is a service that does not come up in a box somebody is waiting on.
## Network defaults ## Network defaults
`/etc/init.d/guest-net` reads `nestri.ip=`/`nestri.gw=` off the kernel `nesinit` reads `nestri.ip=`/`nestri.gw=` off the kernel command line,
command line, falling back to `172.30.0.2/24` via `172.30.0.1` if neither is falling back to `172.30.0.2/24` via `172.30.0.1` if neither is set — the
set — nesbox's own default tap addressing. Keep these in step if that host's own default tap addressing. Keep these in step if that changes on the
changes on the nesbox side. host side. A box started with no network device at all is a valid box and
boots without one.
The address is a per-boot parameter rather than an image setting because the
alternative makes every box built from this image the same host on the
network, and two of them collide the moment they run together. Same reasoning
for `/etc/machine-id`, which is a symlink into a tmpfs that init fills at
boot — the previous image baked one in, so every box built from it was the
same machine to anything that asked.
This is also the one thing in the image that keeps `iproute2` installed:
init runs `ip` rather than talking netlink, which is a hundred lines of
`unsafe` saved for an interface configured once.
build/etc/conf.d/agetty.hvc0.debug
-3
View File
@@ -1,3 +0,0 @@
baud="115200"
term_type="vt100"
agetty_options="--autologin root --noclear"
build/etc/conf.d/agetty.hvc0.prod
-3
View File
@@ -1,3 +0,0 @@
baud="115200"
term_type="vt100"
agetty_options="--noclear"
build/etc/conf.d/nescope
-16
View File
@@ -1,16 +0,0 @@
# Configuration for nescope, the headless compositor.
#
# Sources nestri-user-env rather than symlinking it, for the same reason
# neshub and neswire do.
. /etc/conf.d/nestri-user-env
RUST_LOG="${RUST_LOG:-nescope=info}"
export RUST_LOG
# Started in plain-compositor mode by /etc/init.d/nescope (no command after
# `--`): it comes up and waits for something to connect rather than wrapping
# a payload. Starting an actual payload — a game, a desktop — is nesinit's
# job, and nesinit is not open code yet. See build/README.md.
NESCOPE_SOCKET="${NESCOPE_SOCKET:-nescope-0}"
NESCOPE_INPUT_IPC="${NESCOPE_INPUT_IPC:-/tmp/nestri-input.sock}"
export NESCOPE_SOCKET NESCOPE_INPUT_IPC
build/etc/conf.d/neshub
-18
View File
@@ -1,18 +0,0 @@
# Configuration for neshub, the media hub.
#
# Not a symlink to nestri-user-env like pipewire and friends: neshub wants a
# setting of its own, so it sources that file rather than replacing it — same
# pattern as nescope and neswire below.
. /etc/conf.d/nestri-user-env
# What neshub logs, and nothing else.
#
# Scoped and exported for the reason spelled out at length in the old
# nestri-guest-hub conf.d this replaces: an unscoped `RUST_LOG=info` turns on
# every crate linked in, iroh included, and OpenRC hands a service its own
# environment, not the shell's locals — an unexported variable here goes
# nowhere.
#
# Overridable, so a bad boot can be re-run with more without a rebuild.
RUST_LOG="${RUST_LOG:-neshub=info}"
export RUST_LOG
build/etc/conf.d/nestri-user-env
-34
View File
@@ -1,34 +0,0 @@
HOME="/home/nestri"
USER="nestri"
LOGNAME="nestri"
XDG_RUNTIME_DIR="/run/user/1000"
XDG_CONFIG_HOME="/home/nestri/.config"
XDG_DATA_HOME="/home/nestri/.local/share"
XDG_CACHE_HOME="/home/nestri/.cache"
XDG_STATE_HOME="/home/nestri/.local/state"
XDG_DATA_DIRS="/usr/local/share:/usr/share"
XDG_SESSION_TYPE="wayland"
XDG_SESSION_DESKTOP="nestri"
DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/1000/bus"
# Driver forcing — currently off, in both copies. Kept, commented, because
# the reasoning still applies if it is ever needed again: capture happens
# through a Vulkan layer, so a process reaching the GPU via native OpenGL
# would render fine and capture nothing, and zink is what makes such a
# process capturable at all. Nothing has needed it so far.
#
# /etc/profile.d/nestri-env.sh carries the same block, also commented, and
# only runs for login shells — nothing here is one, every service below runs
# from OpenRC. If these are ever re-enabled, re-enable both: one copy on and
# one off means services and shells reach the GPU by different paths.
#__GLX_VENDOR_LIBRARY_NAME="mesa"
#MESA_LOADER_DRIVER_OVERRIDE="zink"
#GALLIUM_DRIVER="zink"
nestri_export_env() {
export HOME USER LOGNAME
export XDG_RUNTIME_DIR XDG_CONFIG_HOME XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME
export XDG_DATA_DIRS XDG_SESSION_TYPE XDG_SESSION_DESKTOP
export DBUS_SESSION_BUS_ADDRESS
#export __GLX_VENDOR_LIBRARY_NAME MESA_LOADER_DRIVER_OVERRIDE GALLIUM_DRIVER
}
build/etc/conf.d/neswire
-32
View File
@@ -1,32 +0,0 @@
# Configuration for the audio wire.
#
# Not a symlink to nestri-user-env like pipewire and friends, for the same
# reason neshub is not: it needs a setting of its own -- and it still wants
# that file's environment, so it sources it rather than replacing it.
. /etc/conf.d/nestri-user-env
# What neswire logs, and nothing else.
#
# Without this neswire is *silent* -- it starts, fails, and exits leaving an
# empty log, which is indistinguishable from never having been started at
# all. That ambiguity is the reason this file exists.
#
# Scoped, and `export`ed, for the reasons spelled out at length in
# conf.d/neshub: an unscoped `info` turns on every linked crate, and an
# unexported one reaches the service's shell and not the service.
#
# Overridable, so a bad boot can be re-run with more without a rebuild.
RUST_LOG="${RUST_LOG:-neswire=info}"
export RUST_LOG
# neswire's own defaults, spelled out rather than inherited.
#
# All four are clap args with defaults (`neswire --help`), so the service
# works without them. They are here because the IPC path is a contract with
# neshub -- it reads this socket -- and a contract that lives only inside two
# binaries' default values is one nobody can check.
NESWIRE_IPC_PATH="/tmp/nestri-audio.sock"
NESWIRE_CHANNELS="2"
NESWIRE_PACKET_DURATION_MS="5"
NESWIRE_BITRATE_PER_CHANNEL="64"
export NESWIRE_IPC_PATH NESWIRE_CHANNELS NESWIRE_PACKET_DURATION_MS NESWIRE_BITRATE_PER_CHANNEL
build/etc/fstab
-35
View File
@@ -1,35 +0,0 @@
# `ro`, matching how a box is actually started: nesbox passes `ro` on the
# kernel command line and marks the root device `is_read_only: true` (see
# nesbox's examples/vm.json), so the virtio-blk device refuses writes at the
# device level. Saying `rw` here does not make it writable — it only asks
# OpenRC's `root` service to attempt a remount that the device must reject.
# Everything a running box writes to is a tmpfs or a share below.
/dev/vda / ext4 ro,relatime 0 1
devtmpfs /dev devtmpfs rw,nosuid 0 0
proc /proc proc rw,nosuid,nodev,noexec 0 0
sysfs /sys sysfs rw,nosuid,nodev,noexec 0 0
tmpfs /tmp tmpfs rw,nosuid,nodev,size=64M 0 0
tmpfs /run tmpfs rw,nosuid,nodev,size=32M,mode=0755 0 0
tmpfs /var/log tmpfs rw,nosuid,nodev,size=16M 0 0
# POSIX shared memory. devtmpfs does not provide it and nothing else mounts it,
# so without this entry /dev/shm does not exist at all and `shm_open` fails.
#
# PipeWire itself gets by: it allocates buffers with memfd_create and only
# falls back to /dev/shm. Kept as a precaution for anything else that uses
# POSIX shm/semaphores directly — Proton is the known example, and it is not
# part of this image yet (see build/README.md), but a failure here is silent
# rather than fatal, so it costs nothing to have ready.
#
# `nosuid,nodev` and a size cap because everything a game can write to should
# have both.
tmpfs /dev/shm tmpfs rw,nosuid,nodev,size=256M 0 0
# The guest's logs, on a host directory that outlives the VM.
#
# Here rather than mounted by a service, because the failures worth reading
# are the ones that happen *before* anything else has mounted something — a
# log directory that appears only after a successful start cannot record an
# unsuccessful one. The tag is fixed by nessh, so plain fstab works.
#
# `nofail` because a VM started without the share still has to boot: that is
# how somebody gets a shell to find out why it has no share.
logs /nestri/logs virtiofs rw,nofail 0 0
build/etc/init.d/dbus-session
-33
View File
@@ -1,33 +0,0 @@
#!/sbin/openrc-run
description="D-Bus session bus for nestri user"
nestri_export_env
command="/usr/bin/dbus-daemon"
command_args="--session --address=unix:path=/run/user/1000/bus --nofork --nopidfile --print-address"
command_user="nestri:nestri"
command_background="yes"
pidfile="/run/nestri/dbus-session.pid"
depend() {
need xdg-runtime dbus
before pipewire wireplumber
}
start_pre() {
checkpath -d -m 0755 -o "nestri:nestri" /run/nestri
if [ -e "/run/user/1000/bus" ]; then
ewarn "Removing stale bus socket"
rm -f "/run/user/1000/bus"
fi
}
start_post() {
local i=0
while [ ! -S /run/user/1000/bus ] && [ $i -lt 20 ]; do
sleep 0.1
i=$((i+1))
done
[ -S /run/user/1000/bus ] || { eerror "Bus didn't appear"; return 1; }
}
build/etc/init.d/dbus-system
-19
View File
@@ -1,19 +0,0 @@
#!/sbin/openrc-run
description="D-Bus system message bus"
command="/usr/bin/dbus-daemon"
command_args="--system --nofork --nopidfile"
command_background="yes"
pidfile="/run/dbus/dbus.pid"
depend() {
need localmount xdg-runtime
before pipewire
}
start_pre() {
checkpath -d -m 0755 -o root:root /run/dbus
checkpath -d -m 0755 -o messagebus:messagebus /var/run/dbus 2>/dev/null || true
dbus-uuidgen --ensure=/var/lib/dbus/machine-id
}
build/etc/init.d/guest-net
-92
View File
@@ -1,92 +0,0 @@
#!/sbin/openrc-run
description="Configure the guest's network to match the host's tap"
# Overridden from /etc/conf.d/guest-net if present. These defaults match
# nesbox's own defaults; if you change the `network` section in the VM's
# JSON, change these to match.
: ${GUEST_IFACE:=eth0}
: ${GUEST_IP:=172.30.0.2}
: ${GUEST_PREFIX:=24}
: ${GUEST_GATEWAY:=172.30.0.1}
depend() {
need localmount
provide net
keyword -shutdown
}
# Read one `nestri.<key>=<value>` from the kernel command line.
#
# The address has to come from somewhere per-boot, because the alternative --
# baking it into the image -- makes every guest built from that image the
# same host on the network. Two sandboxes then collide the moment they run
# together.
#
# `nestri.`-prefixed rather than the kernel's own `ip=`: that one needs
# CONFIG_IP_PNP and exists to configure NFS root, and the prefix makes it
# obvious whose parameter this is.
cmdline_value() {
local key="$1" word
for word in $(cat /proc/cmdline 2>/dev/null); do
case "$word" in
"nestri.${key}="*) printf '%s' "${word#nestri.${key}=}"; return 0 ;;
esac
done
return 1
}
start() {
ebegin "Bringing up loopback"
ip link set lo up
eend $?
# The command line wins over conf.d when it says anything, and conf.d is
# the fallback so a hand-written VM config with no parameters keeps
# working -- which is how a guest gets debugged.
local source="/etc/conf.d/guest-net"
local cmdline_ip
if cmdline_ip="$(cmdline_value ip)"; then
# Accepts address/prefix; a bare address keeps the configured prefix
# rather than guessing one.
case "$cmdline_ip" in
*/*)
GUEST_IP="${cmdline_ip%%/*}"
GUEST_PREFIX="${cmdline_ip##*/}"
;;
*) GUEST_IP="$cmdline_ip" ;;
esac
source="kernel command line"
fi
local cmdline_gw
if cmdline_gw="$(cmdline_value gw)"; then
GUEST_GATEWAY="$cmdline_gw"
source="kernel command line"
fi
# The VM may have been started with no network device at all, which is a
# perfectly good configuration. Do not fail the boot over it.
if [ ! -e "/sys/class/net/${GUEST_IFACE}" ]; then
einfo "no ${GUEST_IFACE}: this VM has no network device"
return 0
fi
# Says which source won, because "the address is wrong" and "the address
# came from somewhere unexpected" look identical from inside the guest.
ebegin "Configuring ${GUEST_IFACE} as ${GUEST_IP}/${GUEST_PREFIX} via ${GUEST_GATEWAY} (from ${source})"
# `replace` rather than `add` so a restart is not an error.
ip link set "${GUEST_IFACE}" up &&
ip addr replace "${GUEST_IP}/${GUEST_PREFIX}" dev "${GUEST_IFACE}" &&
ip route replace default via "${GUEST_GATEWAY}" dev "${GUEST_IFACE}"
eend $? "could not configure ${GUEST_IFACE}"
}
stop() {
if [ -e "/sys/class/net/${GUEST_IFACE}" ]; then
ebegin "Bringing down ${GUEST_IFACE}"
ip link set "${GUEST_IFACE}" down
eend 0
fi
return 0
}
build/etc/init.d/nescope
-33
View File
@@ -1,33 +0,0 @@
#!/sbin/openrc-run
description="Nestri headless compositor"
nestri_export_env
: "${NESTRI_UID:=1000}"
: "${NESTRI_USER:=nestri}"
command="/usr/bin/nescope"
# No command after `--`: plain-compositor mode. nescope comes up and waits
# for something to connect rather than wrapping a payload — starting one is
# nesinit's job, and nesinit is not open code yet. See build/README.md.
command_user="${NESTRI_USER}:${NESTRI_USER}"
command_background="yes"
pidfile="/run/nestri/nescope.pid"
output_log="/nestri/logs/nescope.log"
error_log="/nestri/logs/nescope.log"
respawn="yes"
respawn_delay="2"
respawn_max="2"
export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}"
depend() {
need xdg-runtime
use neshub
after xdg-runtime
}
start_pre() {
checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri
}
build/etc/init.d/neshub
-39
View File
@@ -1,39 +0,0 @@
#!/sbin/openrc-run
description="Nestri media hub"
nestri_export_env
: "${NESTRI_UID:=1000}"
: "${NESTRI_USER:=nestri}"
command="/usr/bin/neshub"
# Unprivileged. The old nestri-guest-hub ran as root and mounted the
# session's filesystem itself; the open neshub does neither — per its own
# README it only muxes the Unix sockets the other components dial into one
# QUIC endpoint. Whatever ends up owning session mounts (nesinit, presumably)
# is a separate, still-closed piece.
command_user="${NESTRI_USER}:${NESTRI_USER}"
command_background="yes"
pidfile="/run/nestri/neshub.pid"
# Where neshub's output goes. /nestri/logs is a virtiofs share mounted from
# /etc/fstab at boot, before this service starts, so a hub that fails
# immediately still leaves a record, and the record survives the VM.
output_log="/nestri/logs/neshub.log"
error_log="/nestri/logs/neshub.log"
respawn="yes"
respawn_delay="2"
respawn_max="2" # NO infinite — if it fails, it fails for a reason
export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}"
depend() {
need xdg-runtime
use net
after xdg-runtime
}
start_pre() {
checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri
}
build/etc/init.d/neswire
-93
View File
@@ -1,93 +0,0 @@
#!/sbin/openrc-run
description="Nestri pipewire audio sink"
nestri_export_env
: "${NESTRI_UID:=1000}"
: "${NESTRI_USER:=nestri}"
command="/usr/bin/neswire"
command_user="${NESTRI_USER}:${NESTRI_USER}"
command_background="yes"
pidfile="/run/nestri/neswire.pid"
output_log="/nestri/logs/neswire.log"
error_log="/nestri/logs/neswire.log"
respawn="yes"
respawn_delay="2"
respawn_max="2"
export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}"
export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${NESTRI_UID}/bus"
depend() {
# wireplumber is a hard dependency, not a nicety: start_post below pins
# the default sink through the `default` metadata object, and
# WirePlumber is what owns that object. Started concurrently, the pin
# lands in an object pw-metadata created itself, which is destroyed the
# moment it exits.
need xdg-runtime dbus dbus-session pipewire wireplumber
use neshub
after pipewire wireplumber neshub
}
start_pre() {
checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/nestri
}
# Point the graph at neswire's sink, by name.
#
# `neswire` registers itself as media.class = Audio/Sink, node.name = neswire
# ("Neswire Cloud Gaming Audio Sink"). With the hardware monitors off it is
# the only sink, so WirePlumber's find-best hook should land on it anyway --
# this makes it explicit rather than a consequence of there being nothing
# else, so adding a second sink later cannot silently steal the default.
#
# `default.configured.audio.sink` is the metadata WirePlumber's find-selected
# hook reads, and it takes a name; `wpctl set-default` takes a numeric object
# id that changes every boot, which is why this uses pw-metadata instead.
#
# Not fatal if it fails: find-best still has one candidate. A game playing
# into the wrong sink is silent, and a warning here is the only place that
# would say so.
start_post() {
local i=0
while [ $i -lt 50 ]; do
pw-dump 2>/dev/null | grep -q '"node.name": "neswire"' && break
sleep 0.1
i=$((i+1))
done
if ! pw-dump 2>/dev/null | grep -q '"node.name": "neswire"'; then
ewarn "neswire started but its sink never appeared in the graph"
return 0
fi
# Wait for WirePlumber to own the `default` metadata before writing to it.
#
# `need wireplumber` only guarantees its script returned, not that it has
# built its objects. Writing too early is silently useless rather than an
# error: pw-metadata creates the object, sets the key, exits 0, and the
# object dies with the client. So wait for the object to exist, and say
# so if it never does.
i=0
while [ $i -lt 50 ]; do
pw-metadata -n default >/dev/null 2>&1 && break
sleep 0.1
i=$((i+1))
done
if ! pw-metadata -n default >/dev/null 2>&1; then
ewarn "no 'default' metadata: WirePlumber is not running, sink not pinned"
return 0
fi
pw-metadata -n default 0 default.configured.audio.sink '{ "name": "neswire" }' \
>/dev/null 2>&1
# Read it back. A write that did not stick is the failure this whole
# sequence exists to catch, and it is invisible unless checked.
if pw-metadata -n default 2>/dev/null | grep -q "neswire"; then
einfo "default sink pinned to neswire"
else
ewarn "pinned neswire as default sink but it did not stick"
fi
}
build/etc/init.d/pipewire
-45
View File
@@ -1,45 +0,0 @@
#!/sbin/openrc-run
description="PipeWire multimedia daemon (system mode for nestri)"
nestri_export_env
: "${NESTRI_UID:=1000}"
: "${NESTRI_USER:=nestri}"
command="/usr/bin/pipewire"
command_user="${NESTRI_USER}:${NESTRI_USER}"
command_background="yes"
pidfile="/run/pipewire/pipewire.pid"
export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}"
export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${NESTRI_UID}/bus"
depend() {
need xdg-runtime dbus
use dbus
}
start_pre() {
checkpath -d -m 0755 -o "${NESTRI_USER}:${NESTRI_USER}" /run/pipewire
}
# `need pipewire` only waits for this script to return, and with
# command_background that is the moment start-stop-daemon forks -- not the
# moment PipeWire is accepting connections. With rc_parallel="YES" every
# client (wireplumber, neswire) then races the socket and a client that
# loses simply exits.
#
# dbus-session already solved exactly this for the bus socket. Same shape
# here.
start_post() {
local i=0
while [ ! -S "/run/user/${NESTRI_UID}/pipewire-0" ] && [ $i -lt 50 ]; do
sleep 0.1
i=$((i+1))
done
[ -S "/run/user/${NESTRI_UID}/pipewire-0" ] || {
eerror "PipeWire socket never appeared"
return 1
}
}
build/etc/init.d/wireplumber
-20
View File
@@ -1,20 +0,0 @@
#!/sbin/openrc-run
description="WirePlumber session manager for PipeWire"
nestri_export_env
: "${NESTRI_UID:=1000}"
: "${NESTRI_USER:=nestri}"
command="/usr/bin/wireplumber"
command_user="${NESTRI_USER}:${NESTRI_USER}"
command_background="yes"
pidfile="/run/pipewire/wireplumber.pid"
export XDG_RUNTIME_DIR="/run/user/${NESTRI_UID}"
depend() {
need pipewire
after pipewire
}
build/etc/init.d/xdg-runtime
-44
View File
@@ -1,44 +0,0 @@
#!/sbin/openrc-run
description="Create runtime directories (XDG_RUNTIME_DIR, X11 socket dir..)"
NESTRI_UID="${NESTRI_UID:-1000}"
NESTRI_USER="${NESTRI_USER:-nestri}"
depend() {
need localmount
before dbus pipewire
}
start() {
ebegin "Preparing runtime directories"
if ! getent passwd "${NESTRI_USER}" >/dev/null 2>&1; then
eerror "User ${NESTRI_USER} does not exist"
return 1
fi
# Per-user XDG runtime dir
mkdir -p "/run/user/${NESTRI_UID}"
chown "${NESTRI_USER}:${NESTRI_USER}" "/run/user/${NESTRI_UID}"
chmod 0700 "/run/user/${NESTRI_UID}"
# X11 socket dir (Xwayland + any X clients expect this)
mkdir -p /tmp/.X11-unix
chown root:root /tmp/.X11-unix
chmod 1777 /tmp/.X11-unix
# ICE socket dir - some toolkits look for it
mkdir -p /tmp/.ICE-unix
chown root:root /tmp/.ICE-unix
chmod 1777 /tmp/.ICE-unix
eend $?
}
stop() {
ebegin "Removing runtime directories"
rm -rf "/run/user/${NESTRI_UID}"
# Don't rm /tmp/.X11-unix on stop - other things may be using it
eend 0
}
build/etc/profile.d/nestri-env.sh
+15 -7
View File
@@ -14,14 +14,22 @@ if [ "$(id -u)" = "1000" ]; then
export XDG_SESSION_TYPE="${XDG_SESSION_TYPE:-wayland}" export XDG_SESSION_TYPE="${XDG_SESSION_TYPE:-wayland}"
export XDG_SESSION_DESKTOP="${XDG_SESSION_DESKTOP:-nestri}" export XDG_SESSION_DESKTOP="${XDG_SESSION_DESKTOP:-nestri}"
# Ensure proper VAAPI driver is used
#export LIBVA_DRIVER_NAME="radeonsi"
# Force zink usage for OpenGL -> Vulkan translation
#export __GLX_VENDOR_LIBRARY_NAME=mesa
#export MESA_LOADER_DRIVER_OVERRIDE=zink
#export GALLIUM_DRIVER=zink
# Ensure standard XDG dirs exist # Ensure standard XDG dirs exist
mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_CACHE_HOME}" "${XDG_STATE_HOME}" 2>/dev/null || true mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_CACHE_HOME}" "${XDG_STATE_HOME}" 2>/dev/null || true
fi fi
# OpenGL -> Vulkan, for anybody who reaches a shell in here.
#
# Outside the uid test above on purpose, and duplicated from the init on
# purpose. This file is only ever read by a person who got a shell in a box --
# nothing a box runs is started from a login, so every service and every
# workload is exec'd with a cleared environment and never sees this. The init
# sets the same three for what it starts.
#
# It is here so that a debug shell renders the way a session does. A shell that
# silently has no GL driver is how somebody concludes the image is broken while
# the image is fine.
export __GLX_VENDOR_LIBRARY_NAME=mesa
export MESA_LOADER_DRIVER_OVERRIDE=zink
export GALLIUM_DRIVER=zink
build/etc/rc.conf
-27
View File
@@ -1,27 +0,0 @@
# Nestri rootfs OpenRC config
rc_parallel="YES"
rc_depend_strict="NO"
rc_interactive="NO"
rc_shell_timeout="0"
# Every service's start/fail line, on the host directory that outlives the VM.
#
# Without this a guest nobody can log into reports nothing about its own boot:
# a service that was skipped because a dependency failed looks exactly like a
# service that was never registered. /nestri/logs is a virtiofs share, so the
# record survives the guest — see the fstab entry below.
rc_logger="YES"
rc_log_path="/nestri/logs/rc.log"
rc_verbose="NO"
# cgroup v2
rc_cgroup_mode="unified"
# Don't bother trying to set hostname twice
rc_hotplug="!net.*"
# Faster sulogin behavior on emergency
rc_shell="/bin/sh"
# Default umask
umask 022
build/scripts/mkimage.sh
+2 -2
View File
@@ -15,14 +15,14 @@ set -euo pipefail
IMAGE="${1:?usage: mkimage.sh <image-tag> <output-path> [size]}" IMAGE="${1:?usage: mkimage.sh <image-tag> <output-path> [size]}"
OUT="${2:?usage: mkimage.sh <image-tag> <output-path> [size]}" OUT="${2:?usage: mkimage.sh <image-tag> <output-path> [size]}"
SIZE="${3:-5G}" SIZE="${3:-3G}"
if [[ "$(id -u)" -eq 0 ]]; then if [[ "$(id -u)" -eq 0 ]]; then
echo "mkimage.sh should run as yourself, not root/sudo — see the comment at the top of this script" >&2 echo "mkimage.sh should run as yourself, not root/sudo — see the comment at the top of this script" >&2
exit 1 exit 1
fi fi
CONTAINER_RT="$(command -v docker || command -v podman || true)" CONTAINER_RT="$(command -v podman || command -v docker || true)"
[[ -n "$CONTAINER_RT" ]] || { echo "Neither docker nor podman found in PATH" >&2; exit 1; } [[ -n "$CONTAINER_RT" ]] || { echo "Neither docker nor podman found in PATH" >&2; exit 1; }
sudo -v # cache credentials once, rather than prompting mid-pipeline sudo -v # cache credentials once, rather than prompting mid-pipeline
build/scripts/proton-build.sh
Executable
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env bash
# Builds proton-cachyos from the tree proton-fetch.sh laid down. Container-only.
#
# The one thing that matters here is --enable-wow64: it builds wine so that
# 32-bit Windows code runs inside a 64-bit unix process, thunking down to the
# 64-bit host libraries. Without it, Proton needs a complete 32-bit host stack —
# lib32 glibc, a second Mesa built for i686, and a second nescapture layer,
# because a 32-bit game would load the 32-bit Vulkan loader and our 64-bit
# capture layer would be invisible to it. With it, none of that exists.
#
# The cost is that the distro package cannot be used: proton-cachyos-native is
# packaged without the flag, which is exactly why it depends on lib32-*.
set -euo pipefail
: "${GECKO_VER:?}"
: "${MONO_VER:?}"
JOBS="${JOBS:-$(nproc)}"
BUILD_NAME="proton-cachyos"
SRC_DIR="/build/proton-cachyos"
BUILD_DIR="/build/build"
OUT_DIR="/artifacts/proton/usr/share/steam/compatibilitytools.d/${BUILD_NAME}"
[[ -d "${SRC_DIR}" ]] || { echo "no source tree — proton-fetch.sh did not run"; exit 1; }
# ── Toolchain wrappers ──────────────────────────────────
# Proton's build calls the compiler by GNU triplet. Arch's gcc does not install
# under those names, so stand in for them. The i686 set is generated too: with
# wow64 nothing should reach for it, and if something does, failing on a missing
# 32-bit header beats silently building a 32-bit unix library we then have to
# ship libraries for.
WRAP=/build/wrappers
rm -rf "$WRAP" && mkdir -p "$WRAP"
_wrappers() {
local arch="$1" gccflag="$2" ldflag="$3" asflag="$4" stripfmt="$5"
local l t
for l in ar ranlib nm; do
ln -sf "/usr/bin/gcc-${l}" "${WRAP}/${arch}-pc-linux-gnu-${l}"
done
for t in gcc g++; do
printf '#!/usr/bin/bash\n/usr/bin/%s %s "$@"\n' "$t" "$gccflag" \
> "${WRAP}/${arch}-pc-linux-gnu-${t}"
chmod 755 "${WRAP}/${arch}-pc-linux-gnu-${t}"
done
printf '#!/usr/bin/bash\n/usr/bin/ld %s "$@"\n' "$ldflag" > "${WRAP}/${arch}-pc-linux-gnu-ld"
printf '#!/usr/bin/bash\n/usr/bin/as %s "$@"\n' "$asflag" > "${WRAP}/${arch}-pc-linux-gnu-as"
printf '#!/usr/bin/bash\n/usr/bin/strip -F %s "$@"\n' "$stripfmt" > "${WRAP}/${arch}-pc-linux-gnu-strip"
chmod 755 "${WRAP}/${arch}-pc-linux-gnu-"{ld,as,strip}
}
_wrappers x86_64 "-m64" "-melf_x86_64" "--64" "elf64-x86-64"
_wrappers i686 "-m32" "-melf_i386" "--32" "elf32-i386"
export PATH="${WRAP}:${PATH}"
# ── Build ───────────────────────────────────────────────
# -march=nocona matches the distro packaging: Proton has to run on whatever CPU
# the guest is given, and the VMM does not promise a feature level.
export CFLAGS="-O3 -march=nocona -mtune=core-avx2"
export CXXFLAGS="${CFLAGS}"
export RUSTFLAGS="-C opt-level=3 -C target-cpu=nocona"
export LDFLAGS="-Wl,-O1,--sort-common,--as-needed"
export RUSTUP_TOOLCHAIN=stable
mkdir -p "${BUILD_DIR}"
cd "${BUILD_DIR}"
ROOTLESS_CONTAINER="" \
"${SRC_DIR}/configure.sh" \
--container-engine="none" \
--proton-sdk-image="" \
--build-name="${BUILD_NAME}" \
--without-extras=all \
--without-vklayers=all \
--without-steamrt-depends \
--without-tts \
--without-nvidia-libs \
--enable-wow64
# The top-level make is serial by design; SUBJOBS is what it hands to each
# component's build.
SUBJOBS="${JOBS}" make -j1 dist
# ── Install ─────────────────────────────────────────────
mkdir -p "${OUT_DIR}"
cp -a "${BUILD_DIR}/dist/." "${OUT_DIR}/"
# Debug symbols in the bundled PE runtimes are dead weight in a guest image.
cd "${OUT_DIR}/files"
find "share/wine/gecko/wine-gecko-${GECKO_VER}-x86" -name '*.dll' -o -name '*.exe' 2>/dev/null \
| xargs -r i686-w64-mingw32-strip --strip-debug 2>/dev/null || true
find "share/wine/gecko/wine-gecko-${GECKO_VER}-x86_64" -name '*.dll' -o -name '*.exe' 2>/dev/null \
| xargs -r x86_64-w64-mingw32-strip --strip-debug 2>/dev/null || true
find "share/wine/mono/wine-mono-${MONO_VER}" -name '*.dll' -o -name '*.exe' 2>/dev/null \
| xargs -r i686-w64-mingw32-strip --strip-debug 2>/dev/null || true
rm -rf "${BUILD_DIR}"
echo "proton: installed to ${OUT_DIR}"
build/scripts/proton-fetch.sh
Executable
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Fetches proton-cachyos' source and its bundled runtimes. Container-only.
#
# Deliberately its own script, and its own layer: the submodule checkout runs
# well past ten minutes, and it must not be redone every time a build flag or a
# missing dependency changes. Keep everything that can fail *fast* in
# proton-build.sh instead.
set -euo pipefail
: "${PROTON_GIT:?}"
: "${PROTON_TAG:?}"
: "${GECKO_VER:?}"
: "${MONO_VER:?}"
: "${XALIA_VER:?}"
SRC_DIR="/build/proton-cachyos"
git clone --branch "${PROTON_TAG}" --depth=1 "${PROTON_GIT}" "${SRC_DIR}"
cd "${SRC_DIR}"
# Relative submodule paths resolve against origin, so it has to be the real URL
# even though we cloned by tag.
git remote set-url origin "${PROTON_GIT}"
# No --depth here: submodules are pinned to commits that are often not a branch
# tip. --filter=tree:0 keeps the download down instead.
git submodule update --init --filter=tree:0 --recursive
# Still needed with wow64: these are PE-side, and a 32-bit Windows program wants
# the 32-bit gecko and mono regardless of how wine is built.
mkdir -p contrib
for url in \
"https://dl.winehq.org/wine/wine-gecko/${GECKO_VER}/wine-gecko-${GECKO_VER}-x86.tar.xz" \
"https://dl.winehq.org/wine/wine-gecko/${GECKO_VER}/wine-gecko-${GECKO_VER}-x86_64.tar.xz" \
"https://github.com/madewokherd/wine-mono/releases/download/wine-mono-${MONO_VER}/wine-mono-${MONO_VER}-x86.tar.xz" \
"https://github.com/madewokherd/xalia/releases/download/xalia-${XALIA_VER}/xalia-${XALIA_VER}-net48-mono.zip" \
; do
curl -fL --retry 3 -o "contrib/$(basename "$url")" "$url"
done
# Proton's cargo rule runs `cargo build --locked --offline`, so every crate has
# to be in CARGO_HOME before the build starts — including the git dependencies,
# which is what the "you are in the offline mode" failure is really saying. The
# error names a URL that is perfectly reachable; the build simply refuses to go
# out and get it.
#
# gst-plugins-rs is the only cargo component in the tree. Both targets are
# fetched: wow64 should mean nothing builds the i386 unix side, but a fetch is
# metadata only and costs almost nothing next to being wrong about that.
#
# CARGO_HOME is left at its default so it lands in this layer and the build
# layer inherits it.
export CARGO_NET_GIT_FETCH_WITH_CLI=true
export RUSTUP_TOOLCHAIN=stable
cd "${SRC_DIR}/gst-plugins-rs"
cargo fetch --locked --target x86_64-unknown-linux-gnu
cargo fetch --locked --target i686-unknown-linux-gnu
echo "proton: source at ${SRC_DIR}"
crates/nesprotocol/src/lifecycle.rs
+277 -50
View File
@@ -1,14 +1,34 @@
// The lifecycle layer of the control channel between a box and whatever runs // The lifecycle layer of the control channel between a box and whatever runs
// it: the boot descriptor the guest is handed, and what the guest says back // it: the document describing the box, the commands that run things inside it,
// about carrying it out. // and what the guest says back about carrying either out.
// //
// It lives beside the media types for the same reason they live here — one // It lives beside the media types for the same reason they live here — one
// definition, so the two ends cannot drift from each other silently. // definition, so the two ends cannot drift from each other silently.
// //
// Nothing in this module describes *what* the guest runs. A command line, a // Nothing in this module describes *what* the guest runs. A set of share tags,
// set of share tags, an output geometry, and what an exit means: that is the // a command line, and what an exit means: that is the whole vocabulary, and a
// whole vocabulary, and a field that only makes sense for one kind of workload // field that only makes sense for one kind of workload does not belong in it.
// does not belong in it. ref(d-0033) // ref(d-0033)
//
// # The box outlives what runs in it
//
// The guest init is resident: it mounts what the descriptor names, brings up
// the box's service stack, says so, and then takes commands for as long as the
// box lives. So the descriptor describes the *box* — which shares are mounted
// where — and a command describes an occupant. A box may be launched into many
// times. ref(d-0064)
//
// That is why every launch carries an id and every event about a launch carries
// it back. Without one, a second launch's exit is indistinguishable from the
// first's, which reads as an ended session that keeps billing or a running one
// reported as stopped.
//
// # What is deliberately absent
//
// **Output geometry.** The compositor wraps the workload rather than running as
// a service, so it is started by a launch with that launch's geometry in its own
// argv, and the numbers appear nowhere else. Two sources of truth for one number
// is a worse failure than either choice, because the wrong one is used silently.
// //
// The channel also carries a second layer, which the guest relays as opaque // The channel also carries a second layer, which the guest relays as opaque
// bytes and never parses. Those types land with the relay that needs them. // bytes and never parses. Those types land with the relay that needs them.
@@ -34,7 +54,12 @@ pub const CONTROL_PORT: u32 = 7000;
/// ///
/// Adding a variant or a field does not need a bump; removing or renaming one /// Adding a variant or a field does not need a bump; removing or renaming one
/// does. /// does.
pub const CONTROL_VERSION: u32 = 2; ///
/// Version 3 took three fields off the descriptor and put an id on three
/// messages, so a version-2 peer and a version-3 peer do not talk at all.
/// There is deliberately no shim: nothing is deployed, and a shim would be the
/// second definition of this wire that one shared crate exists to prevent.
pub const CONTROL_VERSION: u32 = 3;
/// The command to run, and who runs it. /// The command to run, and who runs it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -76,14 +101,32 @@ pub struct Mount {
pub ro: bool, pub ro: bool,
} }
/// The output the compositor should produce. /// Names one launch, for as long as anything has something to say about it.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] ///
pub struct Geometry { /// Minted by the caller and only ever echoed by the guest. A guest that
pub width: u32, /// generated these would be naming things the caller then has to correlate
pub height: u32, /// against something else.
pub fps: u32, ///
#[serde(default)] /// Opaque on purpose: nothing in this layer parses it, and a caller that wants
pub hdr: bool, /// meaning in it can put meaning in it without this crate having an opinion.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(transparent)]
pub struct LaunchId(pub String);
impl LaunchId {
pub fn new(id: impl Into<String>) -> Self {
Self(id.into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for LaunchId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
} }
/// What the workload exiting means for the session. /// What the workload exiting means for the session.
@@ -95,21 +138,33 @@ pub struct OnExit {
/// reports the exit and stops, and starting something again is a new /// reports the exit and stops, and starting something again is a new
/// command from the caller — the only end that can see whether restarting /// command from the caller — the only end that can see whether restarting
/// is repair or a loop. ref(d-0033) /// is repair or a loop. ref(d-0033)
///
/// It rides on the launch rather than on the descriptor, because a box that
/// can be launched into repeatedly cannot have one answer to this fixed at
/// boot. ref(d-0064)
pub terminal: bool, pub terminal: bool,
} }
/// Everything the guest is told at boot, in one document. /// What the box *is*, in one document: the shares it has and where they land.
/// ///
/// Sent once, immediately after the handshake, and read once. Deliberately not /// Sent once, immediately after the handshake, and read once. Deliberately not
/// a conversation: boot configuration is a document, and a document cannot /// a conversation: this much is a document, and a document cannot half-arrive.
/// half-arrive. /// What runs *in* the box is a conversation, and a separate one — see
/// [`HostToGuest::Launch`]. ref(d-0064)
///
/// An empty `mounts` is legitimate. A box with nothing mounted still boots and
/// still brings up its services.
///
/// `deny_unknown_fields` is load-bearing rather than strictness for its own
/// sake. A descriptor still carrying a command line is a caller that has not
/// been updated, and the default behaviour — ignore what it does not recognise —
/// would mount the shares, silently drop the command, and leave a box that came
/// up correctly and runs nothing. Refusing it says so instead.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BootDescriptor { pub struct BootDescriptor {
pub exec: Exec,
#[serde(default)] #[serde(default)]
pub mounts: Vec<Mount>, pub mounts: Vec<Mount>,
pub geometry: Geometry,
pub on_exit: OnExit,
} }
/// How a workload ended. /// How a workload ended.
@@ -145,6 +200,12 @@ impl Exit {
#[serde(tag = "type", rename_all = "snake_case")] #[serde(tag = "type", rename_all = "snake_case")]
pub enum GuestToHost { pub enum GuestToHost {
/// First line on the connection, before anything else is read or written. /// First line on the connection, before anything else is read or written.
///
/// **This is the handshake, not readiness.** It says a connection exists
/// and both ends speak the same version. Whether anything in the box works
/// is [`GuestToHost::Initialized`], which is a different fact and must not
/// be merged with this one — a caller that treats this as readiness has a
/// wait stage that succeeds before the guest has started anything.
Ready { protocol_version: u32 }, Ready { protocol_version: u32 },
/// Every share the descriptor named is where it said to put it. /// Every share the descriptor named is where it said to put it.
Mounted, Mounted,
@@ -154,13 +215,45 @@ pub enum GuestToHost {
/// looked at: a share that did not appear and a command that did not run /// looked at: a share that did not appear and a command that did not run
/// are not the same incident. /// are not the same incident.
MountFailed { reason: String }, MountFailed { reason: String },
/// The command the descriptor named is running. /// The box's own services are up and it will accept launches.
Started, ///
/// The command could not be run, in the words the operating system used. /// The one fact a caller waits on before it may launch anything. It names
StartFailed { reason: String }, /// what came up, so a log says which — an empty list is a box with no
/// The workload the descriptor named has ended. Terminal or not is the /// service stack, which is legitimate and worth being able to see.
/// descriptor's answer, not this message's. Initialized {
#[serde(default)]
services: Vec<String>,
},
/// The box's services could not be brought up, in the words of whatever
/// refused.
///
/// Kept separate from [`GuestToHost::MountFailed`] and
/// [`GuestToHost::StartFailed`] for the reason those two are separate from
/// each other: a share that did not appear, a box that could not be made,
/// and a command that did not run are three incidents that want three
/// different things looked at. A box in this state cannot be launched into
/// at all, which is what distinguishes it from a refused launch.
InitFailed { reason: String },
/// A service in the box's own stack exited.
///
/// **Reported, never repaired.** Nothing else in the guest is watching
/// these, so a death that is not said here is a box that looks healthy and
/// cannot work. Restarting one is a decision for whoever can see whether
/// restarting is repair or a loop, and that is not this end. ref(d-0064)
ServiceDied {
name: String,
#[serde(flatten)]
exit: Exit,
},
/// The launch with this id is running.
Started { id: LaunchId },
/// The launch with this id could not be run, in the words the operating
/// system used.
StartFailed { id: LaunchId, reason: String },
/// The launch with this id has ended. Terminal or not is the launch's own
/// answer, not this message's.
WorkloadExited { WorkloadExited {
id: LaunchId,
#[serde(flatten)] #[serde(flatten)]
exit: Exit, exit: Exit,
}, },
@@ -182,8 +275,26 @@ pub enum HostToGuest {
#[serde(flatten)] #[serde(flatten)]
descriptor: Box<BootDescriptor>, descriptor: Box<BootDescriptor>,
}, },
/// Stop the workload. Idempotent, and does not end the session. /// Run something in the box. Any number of times, after `initialized`.
Stop, ///
/// The caller mints `id` and every message about this launch carries it
/// back. `on_exit` belongs here rather than on the descriptor because a box
/// that can be launched into repeatedly has one answer per launch, not one
/// per boot. ref(d-0064)
Launch {
id: LaunchId,
exec: Exec,
on_exit: OnExit,
},
/// Stop one launch. Idempotent, and does not end the session.
Stop { id: LaunchId },
/// Stop one launch and start it again with the same command.
///
/// **Defined as a kill followed by a launch of the same `Exec`, and nothing
/// more.** No retry, no backoff, no policy of any kind in the guest — it
/// exists as one message only because a caller sending two has the same
/// effect with a worse race in it. The relaunch keeps the id. ref(d-0064)
Restart { id: LaunchId },
/// Shut the guest down. /// Shut the guest down.
Shutdown, Shutdown,
/// Bytes for the workload, relayed. See [`Payload`]. /// Bytes for the workload, relayed. See [`Payload`].
@@ -260,25 +371,21 @@ mod tests {
fn descriptor() -> BootDescriptor { fn descriptor() -> BootDescriptor {
BootDescriptor { BootDescriptor {
exec: Exec {
argv: vec!["/usr/bin/true".into()],
env: BTreeMap::from([("HOME".to_string(), "/mnt/user".to_string())]),
cwd: Some("/mnt/user".into()),
uid: 1000,
gid: 1000,
},
mounts: vec![Mount { mounts: vec![Mount {
tag: "install".into(), tag: "install".into(),
at: "/mnt/install".into(), at: "/mnt/install".into(),
ro: true, ro: true,
}], }],
geometry: Geometry { }
width: 1920, }
height: 1080,
fps: 60, fn exec() -> Exec {
hdr: false, Exec {
}, argv: vec!["/usr/bin/true".into()],
on_exit: OnExit { terminal: true }, env: BTreeMap::from([("HOME".to_string(), "/mnt/user".to_string())]),
cwd: Some("/mnt/user".into()),
uid: 1000,
gid: 1000,
} }
} }
@@ -300,9 +407,122 @@ mod tests {
); );
} }
#[test]
fn a_launch_round_trips_with_its_id() {
let launch = HostToGuest::Launch {
id: LaunchId::new("l-1"),
exec: exec(),
on_exit: OnExit { terminal: true },
};
let back: HostToGuest = from_line(&to_line(&launch).unwrap()).unwrap();
assert_eq!(back, launch);
}
/// The reason ids exist: a caller has to be able to tell which launch it is
/// being told about, or the second one's exit overwrites the first's record.
#[test]
fn every_event_about_a_launch_carries_the_launch_it_is_about() {
let first = LaunchId::new("l-1");
let second = LaunchId::new("l-2");
let events = [
GuestToHost::Started { id: first.clone() },
GuestToHost::StartFailed {
id: first.clone(),
reason: "ENOENT".into(),
},
GuestToHost::WorkloadExited {
id: first.clone(),
exit: Exit::code(0),
},
];
for event in events {
let line = to_line(&event).unwrap();
assert!(
line.contains(first.as_str()) && !line.contains(second.as_str()),
"an event does not say which launch it is about: {line}"
);
let back: GuestToHost = from_line(&line).unwrap();
assert_eq!(back, event);
}
}
/// The descriptor describes the box. A caller still sending a command in it
/// has not been updated, and the cost of accepting one quietly is a box that
/// mounts, comes up, and runs nothing.
#[test]
fn a_descriptor_carrying_a_command_is_refused_rather_than_ignored() {
let stale = r#"{"exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000},
"mounts":[],
"geometry":{"width":1280,"height":720,"fps":60},
"on_exit":{"terminal":true}}"#;
let parsed: Result<BootDescriptor, _> = from_line(stale);
assert!(
parsed.is_err(),
"a descriptor with a command in it parsed: {parsed:?}"
);
}
/// Geometry is in the launched argv and nowhere else, so there is no field
/// here for it to disagree with.
#[test]
fn geometry_is_not_on_this_layer() {
let line = to_line(&HostToGuest::Boot {
descriptor: Box::new(descriptor()),
})
.unwrap();
for named in ["width", "height", "fps", "hdr", "geometry"] {
assert!(
!line.contains(named),
"the descriptor names {named}, which belongs in the launch: {line}"
);
}
}
/// A dead service is not a dead workload: they are different incidents and
/// want different things looked at.
#[test]
fn a_dead_service_says_which_one_and_how() {
let died = GuestToHost::ServiceDied {
name: "pipewire".into(),
exit: Exit::signal(9),
};
let line = to_line(&died).unwrap();
assert!(line.contains("pipewire"), "no service name: {line}");
assert!(
!line.contains("exit_code"),
"a signalled service has no exit code: {line}"
);
let back: GuestToHost = from_line(&line).unwrap();
assert_eq!(back, died);
}
/// `ready` is the handshake and `initialized` is the box working. A caller
/// waiting on the wrong one succeeds before anything has started.
#[test]
fn readiness_and_initialisation_are_two_messages() {
let ready = to_line(&GuestToHost::Ready {
protocol_version: CONTROL_VERSION,
})
.unwrap();
let initialized = to_line(&GuestToHost::Initialized {
services: vec!["dbus".into(), "pipewire".into()],
})
.unwrap();
assert_ne!(ready, initialized);
// An empty stack is legitimate and has to survive the round trip, or a
// box with no services looks like a box that never came up.
let empty = GuestToHost::Initialized { services: vec![] };
let back: GuestToHost = from_line(&to_line(&empty).unwrap()).unwrap();
assert_eq!(back, empty);
}
#[test] #[test]
fn a_signalled_exit_is_not_a_zero_exit() { fn a_signalled_exit_is_not_a_zero_exit() {
let signalled = to_line(&GuestToHost::WorkloadExited { let signalled = to_line(&GuestToHost::WorkloadExited {
id: LaunchId::new("l-1"),
exit: Exit::signal(9), exit: Exit::signal(9),
}) })
.unwrap(); .unwrap();
@@ -312,6 +532,7 @@ mod tests {
); );
let clean = to_line(&GuestToHost::WorkloadExited { let clean = to_line(&GuestToHost::WorkloadExited {
id: LaunchId::new("l-1"),
exit: Exit::code(0), exit: Exit::code(0),
}) })
.unwrap(); .unwrap();
@@ -381,13 +602,19 @@ mod tests {
#[test] #[test]
fn defaults_cover_what_a_caller_may_leave_out() { fn defaults_cover_what_a_caller_may_leave_out() {
let json = r#"{"exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000}, // A box with nothing mounted is a legitimate box.
"geometry":{"width":1280,"height":720,"fps":30}, let parsed: BootDescriptor = from_line("{}").unwrap();
"on_exit":{"terminal":false}}"#;
let parsed: BootDescriptor = from_line(json).unwrap();
assert!(parsed.mounts.is_empty()); assert!(parsed.mounts.is_empty());
assert!(parsed.exec.env.is_empty());
assert_eq!(parsed.exec.cwd, None); let launch = r#"{"type":"launch","id":"l-1",
assert!(!parsed.geometry.hdr); "exec":{"argv":["/bin/sh"],"uid":1000,"gid":1000},
"on_exit":{"terminal":false}}"#;
let HostToGuest::Launch { id, exec, on_exit } = from_line(launch).unwrap() else {
panic!("not a launch: {launch}")
};
assert_eq!(id, LaunchId::new("l-1"));
assert!(exec.env.is_empty());
assert_eq!(exec.cwd, None);
assert!(!on_exit.terminal);
} }
} }