From 4a0a790fd16112aafe6ba3b65bad387b6d31bb1c Mon Sep 17 00:00:00 2001 From: Wanjohi Date: Sat, 17 Oct 2026 13:37:05 +0000 Subject: [PATCH] feat(nesinit): several launches in one box, and mounting into a running box (#424) Runtime half of the window streams contract (joins 1 and 6). - nesinit runs any number of launches side by side, keyed by launch id. Each process gets NESTRI_LAUNCH_ID (nesprotocol::lifecycle::LAUNCH_ID_ENV), set after the caller's environment so it cannot be overridden. Stop and restart reach one launch only. A second launch under an id that is already running is refused. - A terminal launch ends the box only when no other launch is running, so the box stops when its last launch exits. A box with one launch behaves as before. - New lifecycle messages, additive, no CONTROL_VERSION bump: host to guest `mount {id, images, overlays}`; guest to host `launch_mounted {id}` / `launch_mount_failed {id, reason}`. A failed mount does not take the box down. - A mount source that is a regular file (in a virtiofs share) is attached to a loop device first (LOOP_CONFIGURE, autoclear), because nesbox cannot hot-add block devices to a running VM. The guest kernel fragment gains CONFIG_BLK_DEV_LOOP=y, so this needs a new guest kernel/image to work end to end. Tests: cargo test -p nesinit (117 passed) and the nesprotocol lifecycle tests on nestripc-1. New session tests cover two launches side by side, refusing a duplicate id, stopping one launch, the box ending with its last launch, and mount success/failure/before boot. Not run in a real box. Companion: nestrilabs/nescore feat/window-streams-runtime (neslet launch.start/launch.stop, nesteam --join-compositor). Content layout and trust: - Game and compat images come from a read-only virtiofs share `launch-images` (mounted read-only at /nestri/launch-images//build.erofs and compat.erofs). The per-launch writable layer is in the read-write share `launches` (/nestri/launches//upper.ext4). Which shares exist is decided host-side in the boot descriptor; nesinit mounts what it is told. - The share's only guest-writable files are the per-launch upper.ext4 files. - Mount sources are opened with openat2(RESOLVE_NO_SYMLINKS): a link at any component is refused, and only block devices and regular files are accepted. Loop attach and mounts run on the blocking pool, so other launches' exits and stops are not stalled. - CONFIG_BLK_DEV_LOOP must ship in the guest kernel together with this nesinit (build/kernel/nestri.fragment has it); without it a mount into a running box fails with a launch_mount_failed. --- apps/nesinit/src/session.rs | 647 ++++++++++++++++++++++++---- apps/nesinit/src/workload.rs | 463 ++++++++++++++++++-- build/kernel/nestri.fragment | 2 + crates/nesprotocol/src/lifecycle.rs | 80 ++++ 4 files changed, 1081 insertions(+), 111 deletions(-) diff --git a/apps/nesinit/src/session.rs b/apps/nesinit/src/session.rs index 6032902a..9cc6dddc 100644 --- a/apps/nesinit/src/session.rs +++ b/apps/nesinit/src/session.rs @@ -10,14 +10,22 @@ // something again is the caller's decision, because the caller is the only end // that can see whether restarting is repair or a loop. ref(d-0033) // -// # One launch at a time +// # Several launches at once // -// A box may be launched into many times over its life, and not twice at once. -// A launch arriving while one is running is refused with the id it was asked -// for, rather than queued or silently replacing it — a box has one compositor -// and one screen, so a second concurrent launch has nowhere to draw. Nothing -// in the wire format forbids it if that ever changes; this is a property of -// this implementation and it says so where it refuses. +// A box is a person's machine with any of their games in it, and every launch +// in it is named by its id. ref(d-0085) A second launch runs beside the first +// rather than replacing it; only a second launch under an id that is already +// running is refused, because two processes answering to one id cannot be told +// apart by anything reported about them. Content for a launch may be mounted +// after boot (`Mount`), which is how a game the box did not boot with reaches it. +// +// # The box ends when its last terminal launch has gone +// +// A launch whose exit is terminal ends the session when it exits, as it always +// has -- unless other launches are still running. Then the box stays up for +// them, and ends when the last of them exits: closing every window is the +// person saying they are done, and closing one is not. A box with one launch +// behaves exactly as it did before there could be two. use nesprotocol::lifecycle::{ CONTROL_VERSION, Exec, Exit, GuestToHost, HostToGuest, LaunchId, OnExit, Payload, from_line, @@ -27,7 +35,12 @@ use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader use crate::payload::Ports; use crate::services::{Died, Services}; -use crate::workload::{Exited, Failure, Workload}; +use crate::workload::{Failure, Workload}; +use std::collections::BTreeMap; +use tokio::task::JoinSet; + +/// Every launch's exit, as it arrives, named by the launch. +type Exits = JoinSet<(LaunchId, std::io::Result)>; use tokio::sync::mpsc::Receiver; /// How a session ended. @@ -49,14 +62,12 @@ pub enum Outcome { Refused(Failure), } -/// The launch that is running, and what it would take to start it again. +/// A launch that is running, and what it would take to start it again. struct Current { - id: LaunchId, /// Kept so a restart is the same command rather than a new one the caller /// has to re-send. exec: Exec, on_exit: OnExit, - exited: Exited, /// Set by `restart`, read when the exit arrives. A restart is a kill /// followed by a launch and this is the "followed by". relaunch: bool, @@ -129,7 +140,16 @@ where ) .await?; - let mut running: Option = None; + let mut running: BTreeMap = BTreeMap::new(); + let mut exits: Exits = JoinSet::new(); + // A terminal launch's exit, once it has ended while others were still + // running. The box stays up for them, takes no new launches, and ends with + // this exit when the last of them has gone. + let mut ending: Option = None; + // Each launch's content mounted after boot, by launch, so it can be taken + // down with the launch. Recorded when asked for, before it has finished. + let mut mounted: Mounted = BTreeMap::new(); + let mut mounting: JoinSet<(LaunchId, Result<(), Failure>)> = JoinSet::new(); let mut booted = false; let mut relay_open = true; let mut carrier_open = true; @@ -139,32 +159,37 @@ where let mut services_open = true; loop { - let event = match running.as_mut() { - Some(current) => tokio::select! { - ended = &mut current.exited => Event::Ended(ended?), - line = lines.next_line() => Event::Line(line?), - up = payload.from_workload.recv(), if relay_open => Event::FromWorkload(up), - found = addresses.recv(), if carrier_open => Event::Address(found), - gone = services.deaths().recv(), if services_open => Event::ServiceDied(gone), + // The exits are guarded like the channels below: an empty set resolves + // at once, and an unguarded branch on it would spin. + let event = tokio::select! { + ended = exits.join_next(), if !exits.is_empty() => match ended { + Some(Ok((id, exit))) => Event::Ended(id, exit?), + Some(Err(error)) => return Err(std::io::Error::other(error)), + None => continue, }, - None => tokio::select! { - line = lines.next_line() => Event::Line(line?), - up = payload.from_workload.recv(), if relay_open => Event::FromWorkload(up), - found = addresses.recv(), if carrier_open => Event::Address(found), - gone = services.deaths().recv(), if services_open => Event::ServiceDied(gone), + done = mounting.join_next(), if !mounting.is_empty() => match done { + Some(Ok((id, result))) => Event::MountDone(id, result), + Some(Err(error)) => return Err(std::io::Error::other(error)), + None => continue, }, + line = lines.next_line() => Event::Line(line?), + up = payload.from_workload.recv(), if relay_open => Event::FromWorkload(up), + found = addresses.recv(), if carrier_open => Event::Address(found), + gone = services.deaths().recv(), if services_open => Event::ServiceDied(gone), }; let line = match event { - Event::Ended(exit) => { + Event::Ended(id, exit) => { // Taken before anything is sent: whatever happens next, this - // launch is no longer the running one, and a failure to + // launch is no longer a running one, and a failure to // relaunch must not leave a dead launch looking live. - let ended = running.take().expect("an exit came from a launch"); + let ended = running + .remove(&id) + .expect("an exit came from a running launch"); send( &mut writer, &GuestToHost::WorkloadExited { - id: ended.id.clone(), + id: id.clone(), exit, }, ) @@ -175,26 +200,76 @@ where // command, and this is the second half. The id is kept, so // a caller sees the exit and then the same launch running // again rather than having to correlate a new name. - running = start( + let started = start( &mut writer, workload, untrusted, - ended.id, + (&mut running, &mut exits), + id.clone(), ended.exec, ended.on_exit, ) .await?; + if !started { + tear_down(workload, &mut mounted, &id); + // The last launch of an ending box would not come + // back, so nothing is left to wait for. + if let (Some(exit), true) = (ending, running.is_empty()) { + return Ok(Outcome::WorkloadExited(exit)); + } + } continue; } + tear_down(workload, &mut mounted, &id); // Terminal is the launch's own answer, not this end's. A // non-terminal exit leaves the box up and waiting to be // launched into again, which is the whole point of the box // outliving what runs in it. ref(d-0064) - if ended.on_exit.terminal { + // + // A terminal one ends the box only once nothing else is + // running in it. ref(d-0085) + // + // The exit the box ends with is the terminal launch's own, + // whichever launch happens to be the last to go. + if ended.on_exit.terminal && ending.is_none() { + ending = Some(exit); + } + if let (Some(exit), true) = (ending, running.is_empty()) { return Ok(Outcome::WorkloadExited(exit)); } - tracing::info!(launch = %ended.id, "a launch ended and the box stays up"); + if ending.is_some() { + tracing::info!( + launch = %id, + still_running = running.len(), + "a terminal launch ended; the box stays up until the others have" + ); + } else { + tracing::info!(launch = %id, "a launch ended and the box stays up"); + } + continue; + } + Event::MountDone(id, result) => { + match result { + Ok(()) => { + tracing::info!(launch = %id, "mounted a launch's content"); + send(&mut writer, &GuestToHost::LaunchMounted { id }).await? + } + Err(failure) => { + // Nothing of it is recorded as mounted: what did mount + // before the failure is taken down with it. + tear_down(workload, &mut mounted, &id); + tracing::warn!(launch = %id, reason = %failure.reason, "a launch's content would not mount"); + send( + &mut writer, + &GuestToHost::LaunchMountFailed { + id, + reason: failure.reason, + }, + ) + .await? + } + } continue; } Event::ServiceDied(Some(Died { name, exit })) => { @@ -351,13 +426,67 @@ where refuse(&mut writer, id, "the box has not been told what it is").await?; continue; } - if let Some(current) = &running { - let reason = format!("this box is already running a launch: {}", current.id); + if running.contains_key(&id) { + // Two processes under one id could not be told apart by + // anything reported about either. + let reason = format!("a launch with this id is already running: {id}"); refuse(&mut writer, id, &reason).await?; continue; } + if ending.is_some() { + // The box was ended by a terminal launch and is waiting + // only for the launches already in it. A new one would + // keep alive a box its person has finished with. + refuse( + &mut writer, + id, + "the box is ending: a launch that ends it has exited", + ) + .await?; + continue; + } - running = start(&mut writer, workload, untrusted, id, exec, on_exit).await?; + let started = start( + &mut writer, + workload, + untrusted, + (&mut running, &mut exits), + id.clone(), + exec, + on_exit, + ) + .await?; + if !started { + tear_down(workload, &mut mounted, &id); + } + } + HostToGuest::Mount { + id, + images, + overlays, + } => { + // A launch's content, mounted into a box that is already up. + // Never ends the session: a game whose image will not mount is + // one launch that cannot start. ref(d-0085) + // + // Carried out off this loop, which also carries every other + // launch's exits and stops: a mount over a share can take a + // while, and nothing else should wait on it. + let refused = if !booted { + Some("the box has not been told what it is".to_string()) + } else if mounted.contains_key(&id) { + Some(format!("content is already mounted for this launch: {id}")) + } else { + None + }; + if let Some(reason) = refused { + tracing::warn!(launch = %id, reason, "refusing a mount"); + send(&mut writer, &GuestToHost::LaunchMountFailed { id, reason }).await?; + continue; + } + mounted.insert(id.clone(), (images.clone(), overlays.clone())); + let pending = workload.mount_launch(images, overlays); + mounting.spawn(async move { (id, pending.await) }); } HostToGuest::Attach { id, exec, size } => { // A shell is not a launch: it does not touch `running`, takes no @@ -377,21 +506,21 @@ where .await?; } } - HostToGuest::Stop { id } => match &running { + HostToGuest::Stop { id } => match running.contains_key(&id) { // Idempotent, and never ends the session by itself. - Some(current) if current.id == id => workload.signal_stop(), + true => workload.stop_launch(&id), // Not an error and not worth failing: a caller stopping // something that has already stopped got what it asked for. // Signalling anyway would aim a kill at whatever is running // now, which is the one thing this must not do. - _ => tracing::debug!(launch = %id, "nothing with that id is running to stop"), + false => tracing::debug!(launch = %id, "nothing with that id is running to stop"), }, - HostToGuest::Restart { id } => match running.as_mut() { - Some(current) if current.id == id => { + HostToGuest::Restart { id } => match running.get_mut(&id) { + Some(current) => { // Kill now, launch when the exit arrives. Doing both here // would start the second before the first had gone. current.relaunch = true; - workload.signal_stop(); + workload.stop_launch(&id); } // Nothing is remembered about a launch that has already ended, // so this cannot be a launch in disguise: the caller has the @@ -413,16 +542,19 @@ where /// Start one launch, reporting which of the two things happened. /// -/// Returns `None` when it could not be started, which is not the end of the -/// session: a box that dies of a bad `argv` cannot be told a better one. +/// A launch that could not be started is reported and is not the end of the +/// session: a box that dies of a bad `argv` cannot be told a better one. One +/// that started is added to the running set, and its exit to `exits`. +#[allow(clippy::too_many_arguments)] async fn start( writer: &mut Wr, workload: &mut W, untrusted: &crate::ticket::Untrusted, + (running, exits): (&mut BTreeMap, &mut Exits), id: LaunchId, exec: Exec, on_exit: OnExit, -) -> std::io::Result> +) -> std::io::Result where W: Workload, Wr: AsyncWrite + Unpin, @@ -448,7 +580,7 @@ where ), ) .await?; - return Ok(None); + return Ok(false); } // Before the workload exists, so there is no window in which it is running @@ -456,24 +588,49 @@ where // address. untrusted.is(exec.uid); - match workload.start(&exec) { + match workload.start(&id, &exec) { Ok(exited) => { send(writer, &GuestToHost::Started { id: id.clone() }).await?; - Ok(Some(Current { - id, - exec, - on_exit, - exited, - relaunch: false, - })) + running.insert( + id.clone(), + Current { + exec, + on_exit, + relaunch: false, + }, + ); + exits.spawn(async move { (id, exited.await) }); + Ok(true) } Err(failure) => { refuse(writer, id, &failure.reason).await?; - Ok(None) + Ok(false) } } } +/// Each launch's content mounted after boot: its images and its overlays. +type Mounted = BTreeMap, Vec)>; + +/// Take down what was mounted for a launch that has gone, so the loop devices +/// under it clear themselves. Off this loop, like the mount, and logged +/// rather than reported: the launch it was for has already been reported. +fn tear_down(workload: &mut W, mounted: &mut Mounted, id: &LaunchId) { + let Some((images, overlays)) = mounted.remove(id) else { + return; + }; + let pending = workload.unmount_launch(images, overlays); + let id = id.clone(); + tokio::spawn(async move { + match pending.await { + Ok(()) => tracing::info!(launch = %id, "unmounted a launch's content"), + Err(failure) => { + tracing::warn!(launch = %id, reason = %failure.reason, "a launch's content did not unmount") + } + } + }); +} + /// Say that a launch will not happen, in the words of whatever refused it. async fn refuse(writer: &mut Wr, id: LaunchId, reason: &str) -> std::io::Result<()> where @@ -493,7 +650,8 @@ where /// What the session is waiting on. enum Event { Line(Option), - Ended(Exit), + Ended(LaunchId, Exit), + MountDone(LaunchId, Result<(), Failure>), FromWorkload(Option), Address(Option), ServiceDied(Option), @@ -859,14 +1017,318 @@ mod tests { assert!(workload.started.is_empty()); } - /// Two at once has nowhere to draw. The refusal carries the id that was - /// refused, not the one that is running. - #[tokio::test] - async fn a_second_launch_is_refused_while_the_first_is_running() { + /// Boot a box and hand back the caller, ready to launch into. + async fn booted(given: Given) -> (Caller, tokio::task::JoinHandle<(Outcome, Double, Stack)>) { let (guest, host) = tokio::io::duplex(4096); let mut caller = Caller::new(host); - let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + let session = spawn(guest, given); + caller.expect_ready().await; + caller + .say(&HostToGuest::Boot { + descriptor: Box::new(descriptor()), + }) + .await; + caller.expect_booted().await; + (caller, session) + } + fn started(id: &str) -> GuestToHost { + GuestToHost::Started { + id: LaunchId::new(id), + } + } + + fn exited(id: &str) -> GuestToHost { + GuestToHost::WorkloadExited { + id: LaunchId::new(id), + exit: Exit::code(0), + } + } + + /// A box is a person's machine with any of their games in it: a second + /// launch runs beside the first. ref(d-0085) + #[tokio::test] + async fn a_second_launch_runs_beside_the_first() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!( + workload.started_ids, + vec![LaunchId::new("l-1"), LaunchId::new("l-2")] + ); + } + + /// Two processes under one id could not be told apart, so the second is + /// refused, by its id, and the first keeps running. + #[tokio::test] + async fn a_launch_under_an_id_that_is_running_is_refused() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch("l-1")).await; + let GuestToHost::StartFailed { id, reason } = caller.expect().await else { + panic!("a second launch under a running id was accepted") + }; + assert_eq!(id, LaunchId::new("l-1")); + assert!(reason.contains("already running"), "{reason}"); + + caller.say(&HostToGuest::Shutdown).await; + let (_, workload, _) = session.await.unwrap(); + assert_eq!(workload.started.len(), 1); + assert!(workload.stopped.iter().all(|s| s.is_none())); + } + + /// A stop names one launch and reaches that one only; the box and the + /// other launch carry on. + #[tokio::test] + async fn a_stop_reaches_its_own_launch_and_no_other() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch_and_stay("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + + caller + .say(&HostToGuest::Stop { + id: LaunchId::new("l-2"), + }) + .await; + assert_eq!(caller.expect().await, exited("l-2")); + + caller.say(&HostToGuest::Shutdown).await; + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::Shutdown); + assert_eq!(workload.stopped, vec![Some(LaunchId::new("l-2"))]); + } + + /// A terminal launch ending while another runs leaves the box up; the + /// box ends when the last launch has gone. ref(d-0085) + #[tokio::test] + async fn the_box_ends_when_its_last_launch_has_gone() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + + caller + .say(&HostToGuest::Stop { + id: LaunchId::new("l-1"), + }) + .await; + assert_eq!(caller.expect().await, exited("l-1")); + + // Still up, for the launch still in it. + for id in ["l-2"] { + caller + .say(&HostToGuest::Stop { + id: LaunchId::new(id), + }) + .await; + assert_eq!(caller.expect().await, exited(id)); + } + + let (outcome, _, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(0))); + } + + /// The box ends with the terminal launch's exit, not with whichever + /// launch happened to go last. + #[tokio::test] + async fn an_ending_box_reports_the_exit_of_the_launch_that_ended_it() { + let mut double = Double::exits_when_stopped(Exit::code(0)); + double.exit_for.insert(LaunchId::new("l-1"), Exit::code(7)); + let (mut caller, session) = booted(Given::new(double)).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-1") }).await; + assert!(matches!(caller.expect().await, GuestToHost::WorkloadExited { .. })); + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-2") }).await; + assert_eq!(caller.expect().await, exited("l-2")); + + let (outcome, _, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(7))); + } + + /// Once a terminal launch has ended, the box waits only for what is + /// already in it: a new launch is refused. + #[tokio::test] + async fn an_ending_box_takes_no_new_launch() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-1") }).await; + assert_eq!(caller.expect().await, exited("l-1")); + + caller.say(&launch_and_stay("l-3")).await; + let GuestToHost::StartFailed { id, reason } = caller.expect().await else { + panic!("an ending box took a new launch") + }; + assert_eq!(id, LaunchId::new("l-3")); + assert!(reason.contains("ending"), "{reason}"); + + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-2") }).await; + assert_eq!(caller.expect().await, exited("l-2")); + let (outcome, workload, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(0))); + assert_eq!(workload.started.len(), 2); + } + + /// The last launch of an ending box failing to restart leaves nothing to + /// wait for, and the box ends at once. + #[tokio::test] + async fn an_ending_box_whose_last_restart_fails_ends() { + let mut double = Double::exits_when_stopped(Exit::code(0)); + double.exit_for.insert(LaunchId::new("l-1"), Exit::code(3)); + double.fail_restart_of = Some(LaunchId::new("l-2")); + let (mut caller, session) = booted(Given::new(double)).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-1") }).await; + assert!(matches!(caller.expect().await, GuestToHost::WorkloadExited { .. })); + + caller.say(&HostToGuest::Restart { id: LaunchId::new("l-2") }).await; + assert_eq!(caller.expect().await, exited("l-2")); + assert!(matches!(caller.expect().await, GuestToHost::StartFailed { .. })); + + let (outcome, _, _) = session.await.unwrap(); + assert_eq!(outcome, Outcome::WorkloadExited(Exit::code(3))); + } + + fn overlay_for(id: &str) -> nesprotocol::lifecycle::Overlay { + nesprotocol::lifecycle::Overlay { + lower: format!("/nestri/launch-images/{id}/build.erofs"), + upper: format!("/nestri/launches/{id}/upper.ext4"), + at: format!("/nestri/installs/{id}"), + } + } + + /// A second mount for the same launch is refused, and what a launch + /// mounted is taken down when it ends. + #[tokio::test] + async fn a_launchs_content_is_mounted_once_and_taken_down_with_it() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + + let mount = HostToGuest::Mount { + id: LaunchId::new("l-2"), + images: Vec::new(), + overlays: vec![overlay_for("l-2")], + }; + caller.say(&mount).await; + assert_eq!(caller.expect().await, GuestToHost::LaunchMounted { id: LaunchId::new("l-2") }); + caller.say(&mount).await; + let GuestToHost::LaunchMountFailed { reason, .. } = caller.expect().await else { + panic!("a second mount for one launch was accepted") + }; + assert!(reason.contains("already mounted"), "{reason}"); + + caller.say(&launch_and_stay("l-2")).await; + assert_eq!(caller.expect().await, started("l-2")); + caller.say(&HostToGuest::Stop { id: LaunchId::new("l-2") }).await; + assert_eq!(caller.expect().await, exited("l-2")); + + caller.say(&HostToGuest::Shutdown).await; + let (_, workload, _) = session.await.unwrap(); + assert_eq!(workload.unmounted, vec![(Vec::new(), vec![overlay_for("l-2")])]); + } + + /// Content mounted for a launch that then will not start is taken down + /// too, rather than left for the life of the box. + #[tokio::test] + async fn content_for_a_launch_that_will_not_start_is_taken_down() { + let mut double = Double::exits_when_stopped(Exit::code(0)); + double.start_failure = Some(Failure::new("ENOENT")); + let (mut caller, session) = booted(Given::new(double)).await; + caller + .say(&HostToGuest::Mount { + id: LaunchId::new("l-2"), + images: Vec::new(), + overlays: vec![overlay_for("l-2")], + }) + .await; + assert_eq!(caller.expect().await, GuestToHost::LaunchMounted { id: LaunchId::new("l-2") }); + caller.say(&launch("l-2")).await; + assert!(matches!(caller.expect().await, GuestToHost::StartFailed { .. })); + caller.say(&HostToGuest::Shutdown).await; + let (_, workload, _) = session.await.unwrap(); + assert_eq!(workload.unmounted.len(), 1); + } + + /// Content for a launch reaches a box that is already up, and the answer + /// names the launch. + #[tokio::test] + async fn a_running_box_mounts_a_launchs_content_on_request() { + let (mut caller, session) = + booted(Given::new(Double::exits_when_stopped(Exit::code(0)))).await; + + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); + + let overlay = nesprotocol::lifecycle::Overlay { + lower: "/nestri/launch-images/l-2/build.erofs".into(), + upper: "/nestri/launches/l-2/upper.ext4".into(), + at: "/nestri/installs/l-2".into(), + }; + caller + .say(&HostToGuest::Mount { + id: LaunchId::new("l-2"), + images: Vec::new(), + overlays: vec![overlay.clone()], + }) + .await; + assert_eq!( + caller.expect().await, + GuestToHost::LaunchMounted { + id: LaunchId::new("l-2") + } + ); + + caller.say(&HostToGuest::Shutdown).await; + let (_, workload, _) = session.await.unwrap(); + assert_eq!(workload.overlays.last(), Some(&vec![overlay])); + assert!( + workload.stopped.is_empty(), + "mounting for one launch touched another" + ); + } + + /// A launch's content that will not mount is that launch's failure, and + /// the box stays up. + #[tokio::test] + async fn content_that_will_not_mount_does_not_take_the_box_down() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let mut double = Double::exits_when_stopped(Exit::code(0)); + double.named_image_failure = Some(Failure::new("/nestri/compat-2: ENOENT")); + let session = spawn(guest, Given::new(double)); caller.expect_ready().await; caller .say(&HostToGuest::Boot { @@ -875,35 +1337,48 @@ mod tests { .await; caller.expect_booted().await; - caller.say(&launch("l-1")).await; - assert_eq!( - caller.expect().await, - GuestToHost::Started { - id: LaunchId::new("l-1") - } - ); - - caller.say(&launch("l-2")).await; - let GuestToHost::StartFailed { id, reason } = caller.expect().await else { - panic!("a second concurrent launch was accepted") + caller + .say(&HostToGuest::Mount { + id: LaunchId::new("l-2"), + images: vec![nesprotocol::lifecycle::Image { + device: "/nestri/launch-images/l-2/compat.erofs".into(), + at: "/nestri/compat-2".into(), + }], + overlays: Vec::new(), + }) + .await; + let GuestToHost::LaunchMountFailed { id, reason } = caller.expect().await else { + panic!("a failed mount was not reported against its launch") }; - assert_eq!( - id, - LaunchId::new("l-2"), - "the refusal names the launch that is running rather than the one refused" - ); - assert!( - reason.contains("l-1"), - "the refusal does not say why: {reason}" - ); + assert_eq!(id, LaunchId::new("l-2")); + assert!(reason.contains("ENOENT"), "{reason}"); + caller.say(&launch("l-1")).await; + assert_eq!(caller.expect().await, started("l-1")); caller.say(&HostToGuest::Shutdown).await; - let (_, workload, _) = session.await.unwrap(); - assert_eq!( - workload.started.len(), - 1, - "the second launch ran anyway, so the first was replaced underneath it" - ); + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); + } + + /// Before the descriptor there is nothing to mount into. + #[tokio::test] + async fn a_mount_before_the_box_exists_is_refused_by_its_launch() { + let (guest, host) = tokio::io::duplex(4096); + let mut caller = Caller::new(host); + let session = spawn(guest, Given::new(Double::exits_when_stopped(Exit::code(0)))); + caller.expect_ready().await; + caller + .say(&HostToGuest::Mount { + id: LaunchId::new("l-2"), + images: Vec::new(), + overlays: Vec::new(), + }) + .await; + assert!(matches!( + caller.expect().await, + GuestToHost::LaunchMountFailed { .. } + )); + caller.say(&HostToGuest::Shutdown).await; + assert_eq!(session.await.unwrap().0, Outcome::Shutdown); } /// A launch that says its ending is not the session's leaves the box up and diff --git a/apps/nesinit/src/workload.rs b/apps/nesinit/src/workload.rs index 47ac774f..cd9d7c4c 100644 --- a/apps/nesinit/src/workload.rs +++ b/apps/nesinit/src/workload.rs @@ -11,7 +11,9 @@ use std::future::Future; use std::io; use std::pin::Pin; -use nesprotocol::lifecycle::{AttachId, Exec, Exit, Image, Mount, Overlay, Winsize}; +use nesprotocol::lifecycle::{ + AttachId, Exec, Exit, Image, LAUNCH_ID_ENV, LaunchId, Mount, Overlay, Winsize, +}; use std::os::unix::process::CommandExt; @@ -37,6 +39,9 @@ impl Failure { /// Resolves when the workload ends. pub type Exited = Pin> + Send>>; +/// Resolves when a mount or an unmount made off the session's loop is done. +pub type Pending = Pin> + Send>>; + pub trait Workload { /// Make the shares the descriptor names, where it says to put them. fn mount(&mut self, mounts: &[Mount]) -> Result<(), Failure>; @@ -51,6 +56,15 @@ pub trait Workload { /// layer, and the two together where it says. fn mount_overlays(&mut self, overlays: &[Overlay]) -> Result<(), Failure>; + /// Mount one launch's content into a box that is already up: its images, + /// then its overlays. Off the caller's thread, because the caller also + /// carries every other launch's exits and stops. ref(d-0085) + fn mount_launch(&mut self, images: Vec, overlays: Vec) -> Pending; + + /// Take down what [`Workload::mount_launch`] mounted, overlays first, so + /// the loop devices under it clear themselves. Lazily when busy. + fn unmount_launch(&mut self, images: Vec, overlays: Vec) -> Pending; + /// Start a process on a terminal and connect it to the host on a data port /// of its own. /// @@ -60,14 +74,22 @@ pub trait Workload { /// is made in the background. fn attach(&mut self, id: AttachId, exec: &Exec, size: Option) -> Result<(), Failure>; - /// Start the command the descriptor names. + /// Start one launch's command, named by its launch. /// /// Returning the exit as a future, rather than a `wait` method, is what /// keeps the caller free to read the channel while the workload runs — a /// stop has to arrive during the workload's life or it is not a stop. - fn start(&mut self, exec: &Exec) -> Result; + /// + /// Several launches may be running at once: a box is a person's machine + /// with any of their games in it, not one game's. ref(d-0085) + fn start(&mut self, id: &LaunchId, exec: &Exec) -> Result; - /// Ask the workload to stop. Idempotent, and never ends the session by + /// Ask one launch to stop. Idempotent, and never ends the session by + /// itself. A launch that is not running is not signalled: its pid may + /// already belong to something else. + fn stop_launch(&mut self, id: &LaunchId); + + /// Ask every launch to stop. Idempotent, and never ends the session by /// itself. fn signal_stop(&mut self); } @@ -75,7 +97,10 @@ pub trait Workload { /// The workload as a local process. pub struct Process { waiters: Waiters, - running: Option, + /// Every launch started, by id. An entry stays after its exit is + /// delivered and `Watched::running` is what tells the two apart; it is + /// replaced when the same id is started again (a restart). + running: std::collections::BTreeMap, } impl Process { @@ -84,7 +109,7 @@ impl Process { pub fn new(waiters: Waiters) -> Self { Self { waiters, - running: None, + running: std::collections::BTreeMap::new(), } } @@ -93,7 +118,12 @@ impl Process { /// Nothing here ever signals the process group or every process: the order /// a shutdown promises is only true if the workload can be stopped alone. pub fn signal(&self, signal: libc::c_int) { - let Some(watched) = &self.running else { return }; + for watched in self.running.values() { + Self::signal_one(watched, signal); + } + } + + fn signal_one(watched: &Watched, signal: libc::c_int) { // Nothing is signalled once the exit has been delivered. The pid was // freed by the reap that produced it, and the kernel is entitled to // give that number to something else — a stop or a kill aimed at it @@ -113,14 +143,17 @@ impl Process { /// one pid rather than for any child, so a slow service cannot be mistaken /// for the workload still running. pub fn await_exit(&self, grace: std::time::Duration) -> bool { - let Some(watched) = &self.running else { - return true; - }; + let deadline = std::time::Instant::now() + grace; + self.running + .values() + .all(|watched| Self::await_one(watched, deadline)) + } + + fn await_one(watched: &Watched, deadline: std::time::Instant) -> bool { if !watched.running() { return true; // already reaped, and its exit already reported } let pid = watched.pid; - let deadline = std::time::Instant::now() + grace; loop { let mut status: libc::c_int = 0; // SAFETY: waitpid writes only into `status`. @@ -188,7 +221,48 @@ impl Workload for Process { Ok(()) } - fn start(&mut self, exec: &Exec) -> Result { + fn mount_launch(&mut self, images: Vec, overlays: Vec) -> Pending { + off_loop(move || { + for image in &images { + mount_image(image)?; + } + for overlay in &overlays { + mount_overlay(overlay)?; + } + Ok(()) + }) + } + + fn unmount_launch(&mut self, images: Vec, overlays: Vec) -> Pending { + off_loop(move || { + // Everything is attempted, and the first failure is the one said. + let mut first = None; + let mut note = |result: Result<(), Failure>| { + if let Err(failure) = result { + first.get_or_insert(failure); + } + }; + for overlay in overlays.iter().rev() { + match OverlayPlan::new(overlay) { + Ok(plan) => { + for at in [&plan.at, &plan.rw_at, &plan.lower_at] { + note(unmount_one(at)); + } + } + Err(failure) => note(Err(failure)), + } + } + for image in images.iter().rev() { + match ImagePlan::new(image) { + Ok(plan) => note(unmount_one(&plan.at)), + Err(failure) => note(Err(failure)), + } + } + first.map_or(Ok(()), Err) + }) + } + + fn start(&mut self, id: &LaunchId, exec: &Exec) -> Result { let Some((program, args)) = exec.argv.split_first() else { return Err(Failure::new("the command is empty")); }; @@ -229,7 +303,7 @@ impl Workload for Process { None } }; - command.envs(environment(exec, runtime.as_deref())); + command.envs(launch_environment(exec, runtime.as_deref(), id)); // SAFETY: the closure runs between fork and exec in the child, where // only async-signal-safe calls are allowed. These two are, and it @@ -267,7 +341,10 @@ impl Workload for Process { // The caller gets the exit and reports it; this handle keeps the pid // and whether that pid is still this child's. let exit = watched.take_exit().expect("a new watch has its exit"); - self.running = Some(watched); + // A launch that ended leaves its entry behind until its id is used + // again; dropping those here keeps the map the size of what runs. + self.running.retain(|_, watched| watched.running()); + self.running.insert(id.clone(), watched); Ok(Box::pin(async move { exit.await @@ -275,11 +352,32 @@ impl Workload for Process { })) } + fn stop_launch(&mut self, id: &LaunchId) { + if let Some(watched) = self.running.get(id) { + Self::signal_one(watched, libc::SIGTERM); + } + } + fn signal_stop(&mut self) { self.signal(libc::SIGTERM); } } +/// A launch's environment: everything [`environment`] gives, then its id. +/// +/// The id goes last so nothing in the caller's environment can name a launch +/// other than the one this is. ref(d-0085) +pub(crate) fn launch_environment( + exec: &Exec, + runtime: Option<&str>, + id: &LaunchId, +) -> Vec<(String, String)> { + let mut env = environment(exec, runtime); + env.retain(|(k, _)| k != LAUNCH_ID_ENV); + env.push((LAUNCH_ID_ENV.to_string(), id.as_str().to_string())); + env +} + /// Everything a launch is started with, in the order that decides ties. /// /// The image's own graphics settings first and the caller's environment last, @@ -400,16 +498,20 @@ fn mount_overlay(overlay: &Overlay) -> Result<(), Failure> { // Everything that can be refused without touching the filesystem is // refused first, so a descriptor this cannot act on leaves nothing behind. let plan = OverlayPlan::new(overlay)?; + let lower = device_for(&plan.lower, true) + .map_err(|error| plan.failed("the build image", &plan.lower, error))?; + let upper = device_for(&plan.upper, false) + .map_err(|error| plan.failed("the writable layer", &plan.upper, error))?; mount_one( - &plan.lower, + &lower, &plan.lower_at, c"erofs", plan.lower_flags, None, ) .map_err(|error| plan.failed("the build image", &plan.lower_at, error))?; - mount_one(&plan.upper, &plan.rw_at, c"ext4", plan.upper_flags, None) + mount_one(&upper, &plan.rw_at, c"ext4", plan.upper_flags, None) .map_err(|error| plan.failed("the writable layer", &plan.rw_at, error))?; for dir in [&plan.upper_dir, &plan.work_dir] { @@ -485,8 +587,202 @@ impl ImagePlan { /// Mount one read-only image where the descriptor says. fn mount_image(image: &Image) -> Result<(), Failure> { let plan = ImagePlan::new(image)?; - mount_one(&plan.device, &plan.at, c"erofs", plan.flags, None) - .map_err(|error| plan.failed(error)) + let device = device_for(&plan.device, true).map_err(|error| plan.failed(error))?; + mount_one(&device, &plan.at, c"erofs", plan.flags, None).map_err(|error| plan.failed(error)) +} + +/// Run blocking filesystem work on the runtime's blocking pool. +fn off_loop(work: impl FnOnce() -> Result<(), Failure> + Send + 'static) -> Pending { + Box::pin(async move { + tokio::task::spawn_blocking(work) + .await + .unwrap_or_else(|error| Err(Failure::new(format!("the mount work did not finish: {error}")))) + }) +} + +/// Unmount one filesystem, lazily if it is still in use: a launch that has +/// exited can leave a child holding a file open, and the mount must still go +/// so the loop device under it is freed when that child does. +fn unmount_one(target: &std::ffi::CStr) -> Result<(), Failure> { + // SAFETY: a nul-terminated path that outlives the call. + if unsafe { libc::umount2(target.as_ptr(), 0) } == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + match error.raw_os_error() { + // Never mounted, or already gone: what was asked for. + Some(libc::EINVAL) | Some(libc::ENOENT) => Ok(()), + Some(libc::EBUSY) => { + // SAFETY: as above. + if unsafe { libc::umount2(target.as_ptr(), libc::MNT_DETACH) } == 0 { + Ok(()) + } else { + Err(Failure::new(format!( + "{}: {}", + target.to_string_lossy(), + io::Error::last_os_error() + ))) + } + } + _ => Err(Failure::new(format!("{}: {error}", target.to_string_lossy()))), + } +} + +/// The block device to mount `source` from. +/// +/// A block device is used as it is, which is every mount made at boot. A +/// regular file is attached to a free loop device first: that is how content +/// reaches a box that is already running, as a file in a share, because a +/// box cannot be given a new block device after it has booted. ref(d-0085) +/// +/// The loop device clears itself when the filesystem on it is unmounted, so +/// nothing here has to remember it. +/// +/// A symbolic link anywhere in the path is refused rather than followed: a +/// file in a share is the host's to name, and a link would let whatever wrote +/// it name another. Anything that is neither a block device nor a regular file +/// is refused too. +fn device_for(source: &std::ffi::CStr, read_only: bool) -> io::Result { + use std::os::unix::fs::FileTypeExt; + let probe = open_no_symlinks(source, libc::O_RDONLY)?; + let kind = probe.metadata()?.file_type(); + if kind.is_block_device() { + return Ok(source.to_owned()); + } + if !kind.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a mount source has to be a block device or a regular file", + )); + } + drop(probe); + loop_attach(source, read_only) +} + +/// `struct open_how` from ``. +#[repr(C)] +struct OpenHow { + flags: u64, + mode: u64, + resolve: u64, +} + +const RESOLVE_NO_SYMLINKS: u64 = 0x04; + +/// Open a path, refusing a symbolic link at any component (`openat2`), not +/// only the last one. +fn open_no_symlinks(path: &std::ffi::CStr, flags: libc::c_int) -> io::Result { + use std::os::fd::FromRawFd; + let how = OpenHow { + flags: (flags | libc::O_CLOEXEC | libc::O_NONBLOCK) as u64, + mode: 0, + resolve: RESOLVE_NO_SYMLINKS, + }; + // SAFETY: a nul-terminated path and an `open_how` that outlive the call. + let fd = unsafe { + libc::syscall( + libc::SYS_openat2, + libc::AT_FDCWD, + path.as_ptr(), + &how as *const OpenHow, + std::mem::size_of::(), + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: a descriptor the kernel just returned, owned by nothing else. + Ok(unsafe { std::fs::File::from_raw_fd(fd as i32) }) +} + +/// `struct loop_info64` from ``. +#[repr(C)] +struct LoopInfo64 { + device: u64, + inode: u64, + rdevice: u64, + offset: u64, + sizelimit: u64, + number: u32, + encrypt_type: u32, + encrypt_key_size: u32, + flags: u32, + file_name: [u8; 64], + crypt_name: [u8; 64], + encrypt_key: [u8; 32], + init: [u64; 2], +} + +/// `struct loop_config` from ``. +#[repr(C)] +struct LoopConfig { + fd: u32, + block_size: u32, + info: LoopInfo64, + reserved: [u64; 8], +} + +const LOOP_CTL_GET_FREE: libc::c_ulong = 0x4C82; +const LOOP_CONFIGURE: libc::c_ulong = 0x4C0A; +const LO_FLAGS_READ_ONLY: u32 = 1; +const LO_FLAGS_AUTOCLEAR: u32 = 4; + +/// Attach a file to a free loop device and name the device. +/// +/// Retried a few times on `EBUSY`: a device handed out as free can be taken +/// by somebody else before it is configured. +fn loop_attach(file: &std::ffi::CStr, read_only: bool) -> io::Result { + use std::os::fd::AsRawFd; + + // Opened again, with the same refusal, because the file may have been + // swapped since it was looked at. + let backing = open_no_symlinks(file, if read_only { libc::O_RDONLY } else { libc::O_RDWR })?; + if !backing.metadata()?.is_file() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "not a regular file")); + } + let control = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/loop-control")?; + + let mut last = io::Error::from_raw_os_error(libc::EBUSY); + for _ in 0..8 { + // SAFETY: an ioctl with no argument on a descriptor this owns. + let number = unsafe { libc::ioctl(control.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; + if number < 0 { + return Err(io::Error::last_os_error()); + } + let path = format!("/dev/loop{number}"); + let device = std::fs::OpenOptions::new() + .read(true) + .write(!read_only) + .open(&path)?; + let mut flags = LO_FLAGS_AUTOCLEAR; + if read_only { + flags |= LO_FLAGS_READ_ONLY; + } + // SAFETY: zero is a valid value for every field of this plain C struct. + let mut config: LoopConfig = unsafe { std::mem::zeroed() }; + config.fd = backing.as_raw_fd() as u32; + config.info.flags = flags; + // SAFETY: the kernel reads `config` and nothing else, and it outlives + // the call. + let done = unsafe { + libc::ioctl( + device.as_raw_fd(), + LOOP_CONFIGURE as _, + &config as *const LoopConfig, + ) + }; + if done == 0 { + return CString::new(path).map_err(io::Error::other); + } + last = io::Error::last_os_error(); + if last.raw_os_error() != Some(libc::EBUSY) { + return Err(last); + } + } + Err(last) } /// Mount one filesystem, creating its mount point first. @@ -684,6 +980,75 @@ fn failed(share: &Mount, error: io::Error) -> Failure { Failure::new(format!("{}: {error}", share.at)) } +#[cfg(test)] +mod loop_tests { + use super::*; + + /// The layouts the kernel reads, checked against ``. + #[test] + fn the_loop_structs_have_the_kernels_layout() { + assert_eq!(std::mem::size_of::(), 232); + assert_eq!(std::mem::size_of::(), 304); + assert_eq!(std::mem::offset_of!(LoopConfig, info), 8); + } + + /// Only a block device (every mount made at boot) or a regular file is a + /// source. A character device, a directory and the like are refused. + #[test] + fn a_source_that_is_neither_a_block_device_nor_a_file_is_refused() { + for path in [c"/dev/null", c"/tmp"] { + let error = device_for(path, true).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput, "{path:?}: {error}"); + } + } + + /// A link is refused at the last component and at any earlier one. + #[test] + fn a_source_reached_through_a_symbolic_link_is_refused() { + let dir = std::env::temp_dir().join(format!("nesinit-link-{}", std::process::id())); + let real = dir.join("real"); + std::fs::create_dir_all(&real).unwrap(); + std::fs::write(real.join("build.erofs"), b"x").unwrap(); + std::os::unix::fs::symlink(real.join("build.erofs"), dir.join("file-link")).unwrap(); + std::os::unix::fs::symlink(&real, dir.join("dir-link")).unwrap(); + for path in [dir.join("file-link"), dir.join("dir-link").join("build.erofs")] { + let source = CString::new(path.to_string_lossy().as_bytes()).unwrap(); + let error = device_for(&source, true).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(libc::ELOOP), "{path:?}: {error}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn unmounting_what_is_not_mounted_is_not_a_failure() { + assert_eq!(unmount_one(c"/nonexistent/nesinit-test"), Ok(())); + } + + #[test] + fn a_source_that_is_not_there_names_the_error() { + let error = device_for(c"/nonexistent/build.erofs", true).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + /// The launch id is in the environment, and the caller cannot name + /// another launch in its place. + #[test] + fn a_launch_is_told_its_own_id() { + let mut exec = Exec { + argv: vec!["/bin/true".into()], + env: Default::default(), + cwd: None, + uid: 1001, + gid: 1001, + }; + exec.env.insert(LAUNCH_ID_ENV.into(), "launch_other".into()); + let env = launch_environment(&exec, None, &LaunchId::new("launch_01")); + let ids: Vec<_> = env.iter().filter(|(k, _)| k == LAUNCH_ID_ENV).collect(); + assert_eq!(ids.len(), 1, "{env:?}"); + assert_eq!(ids[0].1, "launch_01"); + } +} + #[cfg(test)] mod tests { use super::*; @@ -988,12 +1353,25 @@ pub mod double { pub attached: Vec<(AttachId, Exec, Option)>, pub attach_failure: Option, pub started: Vec, + /// An exit of its own for a launch, over `exit`. + pub exit_for: std::collections::BTreeMap, + /// Launches whose start fails after the first, for a restart test. + pub fail_restart_of: Option, + /// What `unmount_launch` was asked to take down, in order. + pub unmounted: Vec<(Vec, Vec)>, + /// Which launch each start was for, in the same order as `started`. + pub started_ids: Vec, + /// Every stop asked for, one launch or all (`None`). + pub stopped: Vec>, pub stops: usize, pub mount_failure: Option, pub image_failure: Option, + /// Like `image_failure`, but only for a call that names an image, so a + /// box can boot with none and fail a later mount. + pub named_image_failure: Option, pub start_failure: Option, exit: Exit, - on_stop: Option>, + on_stop: std::collections::BTreeMap>, holds_until_stopped: bool, } @@ -1017,12 +1395,18 @@ pub mod double { attached: Vec::new(), attach_failure: None, started: Vec::new(), + unmounted: Vec::new(), + exit_for: Default::default(), + fail_restart_of: None, + started_ids: Vec::new(), + stopped: Vec::new(), stops: 0, mount_failure: None, image_failure: None, + named_image_failure: None, start_failure: None, exit, - on_stop: None, + on_stop: Default::default(), holds_until_stopped, } } @@ -1054,12 +1438,27 @@ pub mod double { fn mount_images(&mut self, images: &[Image]) -> Result<(), Failure> { self.order.push("images"); self.images.push(images.to_vec()); + if let Some(failure) = self.named_image_failure.as_ref().filter(|_| !images.is_empty()) { + return Err(failure.clone()); + } match &self.image_failure { Some(failure) => Err(failure.clone()), None => Ok(()), } } + fn mount_launch(&mut self, images: Vec, overlays: Vec) -> Pending { + let result = self + .mount_images(&images) + .and_then(|()| self.mount_overlays(&overlays)); + Box::pin(async move { result }) + } + + fn unmount_launch(&mut self, images: Vec, overlays: Vec) -> Pending { + self.unmounted.push((images, overlays)); + Box::pin(async { Ok(()) }) + } + fn mount_overlays(&mut self, overlays: &[Overlay]) -> Result<(), Failure> { self.order.push("overlays"); self.overlays.push(overlays.to_vec()); @@ -1069,27 +1468,41 @@ pub mod double { } } - fn start(&mut self, exec: &Exec) -> Result { + fn start(&mut self, id: &LaunchId, exec: &Exec) -> Result { self.started.push(exec.clone()); + let again = self.started_ids.contains(id); + self.started_ids.push(id.clone()); if let Some(failure) = &self.start_failure { return Err(failure.clone()); } - let exit = self.exit; + if again && self.fail_restart_of.as_ref() == Some(id) { + return Err(Failure::new("it would not come back")); + } + let exit = self.exit_for.get(id).copied().unwrap_or(self.exit); if !self.holds_until_stopped { return Ok(Box::pin(async move { Ok(exit) })); } let (tx, rx) = oneshot::channel(); - self.on_stop = Some(tx); + self.on_stop.insert(id.clone(), tx); Ok(Box::pin(async move { rx.await .map_err(|_| io::Error::other("the workload was dropped")) })) } + fn stop_launch(&mut self, id: &LaunchId) { + self.stops += 1; + self.stopped.push(Some(id.clone())); + if let Some(tx) = self.on_stop.remove(id) { + let _ = tx.send(self.exit_for.get(id).copied().unwrap_or(self.exit)); + } + } + fn signal_stop(&mut self) { self.stops += 1; - if let Some(tx) = self.on_stop.take() { - let _ = tx.send(self.exit); + self.stopped.push(None); + for (id, tx) in std::mem::take(&mut self.on_stop) { + let _ = tx.send(self.exit_for.get(&id).copied().unwrap_or(self.exit)); } } } diff --git a/build/kernel/nestri.fragment b/build/kernel/nestri.fragment index 0065e818..64a468b4 100644 --- a/build/kernel/nestri.fragment +++ b/build/kernel/nestri.fragment @@ -129,6 +129,8 @@ CONFIG_EXT4_FS=y # output/rootfs.ext4, and a box's writable install CONFIG_MISC_FILESYSTEMS=y # the menu EROFS lives under; off in base.config, which drops it silently CONFIG_EROFS_FS=y # a game's build image, the install overlay's lower layer CONFIG_OVERLAY_FS=y # the session overlay over a read-only install +CONFIG_BLK_DEV_LOOP=y # a game mounted into a running box arrives as a file in a share, ref(d-0085) +CONFIG_BLK_DEV_LOOP_MIN_COUNT=0 # devices are made on demand through /dev/loop-control # ── Memory typing ──────────────────────────────────────── # Page Attribute Table, and the MTRR support it depends on. diff --git a/crates/nesprotocol/src/lifecycle.rs b/crates/nesprotocol/src/lifecycle.rs index b55c3510..e88a2aa2 100644 --- a/crates/nesprotocol/src/lifecycle.rs +++ b/crates/nesprotocol/src/lifecycle.rs @@ -80,6 +80,15 @@ pub const ATTACH_PORT: u32 = 7001; /// named at the handshake and not found as a launch that cannot start. pub const CONTROL_VERSION: u32 = 5; +/// The variable a launched process finds its launch id in. +/// +/// Set by the guest on every launch, over anything the caller put in the +/// launch's own environment, because the id on the message is the one every +/// report about the launch carries. What reads it: the compositor, to say +/// which launch a window belongs to (`/proc//environ` of the window's +/// owner), and so a client can find its own window. ref(d-0085) +pub const LAUNCH_ID_ENV: &str = "NESTRI_LAUNCH_ID"; + /// The command to run, and who runs it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Exec { @@ -397,6 +406,15 @@ pub enum GuestToHost { #[serde(flatten)] exit: Exit, }, + /// Everything a [`HostToGuest::Mount`] for this launch named is in place. + /// + /// Answers that command and nothing else: the boot descriptor's own mounts + /// are still answered by [`GuestToHost::Mounted`], once. + LaunchMounted { id: LaunchId }, + /// A [`HostToGuest::Mount`] for this launch could not be carried out, in + /// the operating system's words. The box stays up: a game whose image will + /// not mount is one launch that cannot start, not a box that cannot work. + LaunchMountFailed { id: LaunchId, reason: String }, /// How a client reaches this box's media, once it is known. Ticket { ticket: String }, /// How many clients are watching this box. Sent once when first known and @@ -430,6 +448,27 @@ pub enum HostToGuest { exec: Exec, on_exit: OnExit, }, + /// Mount content for a launch into a box that is already running. + /// + /// How a second game reaches a box: the host already holds its build, and + /// the box mounts it on request rather than at boot. ref(d-0085) The same + /// shapes as the descriptor's, carried out in the same order (images, then + /// overlays), and answered with [`GuestToHost::LaunchMounted`] or + /// [`GuestToHost::LaunchMountFailed`] for this id. + /// + /// A device here may also be a regular file inside a share, which the + /// guest attaches to a loop device first: a box cannot be given a new + /// block device once it has booted. + /// + /// Added without a version bump, as an addition: a guest that predates it + /// skips the line, and the host's wait for the answer is what notices. + Mount { + id: LaunchId, + #[serde(default)] + images: Vec, + #[serde(default)] + overlays: Vec, + }, /// Stop one launch. Idempotent, and does not end the session. Stop { id: LaunchId }, /// Stop one launch and start it again with the same command. @@ -551,6 +590,47 @@ mod tests { } } + /// A mount for a running box is named by its launch on the wire, both ways, + /// and an empty list may be left out. + #[test] + fn a_mount_into_a_running_box_round_trips_by_launch() { + let mount = HostToGuest::Mount { + id: LaunchId::new("launch_01"), + images: Vec::new(), + overlays: vec![Overlay { + lower: "/nestri/launch-images/launch_01/build.erofs".into(), + upper: "/nestri/launches/launch_01/upper.ext4".into(), + at: "/nestri/installs/launch_01".into(), + }], + }; + let line = to_line(&mount).unwrap(); + assert!(line.contains(r#""type":"mount""#), "{line}"); + assert_eq!(from_line::(&line).unwrap(), mount); + + let bare: HostToGuest = from_line(r#"{"type":"mount","id":"launch_02"}"#).unwrap(); + assert_eq!( + bare, + HostToGuest::Mount { + id: LaunchId::new("launch_02"), + images: Vec::new(), + overlays: Vec::new(), + } + ); + + let failed = GuestToHost::LaunchMountFailed { + id: LaunchId::new("launch_01"), + reason: "ENOENT".into(), + }; + let line = to_line(&failed).unwrap(); + assert!(line.contains(r#""type":"launch_mount_failed""#), "{line}"); + assert_eq!(from_line::(&line).unwrap(), failed); + let line = to_line(&GuestToHost::LaunchMounted { + id: LaunchId::new("launch_01"), + }) + .unwrap(); + assert!(line.contains(r#""type":"launch_mounted""#), "{line}"); + } + #[test] fn a_line_round_trips() { let line = to_line(&HostToGuest::Boot {