diff --git a/apps/nesinit/src/filesystems.rs b/apps/nesinit/src/filesystems.rs new file mode 100644 index 00000000..8c2768a8 --- /dev/null +++ b/apps/nesinit/src/filesystems.rs @@ -0,0 +1,215 @@ +// The filesystems PID 1 has to establish before anything asks for them. +// +// The root arrives read-only — the host attaches it that way and nothing in +// the guest may write to it — and there is no init system behind this process +// to make up the difference. So a guest gets `/proc`, and it gets somewhere to +// put a socket, only if this mounts them. +// +// Without that the failure is not an error anyone sees. Everything that needs +// a writable path fails one layer down, separately, as `EROFS` on a socket: +// the payload relay never binds, whatever serves the session's address never +// binds either, and the session is reported as a workload that ran and +// published nothing. Three unrelated-looking symptoms, one missing mount. + +use std::ffi::CString; +use std::path::Path; + +/// A filesystem this process mounts, and why it has to exist. +struct Early { + /// What appears in `/proc/mounts` as the source. Conventionally the type. + source: &'static str, + target: &'static str, + fstype: &'static str, + flags: libc::c_ulong, + /// Mount options, or empty for none. + data: &'static str, + /// Said when it could not be mounted, in terms of what stops working. + cost: &'static str, +} + +/// `nosuid` and `nodev` on everything: none of these carry an image's files, +/// so a device node or a setuid bit appearing in one did not come from us. +const NOSUID_NODEV: libc::c_ulong = libc::MS_NOSUID | libc::MS_NODEV; + +const EARLY: &[Early] = &[ + Early { + source: "proc", + target: "/proc", + fstype: "proc", + flags: NOSUID_NODEV | libc::MS_NOEXEC, + data: "", + cost: "this process cannot make itself ineligible for the OOM killer, \ + and nothing in the guest can read its own state", + }, + Early { + source: "sysfs", + target: "/sys", + fstype: "sysfs", + flags: NOSUID_NODEV | libc::MS_NOEXEC, + data: "", + cost: "a workload that looks up a device finds nothing", + }, + // Writable, and the reason any of this is here. Both sockets in this + // component live on a tmpfs because the root they would otherwise sit on + // is read-only. + Early { + source: "tmpfs", + target: "/tmp", + fstype: "tmpfs", + flags: NOSUID_NODEV, + // The sticky bit, because the workload does not run as this process + // does and what it binds here is its own. + data: "mode=1777", + cost: "whatever serves this session's address cannot bind its socket, \ + so the session never gets one", + }, + Early { + source: "tmpfs", + target: crate::payload::DIRECTORY, + fstype: "tmpfs", + flags: NOSUID_NODEV | libc::MS_NOEXEC, + // Only this process and the workload it starts, and they are the only + // two that ever have business here. + data: "mode=0770", + cost: "the payload relay cannot bind, so nothing reaches the workload \ + over the channel", + }, +]; + +/// Mount what the rest of this component assumes is already there. +/// +/// Best effort, one line per failure. Refusing to boot over any of these would +/// replace a session that fails with a reason with a guest that never dialled +/// out at all, and the second is strictly harder to diagnose from the host. +pub fn establish() { + // `/proc` first and unconditionally: it is the only way to find out what is + // already mounted, so everything after it can be skipped when an image has + // done it already, and it cannot itself be checked that way. + mount(&EARLY[0]); + + let existing = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default(); + for early in &EARLY[1..] { + if mounted_at(&existing, early.target) { + tracing::debug!(target = early.target, "already mounted by the image"); + continue; + } + mount(early); + } +} + +/// Whether `mountinfo` already has a mount at this exact path. +/// +/// The mount point is the fifth field and it is the one that has to match: +/// a prefix test would read `/tmpfoo` as `/tmp`, and a substring test would +/// find the path in the options of something else entirely. +fn mounted_at(mountinfo: &str, target: &str) -> bool { + mountinfo + .lines() + .filter_map(|line| line.split_whitespace().nth(4)) + .any(|point| point == target) +} + +fn mount(early: &Early) { + // A mount point that is not in the image cannot be created on a read-only + // root, so this is allowed to fail and the mount below reports it. + if !Path::new(early.target).exists() { + let _ = std::fs::create_dir_all(early.target); + } + + let (Ok(source), Ok(target), Ok(fstype), Ok(data)) = ( + CString::new(early.source), + CString::new(early.target), + CString::new(early.fstype), + CString::new(early.data), + ) else { + // Every one of these is a literal in this file, so this is + // unreachable rather than a case to handle. + tracing::error!(target = early.target, "a mount table entry has a nul byte"); + return; + }; + let data = if early.data.is_empty() { + std::ptr::null() + } else { + data.as_ptr().cast() + }; + + // SAFETY: four pointers that outlive the call, and a flag word. + let mounted = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + fstype.as_ptr(), + early.flags, + data, + ) + }; + if mounted != 0 { + tracing::warn!( + target = early.target, + error = %std::io::Error::last_os_error(), + cost = early.cost, + "could not mount" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The whole point of reading `mountinfo` is to not mount twice over + /// something an image already did. + #[test] + fn a_mount_point_that_is_present_is_recognised() { + let info = "\ +23 1 0:5 / /proc rw,nosuid,nodev,noexec - proc proc rw +24 1 0:6 / /tmp rw,nosuid,nodev - tmpfs tmpfs rw,mode=1777"; + assert!(mounted_at(info, "/proc")); + assert!(mounted_at(info, "/tmp")); + } + + /// A prefix is not a mount point, and neither is a path that only appears + /// in another line's options. + #[test] + fn something_else_is_not_mistaken_for_a_mount_point() { + let info = "\ +23 1 0:5 / /tmpfoo rw - tmpfs tmpfs rw +24 1 0:6 / /var rw - ext4 /dev/vda rw,journal_path=/nestri"; + assert!(!mounted_at(info, "/tmp")); + assert!(!mounted_at(info, "/nestri")); + } + + /// Every entry has to be nul-free, because `establish` treats a nul as + /// unreachable rather than handling it. + #[test] + fn the_mount_table_can_be_carried_out() { + for early in EARLY { + assert!(CString::new(early.source).is_ok(), "{}", early.target); + assert!(CString::new(early.target).is_ok(), "{}", early.target); + assert!(CString::new(early.fstype).is_ok(), "{}", early.target); + assert!(CString::new(early.data).is_ok(), "{}", early.target); + assert!(!early.cost.is_empty(), "{} has no cost", early.target); + } + } + + /// `/proc` is mounted before `mountinfo` is read, so it has to be first. + #[test] + fn proc_is_the_first_entry() { + assert_eq!(EARLY[0].target, "/proc"); + } + + /// The relay's directory is the one this cannot hardcode: it belongs to + /// `payload`, and a rename there that missed this file would take the + /// relay down again in exactly the way this exists to prevent. + #[test] + fn the_relay_directory_is_the_one_the_relay_uses() { + let entry = EARLY + .iter() + .find(|e| e.target == crate::payload::DIRECTORY) + .expect("the relay's directory is mounted"); + assert!( + crate::payload::SOCKET.starts_with(entry.target), + "the relay's socket is not under the directory that is mounted for it" + ); + } +} diff --git a/apps/nesinit/src/lib.rs b/apps/nesinit/src/lib.rs index 1f2b3588..32df7edd 100644 --- a/apps/nesinit/src/lib.rs +++ b/apps/nesinit/src/lib.rs @@ -8,6 +8,7 @@ // and what an exit means, and it carries that out; a field that only makes // sense for one kind of workload cannot reach it. ref(d-0033) +pub mod filesystems; pub mod payload; pub mod reap; pub mod session; diff --git a/apps/nesinit/src/main.rs b/apps/nesinit/src/main.rs index aa157c6a..239be9b7 100644 --- a/apps/nesinit/src/main.rs +++ b/apps/nesinit/src/main.rs @@ -40,6 +40,11 @@ fn main() -> anyhow::Result<()> { ) .init(); + // Before everything, including the two below: the root is read-only and + // nothing else in this guest is an init system, so until this runs there + // is no `/proc` to score this process in and nowhere to put a socket. + nesinit::filesystems::establish(); + // Both before anything is started, so nothing can be orphaned or scored // in the window where neither is true yet. if let Err(error) = reap::become_subreaper() { diff --git a/apps/nesinit/src/payload.rs b/apps/nesinit/src/payload.rs index 96b3a4ad..944bcbca 100644 --- a/apps/nesinit/src/payload.rs +++ b/apps/nesinit/src/payload.rs @@ -16,6 +16,14 @@ use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::{UnixListener, UnixStream}; use tokio::sync::mpsc::{Receiver, Sender}; +/// The directory the relay's socket lives in. +/// +/// Named separately because it is mounted before it is used: the guest's root +/// is read-only, so this is a tmpfs that `filesystems` puts there, and a +/// rename here that did not reach the mount table would take the relay down +/// with an `EROFS` that looks like nothing to do with a path. +pub const DIRECTORY: &str = "/nestri"; + /// Where the workload finds the relay. pub const SOCKET: &str = "/nestri/payload.sock";