diff --git a/apps/api/app/routes/enrolment.ts b/apps/api/app/routes/enrolment.ts index 39828205..1391e62d 100644 --- a/apps/api/app/routes/enrolment.ts +++ b/apps/api/app/routes/enrolment.ts @@ -1,6 +1,9 @@ import { Actor } from '@nestri/core/actor'; +import { Env } from '@nestri/core/env'; import { ErrorCodes, VisibleError } from '@nestri/core/error'; +import { Machine } from '@nestri/core/machine/index'; import { Enrolment } from '@nestri/core/steam/enrolment'; +import { Identity } from '@nestri/core/user/identity'; import { Hono } from 'hono'; import { describeRoute } from 'hono-openapi'; import { z } from 'zod'; @@ -27,6 +30,39 @@ import { ErrorResponses, machineOnly, Result, validator } from '../utils'; * ask, and none of them re-ask it. */ export namespace EnrolmentApi { + /** + * A sign-in on one of Nestri's own machines is also proof of the Steam + * account, so it links it: the person scanned a QR code on hardware we + * run, the Steam id is the subject of the token Valve issued there, and + * who they are came from the edge. Asking them to sign in to Steam a + * second time, in a browser, would prove nothing new. + * + * **Only the fleet.** On a host somebody brought, the report is that + * person's machine vouching for itself, and a machine can say anything. + * Linking from it would let any host operator attach any Steam account to + * any user enrolled on it. + * + * Never a reason to refuse the report. The sign-in happened and the + * enrolment is recorded either way; a Steam account already linked to + * somebody else stays theirs, and is left for the person to sort out on + * the website. + */ + async function linkIfFleet(machineId: string, userId: string, steamId: string) { + const fleet = Env.get().FLEET_ORGANISATION_ID; + if (!fleet) return; + const machine = await Machine.fromID(machineId); + if (machine?.organisationId !== fleet) return; + try { + await Identity.linkSteam({ userId, steamId }); + } catch (err) { + if (err instanceof VisibleError) { + console.warn('a fleet sign-in was not linked', { machineId, userId, reason: err.message }); + return; + } + throw err; + } + } + // Picked from the domain schema rather than restated, so the shape a host // must send and the shape the record has cannot drift apart — including the // Steam id's format, which is checked here at the boundary and therefore @@ -61,13 +97,13 @@ export namespace EnrolmentApi { validator('json', Reported), async (c) => { const body = c.req.valid('json'); - return c.json({ - data: await Enrolment.record({ - machineId: Actor.machineID, - userId: body.userId, - steamId: body.steamId - }) + const enrolment = await Enrolment.record({ + machineId: Actor.machineID, + userId: body.userId, + steamId: body.steamId }); + await linkIfFleet(Actor.machineID, body.userId, body.steamId); + return c.json({ data: enrolment }); } ) .post( diff --git a/apps/api/install/install.sh b/apps/api/install/install.sh index cbb1ce9c..93fa3067 100755 --- a/apps/api/install/install.sh +++ b/apps/api/install/install.sh @@ -21,7 +21,7 @@ set -eu API="${NESTRI_API:-https://api.nestri.io}" # Pinned, not "latest", so the script and the binary it installs are a pair # somebody chose. Bump when cutting a release; NESTRI_HOST_VERSION overrides it. -DEFAULT_VERSION="0.2.2" +DEFAULT_VERSION="0.2.3" VERSION="${NESTRI_HOST_VERSION:-$DEFAULT_VERSION}" BIN_DIR="${NESTRI_BIN_DIR:-$HOME/.local/bin}" diff --git a/apps/api/test/enrolment.test.ts b/apps/api/test/enrolment.test.ts index 4e228aa7..9331c35f 100644 --- a/apps/api/test/enrolment.test.ts +++ b/apps/api/test/enrolment.test.ts @@ -4,14 +4,16 @@ import { AccessToken } from '@nestri/core/access-token/index'; import { Fixtures } from '@nestri/core/db/fixtures'; import { testDb } from '@nestri/core/db/test'; import { Identifier } from '@nestri/core/id'; +import { Env } from '@nestri/core/env'; import { Machine } from '@nestri/core/machine/index'; import { app } from '../app/index'; -import './setup'; +import { TEST_FRONTEND_URL } from './setup'; const sql = testDb(); const createdUserIds: string[] = []; +const createdOrgIds: string[] = []; /** * A signed-in person, as a personal access token. @@ -78,6 +80,10 @@ function list(host: { headers: Record }) { } afterAll(async () => { + if (createdOrgIds.length > 0) { + await sql`delete from machine where organisation_id = any(${createdOrgIds})`; + await sql`delete from organisation where id = any(${createdOrgIds})`; + } if (createdUserIds.length > 0) { await sql`delete from "user" where id in ${sql(createdUserIds)}`; createdUserIds.length = 0; @@ -386,3 +392,66 @@ describe('The enrolment surface refuses a token', () => { expect([...accepted].sort()).toEqual(['steamId', 'userId']); }); }); + +describe('A sign-in on the fleet links the Steam account', () => { + /** A fleet host the API knows is the fleet, and a person to enrol on it. */ + async function fleetHost(label: string) { + const operator = await Fixtures.owner(`${label}-operator`); + createdUserIds.push(operator.userId); + const f = await Fixtures.fleet(operator); + createdOrgIds.push(f.organisationId); + Env.init({ + NODE_ENV: 'test', + FRONTEND_URL: TEST_FRONTEND_URL, + FLEET_ORGANISATION_ID: f.organisationId + }); + const person = await Fixtures.owner(label); + createdUserIds.push(person.userId); + const [machine] = f.machines; + return { + person, + headers: { + 'x-nestri-machine-id': machine!.id, + 'x-nestri-machine-secret': machine!.secret, + 'content-type': 'application/json' + } + }; + } + + const linked = (steam: string) => + sql<{ user_id: string }[]>` + select user_id from linked_account + where provider = 'steam' and provider_account_id = ${steam} and time_deleted is null`; + + test('a fleet host reporting a sign-in links that account to that person', async () => { + const host = await fleetHost('fleet-link'); + const res = await enrol(host, { userId: host.person.userId, steamId: steamId(9001) }); + expect(res.status).toBe(200); + expect((await linked(steamId(9001))).map((r) => r.user_id)).toEqual([host.person.userId]); + + // Restated, as a host does after a restart: still one link. + expect((await enrol(host, { userId: host.person.userId, steamId: steamId(9001) })).status).toBe( + 200 + ); + expect(await linked(steamId(9001))).toHaveLength(1); + }); + + test('a host somebody brought cannot link anything', async () => { + const host = await registeredHost('own-host-link'); + const res = await enrol(host, { userId: host.userId, steamId: steamId(9002) }); + expect(res.status).toBe(200); + expect(await linked(steamId(9002))).toHaveLength(0); + }); + + test('an account already linked elsewhere stays there, and the report still lands', async () => { + const host = await fleetHost('fleet-link-taken'); + const first = await Fixtures.owner('first-holder'); + createdUserIds.push(first.userId); + await sql`insert into linked_account (id, user_id, provider, provider_account_id) + values (${Identifier.ascending('linkedAccount')}, ${first.userId}, 'steam', ${steamId(9003)})`; + + const res = await enrol(host, { userId: host.person.userId, steamId: steamId(9003) }); + expect(res.status).toBe(200); + expect((await linked(steamId(9003))).map((r) => r.user_id)).toEqual([first.userId]); + }); +}); diff --git a/apps/api/test/session.test.ts b/apps/api/test/session.test.ts index 6d9bc5bd..21c10987 100644 --- a/apps/api/test/session.test.ts +++ b/apps/api/test/session.test.ts @@ -428,7 +428,8 @@ describe('GET /machine/jobs', () => { boxTier: 'sm', gameId: s.gameId, steamAppId: 5520, - linkedAccountId: s.owner.linkedAccountId + linkedAccountId: s.owner.linkedAccountId, + userId: s.owner.userId }); }); diff --git a/apps/neshub/src/main.rs b/apps/neshub/src/main.rs index 1e678c17..54e9594f 100644 --- a/apps/neshub/src/main.rs +++ b/apps/neshub/src/main.rs @@ -4,6 +4,7 @@ mod dgram; mod e2e_tests; mod ipc_listener; mod keyframe; +mod negotiation; mod pyro; mod screenshot; mod session; @@ -486,6 +487,8 @@ async fn main() -> Result<()> { ) .await; let mgr_clone = mgr.clone(); + let controller = controller.clone(); + let cmd_tx = cmd_tx.clone(); tokio::spawn(async move { conn.closed().await; // Any one of them going means the session goes. A client @@ -498,7 +501,9 @@ async fn main() -> Result<()> { carrier = carrier.label(), "carrier closed, ending session" ); - mgr_clone.remove_session(&remote_id).await; + mgr_clone + .remove_session(&remote_id, &controller, &cmd_tx) + .await; }); } Err(e) => { diff --git a/apps/neshub/src/negotiation.rs b/apps/neshub/src/negotiation.rs new file mode 100644 index 00000000..7adbbbef --- /dev/null +++ b/apps/neshub/src/negotiation.rs @@ -0,0 +1,269 @@ +//! Which codecs the encoder may pick, given everyone watching. +//! +//! There is one encoder and every client receives its output, so the codec it +//! picks has to be one that every client can decode. Each client states what it +//! decodes when it connects; forwarding each statement as it arrived let the +//! newest client decide, so a client that only decodes H.264 went black the +//! moment one that decodes AV1 joined. The hub is the only place that sees all +//! of them, so it keeps the statements and hands the encoder their +//! intersection instead -- and recomputes it when a client leaves, so the room +//! moves to a better codec once the client holding it back is gone. + +use std::collections::HashMap; +use std::hash::Hash; + +use nesprotocol::{CODEC_PYROWAVE, ClientCaps, ControlMode, DEPTH_8, DEPTH_10}; +use tokio::sync::Mutex; +use tracing::{info, warn}; + +use crate::control::Controller; + +/// Every client's stated capabilities, and the joint set last given to the +/// encoder. +pub struct Agreement { + stated: HashMap, + sent: Option, +} + +impl Default for Agreement { + fn default() -> Self { + Self { + stated: HashMap::new(), + sent: None, + } + } +} + +impl Agreement { + /// Record what `client` decodes, replacing anything it said before. + pub fn state(&mut self, client: K, caps: ClientCaps) { + self.stated.insert(client, caps); + } + + /// Drop a client that has left. + /// + /// Once nobody is left, the next client to arrive is told about from + /// scratch rather than compared with a room that no longer exists. + pub fn forget(&mut self, client: &K) { + self.stated.remove(client); + if self.stated.is_empty() { + self.sent = None; + } + } + + /// What every client that has spoken can decode. `None` when none has. + /// + /// A client that never states its capabilities is left out rather than + /// read as decoding nothing, which is how the encoder already treats one + /// on its own: it has no opinion, and an empty set would erase everyone + /// else's. + pub fn joint(&self) -> Option { + self.stated + .values() + .copied() + .reduce(|a, b| ClientCaps::from_bits(a.bits() & b.bits())) + } + + /// The joint set, if the encoder has not been given it yet. + pub fn pending(&self) -> Option { + self.joint().filter(|joint| Some(*joint) != self.sent) + } + + /// Note that the encoder now has `caps`. + pub fn mark_sent(&mut self, caps: ClientCaps) { + self.sent = Some(caps); + } +} + +/// A client's capabilities as this hub should count them. +/// +/// A client that decodes some other PyroWave bitstream format cannot read this +/// hub's, so its PyroWave bits are dropped rather than letting it vouch for a +/// stream it would show as garbage. +pub fn effective_caps(caps: ClientCaps, pyrowave_format: Option) -> ClientCaps { + if pyrowave_format == Some(nesprotocol::pyrowave::PYROWAVE_FORMAT) { + return caps; + } + let pyro = ClientCaps::empty() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + ClientCaps::from_bits(caps.bits() & !pyro.bits()) +} + +/// The shared [`Agreement`] for this hub's clients, and the one way of acting +/// on it. +pub struct Negotiator { + agreement: std::sync::Mutex>, +} + +impl Negotiator { + pub fn new() -> Self { + Self { + agreement: std::sync::Mutex::new(Agreement::default()), + } + } + + pub fn state(&self, client: iroh::EndpointId, caps: ClientCaps) { + self.agreement.lock().unwrap().state(client, caps); + } + + pub fn forget(&self, client: &iroh::EndpointId) { + self.agreement.lock().unwrap().forget(client); + } + + /// Give the encoder the joint set if it has changed since it was last + /// given one. + /// + /// Called whenever the set can have changed -- a client stating its + /// capabilities, a client leaving -- and when the controller goes back to + /// automatic. Repeating a set the encoder already has costs nothing there: + /// a change that asks for the current settings is skipped without a + /// rebuild. + /// + /// In manual mode nothing is sent and nothing is marked as sent. A person + /// choosing sets the codec and the depth together, and a client joining + /// must not renegotiate either; the set waits until the mode is automatic + /// again. + pub async fn settle( + &self, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + let manual = controller.lock().await.mode() == ControlMode::Manual; + let mut agreement = self.agreement.lock().unwrap(); + let Some(joint) = agreement.pending() else { + return; + }; + if manual { + info!( + "clients jointly decode {:#010b}, but the encoder is set by hand; leaving it alone", + joint.bits() + ); + return; + } + agreement.mark_sent(joint); + if joint.is_empty() { + // The encoder reads an empty set as no opinion and keeps going, + // which is all that is left to do: whatever it sends, someone + // cannot show it. + warn!( + "the {} connected clients share no codec; at least one of them will not be \ + able to decode the stream", + agreement.stated.len() + ); + return; + } + info!( + "{} client(s) jointly decode {:#010b}", + agreement.stated.len(), + joint.bits() + ); + let mut cmd = vec![nesprotocol::MSG_CLIENT_CAPS]; + nesprotocol::encode_client_caps(&mut cmd, joint); + let _ = cmd_tx.send(cmd); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nesprotocol::{CODEC_AV1, CODEC_H264, CODEC_H265}; + + fn h264_only() -> ClientCaps { + ClientCaps::empty().with(CODEC_H264, DEPTH_8) + } + + fn everything() -> ClientCaps { + ClientCaps::empty() + .with(CODEC_H264, DEPTH_8) + .with(CODEC_H265, DEPTH_8) + .with(CODEC_H265, DEPTH_10) + .with(CODEC_AV1, DEPTH_8) + .with(CODEC_AV1, DEPTH_10) + } + + #[test] + fn a_capable_client_joining_does_not_outvote_a_limited_one() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + a.mark_sent(h264_only()); + + a.state(2, everything()); + assert_eq!(a.joint(), Some(h264_only())); + assert_eq!(a.pending(), None, "the room can still only take H.264"); + } + + #[test] + fn the_limited_client_leaving_frees_the_rest_to_upgrade() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, everything()); + a.mark_sent(h264_only()); + + a.forget(&1); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_limited_client_joining_downgrades_the_room() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + + a.state(2, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + } + + #[test] + fn an_empty_room_starts_over() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.forget(&1); + assert_eq!(a.joint(), None); + + // The encoder may have moved since; the next client is said aloud + // even though it matches what was last sent. + a.state(2, everything()); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_client_that_never_spoke_has_no_vote() { + let a: Agreement = Agreement::default(); + assert_eq!(a.joint(), None); + assert_eq!(a.pending(), None); + } + + #[test] + fn restating_the_same_capabilities_changes_nothing() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.state(1, everything()); + assert_eq!(a.pending(), None); + } + + #[test] + fn clients_sharing_nothing_give_an_empty_set() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, ClientCaps::empty().with(CODEC_AV1, DEPTH_8)); + assert_eq!(a.pending(), Some(ClientCaps::empty())); + } + + #[test] + fn a_foreign_pyrowave_format_does_not_count() { + let caps = everything() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + let foreign = nesprotocol::pyrowave::PYROWAVE_FORMAT.wrapping_add(1); + assert_eq!(effective_caps(caps, Some(foreign)), everything()); + assert_eq!(effective_caps(caps, None), everything()); + assert_eq!( + effective_caps(caps, Some(nesprotocol::pyrowave::PYROWAVE_FORMAT)), + caps + ); + } +} diff --git a/apps/neshub/src/session.rs b/apps/neshub/src/session.rs index 1cb17296..5ec15e5f 100644 --- a/apps/neshub/src/session.rs +++ b/apps/neshub/src/session.rs @@ -16,6 +16,7 @@ use nesprotocol::{ }; use crate::control::{Controller, PathView}; +use crate::negotiation::{Negotiator, effective_caps}; use crate::dgram::{MediaItem, run_datagram_writer}; use bytes::Bytes; @@ -79,6 +80,11 @@ pub struct ClientSession { pending_gamepad_feedback: Option>>, idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, + /// Who this is, for the codec agreement. + client: iroh::EndpointId, + /// Where this client's capabilities go, so the codec is one every client + /// can decode rather than whatever the newest one asked for. + negotiator: Arc, send_video: tokio::sync::mpsc::UnboundedSender, send_audio: tokio::sync::mpsc::UnboundedSender, send_cursor: tokio::sync::mpsc::UnboundedSender>, @@ -115,6 +121,8 @@ impl ClientSession { idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, pyrowave_kbps: u32, + client: iroh::EndpointId, + negotiator: Arc, ) -> Self { let (gamepad_feedback_tx, gamepad_feedback_rx) = tokio::sync::mpsc::unbounded_channel(); let (video_tx, video_rx) = tokio::sync::mpsc::unbounded_channel::(); @@ -138,6 +146,8 @@ impl ClientSession { pending_gamepad_feedback: Some(gamepad_feedback_rx), idr_cmd_tx, controller, + client, + negotiator, send_video: video_tx, send_audio: audio_tx, send_cursor: cursor_tx, @@ -255,8 +265,13 @@ impl ClientSession { ok: self.pyrowave_ok.clone(), default_kbps: self.pyrowave_kbps, }; + let caps = CapsSink { + client: self.client, + negotiator: self.negotiator.clone(), + }; self.tasks.push(tokio::spawn(async move { - run_control_reader(conn, idr, reports, controller, awaiting, pyrowave).await + run_control_reader(conn, idr, reports, controller, awaiting, pyrowave, caps) + .await })); } } @@ -577,6 +592,13 @@ fn pyrowave_request( }) } +/// Where a client's statement of what it decodes is recorded. +#[derive(Clone)] +struct CapsSink { + client: iroh::EndpointId, + negotiator: Arc, +} + /// Everything the client says that is not an input event. async fn run_control_reader( conn: Connection, @@ -585,6 +607,7 @@ async fn run_control_reader( controller: Arc>, awaiting_keyframe: Arc, pyrowave: PyroWaveGate, + caps_sink: CapsSink, ) { run_framed_reader(conn, BIDI_CONTROL, "control", None, |msg_type, payload| { let idr_cmd_tx = idr_cmd_tx.clone(); @@ -592,6 +615,7 @@ async fn run_control_reader( let controller = controller.clone(); let awaiting_keyframe = awaiting_keyframe.clone(); let pyrowave = pyrowave.clone(); + let caps_sink = caps_sink.clone(); async move { match msg_type { MSG_IDR_REQUEST => { @@ -630,25 +654,14 @@ async fn run_control_reader( nesprotocol::pyrowave::PYROWAVE_FORMAT ); } - // Manual means a person is choosing, and the panel - // they chose in sets the codec and the depth - // together. A client joining afterwards must not - // renegotiate either of them: that is the same - // override the controller itself stops doing in - // this mode, and for the same reason. - if controller.lock().await.mode() == ControlMode::Manual { - info!( - "client decodes {:#08b}, but the encoder is set by hand; \ - leaving it alone", - caps.bits() - ); - } else { - info!("client decodes {:#08b}", caps.bits()); - let mut cmd = Vec::with_capacity(1 + payload.len()); - cmd.push(MSG_CLIENT_CAPS); - cmd.extend_from_slice(&payload); - let _ = idr_cmd_tx.send(cmd); - } + // Not forwarded as it stands: the encoder serves + // every client, so it is given what all of them + // decode. See `negotiation`. + info!("client decodes {:#010b}", caps.bits()); + caps_sink + .negotiator + .state(caps_sink.client, effective_caps(caps, format)); + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; } None => debug!("unreadable client capabilities ({} bytes)", payload.len()), } @@ -701,13 +714,20 @@ async fn run_control_reader( }, MSG_CONTROL_MODE => match decode_control_mode(&payload) { Some((mode, ceiling)) => { - let mut controller = controller.lock().await; - controller.set_mode(mode); - controller.set_constant_quality(false); - if let Some(kbps) = ceiling { - controller.set_ceiling(kbps); + { + let mut controller = controller.lock().await; + controller.set_mode(mode); + controller.set_constant_quality(false); + if let Some(kbps) = ceiling { + controller.set_ceiling(kbps); + } } info!("control mode {mode:?}, ceiling {ceiling:?}"); + // Whoever joined or left while a person was choosing + // was not acted on then; now it is. + if mode == ControlMode::Auto { + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; + } } None => debug!("unreadable control mode ({} bytes)", payload.len()), }, @@ -891,6 +911,8 @@ pub struct SessionManager { relay_ms: Arc, // latest relay latency (f32 bits) /// The rate a PyroWave request that names none runs at. pyrowave_kbps: u32, + /// What every connected client decodes, and so what the encoder may pick. + negotiator: Arc, } impl SessionManager { @@ -911,6 +933,7 @@ impl SessionManager { last_audio_bytes: AtomicU64::new(0), relay_ms: Arc::new(AtomicU32::new(0)), pyrowave_kbps, + negotiator: Arc::new(Negotiator::new()), } } @@ -940,6 +963,8 @@ impl SessionManager { idr_cmd_tx, controller, self.pyrowave_kbps, + id, + self.negotiator.clone(), ) }); session.attach(carrier, conn); @@ -950,7 +975,23 @@ impl SessionManager { } } - pub async fn remove_session(&self, id: &iroh::EndpointId) { + /// Drop a client, and let the encoder move to whatever the clients still + /// here can all decode -- a better codec, when the one that left was the + /// one holding the rest back. + pub async fn remove_session( + &self, + id: &iroh::EndpointId, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + if self.drop_session(id).await { + self.negotiator.forget(id); + self.negotiator.settle(controller, cmd_tx).await; + } + } + + /// Whether there was a session to drop. + async fn drop_session(&self, id: &iroh::EndpointId) -> bool { let mut sessions = self.sessions.lock().await; // Said only when something was actually removed. Every carrier of a // client reports its own close, so a client leaving announced its @@ -967,7 +1008,9 @@ impl SessionManager { let mut frame = Vec::with_capacity(7); nesprotocol::gamepad::encode_ipc(&mut frame, session.gamepad_session, &message); let _ = self.gamepad_tx.send(frame); + return true; } + false } /// A receiver for every client's gamepad messages, for the gamepad socket. diff --git a/apps/neswire/src/sink.rs b/apps/neswire/src/sink.rs index 0f976bee..bbce068a 100644 --- a/apps/neswire/src/sink.rs +++ b/apps/neswire/src/sink.rs @@ -3,6 +3,19 @@ use crossbeam_channel::Sender; use pipewire::{self as pw, loop_::Signal, spa, stream::*}; use spa::param::audio::{AudioFormat, AudioInfoRaw}; +/// Frames the graph clock moved past without this stream receiving them. +/// +/// `expected` is where the previous cycle's data ended, `now` where this +/// cycle's begins. Zero on the first cycle, when either position is unknown, +/// and when the clock went backwards -- a restart, which is followed rather +/// than filled. +fn missed_frames(expected: Option, now: Option) -> u64 { + match (expected, now) { + (Some(expected), Some(now)) => now.saturating_sub(expected), + _ => 0, + } +} + pub fn run(sample_rate: u32, channels: u32, frame_size: u32, tx: Sender>) -> Result<()> { pw::init(); tracing::info!("pipewire init ok"); @@ -44,12 +57,20 @@ pub fn run(sample_rate: u32, channels: u32, frame_size: u32, tx: Sender tx: Sender>, accum: Vec, samples_per_frame: usize, + channels: usize, + sample_rate: u32, + /// The graph position the next cycle's data starts at, if the stream + /// has run a cycle yet. See `process` for what it is for. + next_ticks: Option, } let state = State { tx, accum: Vec::with_capacity(samples_per_frame * 2), samples_per_frame, + channels: channels as usize, + sample_rate, + next_ticks: None, }; let _listener = stream @@ -65,6 +86,13 @@ pub fn run(sample_rate: u32, channels: u32, frame_size: u32, tx: Sender } }) .process(move |stream, state| { + // Where the graph clock is this cycle, in frames. Read before the + // buffer: it is this cycle's position whatever the buffer holds. + let ticks = stream + .time() + .ok() + .filter(|t| t.rate().denom == state.sample_rate) + .map(|t| t.ticks()); // Dequeue the buffer from PipeWire if let Some(mut buffer) = stream.dequeue_buffer() { let datas = buffer.datas_mut(); @@ -79,6 +107,31 @@ pub fn run(sample_rate: u32, channels: u32, frame_size: u32, tx: Sender // Reinterpret as f32 samples (we requested F32LE) let samples: &[f32] = bytemuck::cast_slice(audio_bytes); + // **A cycle this stream missed is time the stream + // still has to account for.** When the graph runs a + // cycle and neswire does not get to it in time -- an + // xrun -- that cycle's audio never arrives here, but + // the graph clock moves on. Sending only what arrived + // made the stream run short of realtime by exactly the + // missed cycles, about 3.5% measured, and the client, + // which plays at its own sound card's 48 kHz, drained + // its buffer and underran every few seconds. + // + // So the gap is filled with exactly the frames the clock + // says were missed, as silence: an xrun is a hole of the + // size it really was, and the stream stays at 48 kHz. + // A position that went backwards is a clock restart, + // not a gap, and is simply followed. + let frames = (samples.len() / state.channels) as u64; + let missed = missed_frames(state.next_ticks, ticks) as usize; + if missed > 0 { + state + .accum + .resize(state.accum.len() + missed * state.channels, 0.0); + tracing::debug!(missed, "graph cycle missed; filled with silence"); + } + state.next_ticks = ticks.map(|t| t + frames); + state.accum.extend_from_slice(samples); tracing::trace!( @@ -181,3 +234,22 @@ pub fn run(sample_rate: u32, channels: u32, frame_size: u32, tx: Sender tracing::info!("shutting down"); Ok(()) } + +#[cfg(test)] +mod tests { + use super::missed_frames; + + #[test] + fn a_gap_is_exactly_the_frames_the_clock_moved_past() { + // One 240-frame cycle delivered, the next started 480 later: one missed. + assert_eq!(missed_frames(Some(240), Some(480)), 240); + assert_eq!(missed_frames(Some(240), Some(240)), 0); + } + + #[test] + fn nothing_is_filled_without_two_positions_or_when_the_clock_restarts() { + assert_eq!(missed_frames(None, Some(10_000)), 0); + assert_eq!(missed_frames(Some(240), None), 0); + assert_eq!(missed_frames(Some(10_000), Some(0)), 0); + } +} diff --git a/build/Containerfile b/build/Containerfile index 8031fd3a..b2958de0 100644 --- a/build/Containerfile +++ b/build/Containerfile @@ -202,6 +202,36 @@ RUN --mount=type=cache,target=/root/.cargo/registry \ find /artifacts/nvgpu -type f -printf '/%P\n' > /artifacts/nvgpu/.manifest +# ─────────────────────────────────────────────────────────── +# The guest kernel — shipped inside the image it boots +# +# A host takes it back out as a file beside the root image when it unpacks +# this one (the host agent's onboarding), because the VMM loads a kernel from the host +# and never from the guest's disk. Shipping the two as one image is what +# keeps them a pair: the driver built into this kernel and the Vulkan layer +# above come from the same NVGPU_REF, and a host can only ever have the +# kernel that matches its root. +# +# The same script `make kernel` runs, on a fresh clone every time: an image +# build has no warm tree to reuse, and the layer cache already skips this +# stage while its pins and its inputs are unchanged. +# ─────────────────────────────────────────────────────────── +FROM builder AS kernel-build +ARG KERNEL_GIT=https://github.com/CachyOS/linux.git +ARG KERNEL_REF=cachyos-7.2.6-1 +ARG KERNEL_MARCH=x86-64-v3 +ARG NVGPU_GIT=https://github.com/nestrilabs/virtio-nvgpu.git +ARG NVGPU_REF=dev +RUN pacman -S --noconfirm --needed lld bc cpio perl && pacman -Scc --noconfirm +COPY build/kernel /kernel-build/kernel +COPY build/scripts/kernel-build.sh /kernel-build/scripts/kernel-build.sh +RUN KERNEL_GIT="${KERNEL_GIT}" KERNEL_REF="${KERNEL_REF}" KERNEL_MARCH="${KERNEL_MARCH}" \ + KERNEL_SRC=/build/kernel KERNEL_OUTPUT=/artifacts/kernel/boot/vmlinux \ + NVGPU_GIT="${NVGPU_GIT}" NVGPU_REF="${NVGPU_REF}" NVGPU_WORK=/build/nvgpu \ + bash /kernel-build/scripts/kernel-build.sh && \ + rm -rf /build/kernel /build/nvgpu + + # ─────────────────────────────────────────────────────────── # umu-launcher — built here, not installed from the distribution # @@ -443,6 +473,9 @@ COPY --from=mesa-build /artifacts/mesa / COPY --from=nestri-build /artifacts/nestri / COPY --from=nvgpu-layer-build /artifacts/nvgpu / COPY --from=umu-build /artifacts/umu / +# The kernel this image is booted with, for the host to take out. See the +# kernel-build stage. +COPY --from=kernel-build /artifacts/kernel / # **No Proton here.** It is attached to a box at run time, mounted at # /nestri/compat, so this image has no Proton path for a launch to fall back to diff --git a/build/Makefile b/build/Makefile index e17b777c..d167742b 100644 --- a/build/Makefile +++ b/build/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -.PHONY: build build-debug image image-debug kernel kernel-clean kernel-objclean proton-image proton-push clean help +.PHONY: build push build-debug image image-debug kernel kernel-clean kernel-objclean proton-image proton-push clean help CONTAINER_RT := $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null) ifeq ($(CONTAINER_RT),) @@ -103,11 +103,23 @@ endif build: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ --build-arg NVGPU_GIT=$(NVGPU_GIT) --build-arg NVGPU_REF=$(NVGPU_REF) \ + --build-arg KERNEL_GIT=$(KERNEL_GIT) --build-arg KERNEL_REF=$(KERNEL_REF) \ + --build-arg KERNEL_MARCH=$(KERNEL_MARCH) \ -f Containerfile -t $(IMAGE_NAME):latest --target runtime_prod $(CONTEXT) +# Publishing the image hosts onboard from. Run on the build box, not in CI: a +# full build there takes hours. Pushes the moving tag and one named for the +# commit, so a host can be pinned to exactly what it was tested on. +push: build + $(CONTAINER_RT) tag $(IMAGE_NAME):latest $(IMAGE_NAME):$(shell git rev-parse --short=12 HEAD) + $(CONTAINER_RT) push $(IMAGE_NAME):$(shell git rev-parse --short=12 HEAD) + $(CONTAINER_RT) push $(IMAGE_NAME):latest + build-debug: DOCKER_BUILDKIT=1 $(CONTAINER_RT) build $(if $(FORCE_REBUILD),--no-cache,) \ --build-arg NVGPU_GIT=$(NVGPU_GIT) --build-arg NVGPU_REF=$(NVGPU_REF) \ + --build-arg KERNEL_GIT=$(KERNEL_GIT) --build-arg KERNEL_REF=$(KERNEL_REF) \ + --build-arg KERNEL_MARCH=$(KERNEL_MARCH) \ -f Containerfile -t $(IMAGE_NAME):debug --target runtime_debug $(CONTEXT) # Hours, and only when PROTON_TAG moves. Its context is this directory rather diff --git a/build/etc/pipewire/pipewire.conf.d/50-nestri.conf b/build/etc/pipewire/pipewire.conf.d/50-nestri.conf index 2dda99c3..d19a504d 100644 --- a/build/etc/pipewire/pipewire.conf.d/50-nestri.conf +++ b/build/etc/pipewire/pipewire.conf.d/50-nestri.conf @@ -1,23 +1,38 @@ -# The graph's clock, floored where the audio sender needs it. +# The graph's clock, fixed at one of neswire's packets per cycle. # -# Inside a virtual machine, pipewire.conf raises the graph's minimum quantum to -# 1024 frames (21.3 ms), because a VM's timers jitter and small quanta then -# underrun on real audio hardware. A box is a VM, so that rule applies, and it -# silently overrides neswire's request for its 5 ms packet size. Every sample -# then waits for a 21 ms cycle before it can be encoded. +# 240 frames is neswire's 5 ms Opus packet, and the graph runs exactly that: +# one cycle, one packet, nothing re-cut and nothing carried over. # -# The floor comes down to 256 frames (5.3 ms), the smallest power of two that -# holds one of neswire's packets. It is not lower than that on purpose: nothing -# here needs less, and the reason for the rule, a VM's timer driving the graph, -# still holds, because the Dummy-Driver is exactly that. +# Fixed rather than floored, because a floor never applied. The previous rule +# set min-quantum to 256 for virtual machines, matched on `cpu.vm.name`, which +# PipeWire fills in from DMI -- and nesbox exposes no DMI, so the rule never +# matched. The graph then took neswire's 240-frame request and rounded it down +# to a power of two, 128 frames: a 2.67 ms cycle that a busy guest's timer and +# scheduler miss about twenty times a second. Each missed cycle is audio that +# never reaches neswire, and the stream ran about 3.5% short of realtime -- +# measured 2026-10-07 as 193 packets a second where 200 were due, with +# `pw-top` counting the xruns on neswire. # -# A drop-in appends to a rules array, and every matching rule applies in order, -# so this one runs after the shipped one and wins. +# `clock.power-of-two-quantum = false` is what lets 240 stand: with it on, +# PipeWire rounds every quantum down to a power of two. min and max are both +# 240 so no client can move the graph off it -- a game asking for 480 runs at +# 240 like everything else. +context.properties = { + clock.power-of-two-quantum = false + default.clock.rate = 48000 + default.clock.quantum = 240 + default.clock.min-quantum = 240 + default.clock.max-quantum = 240 +} + +# pipewire.conf ships a rule raising min-quantum to 1024 on any VM it can +# recognise. nesbox is not one it recognises today; if that changes, the rule +# would put min-quantum above max-quantum. This one runs after it and holds 240. context.properties.rules = [ { matches = [ { cpu.vm.name = !null } ] actions = { update-props = { - default.clock.min-quantum = 256 + default.clock.min-quantum = 240 } } } diff --git a/build/kernel/nestri.fragment b/build/kernel/nestri.fragment index c0fdc05b..0065e818 100644 --- a/build/kernel/nestri.fragment +++ b/build/kernel/nestri.fragment @@ -32,8 +32,8 @@ CONFIG_HIDRAW=y # ── Timers ─────────────────────────────────────────────── # The one that cost a day of silent audio. The guest has no sound hardware, so -# PipeWire drives its whole graph off a timerfd at a 2.67ms cycle, which a -# HZ_PERIODIC/250 kernel cannot express. Symptom: everything reports healthy, +# PipeWire drives its whole graph off a timerfd at a 5 ms cycle (240 frames, +# see etc/pipewire), which a HZ_PERIODIC/250 kernel cannot express. Symptom: everything reports healthy, # neswire sends a steady ~3 kbps of Opus, and every sample is zero. CONFIG_HIGH_RES_TIMERS=y CONFIG_NO_HZ_IDLE=y diff --git a/packages/core/src/session/index.ts b/packages/core/src/session/index.ts index 76cdf1c7..b43ded13 100644 --- a/packages/core/src/session/index.ts +++ b/packages/core/src/session/index.ts @@ -336,6 +336,11 @@ export namespace Session { linkedAccountId: z.string().meta({ description: 'Which linked account is playing', example: Examples.Session.linkedAccountId + }), + userId: z.string().meta({ + description: + 'Whose run it is: the box owner, whose Steam sign-in on this host plays it', + example: Examples.User.id }) }) .meta({ @@ -413,7 +418,8 @@ export namespace Session { boxTier: row.box.tier as z.infer['boxTier'], gameId: row.game.id, steamAppId: row.game.steamAppId, - linkedAccountId: row.session.linkedAccountId + linkedAccountId: row.session.linkedAccountId, + userId: row.box.userId }) ) );