diff --git a/apps/neshub/src/main.rs b/apps/neshub/src/main.rs index 1e678c17..54e9594f 100644 --- a/apps/neshub/src/main.rs +++ b/apps/neshub/src/main.rs @@ -4,6 +4,7 @@ mod dgram; mod e2e_tests; mod ipc_listener; mod keyframe; +mod negotiation; mod pyro; mod screenshot; mod session; @@ -486,6 +487,8 @@ async fn main() -> Result<()> { ) .await; let mgr_clone = mgr.clone(); + let controller = controller.clone(); + let cmd_tx = cmd_tx.clone(); tokio::spawn(async move { conn.closed().await; // Any one of them going means the session goes. A client @@ -498,7 +501,9 @@ async fn main() -> Result<()> { carrier = carrier.label(), "carrier closed, ending session" ); - mgr_clone.remove_session(&remote_id).await; + mgr_clone + .remove_session(&remote_id, &controller, &cmd_tx) + .await; }); } Err(e) => { diff --git a/apps/neshub/src/negotiation.rs b/apps/neshub/src/negotiation.rs new file mode 100644 index 00000000..7adbbbef --- /dev/null +++ b/apps/neshub/src/negotiation.rs @@ -0,0 +1,269 @@ +//! Which codecs the encoder may pick, given everyone watching. +//! +//! There is one encoder and every client receives its output, so the codec it +//! picks has to be one that every client can decode. Each client states what it +//! decodes when it connects; forwarding each statement as it arrived let the +//! newest client decide, so a client that only decodes H.264 went black the +//! moment one that decodes AV1 joined. The hub is the only place that sees all +//! of them, so it keeps the statements and hands the encoder their +//! intersection instead -- and recomputes it when a client leaves, so the room +//! moves to a better codec once the client holding it back is gone. + +use std::collections::HashMap; +use std::hash::Hash; + +use nesprotocol::{CODEC_PYROWAVE, ClientCaps, ControlMode, DEPTH_8, DEPTH_10}; +use tokio::sync::Mutex; +use tracing::{info, warn}; + +use crate::control::Controller; + +/// Every client's stated capabilities, and the joint set last given to the +/// encoder. +pub struct Agreement { + stated: HashMap, + sent: Option, +} + +impl Default for Agreement { + fn default() -> Self { + Self { + stated: HashMap::new(), + sent: None, + } + } +} + +impl Agreement { + /// Record what `client` decodes, replacing anything it said before. + pub fn state(&mut self, client: K, caps: ClientCaps) { + self.stated.insert(client, caps); + } + + /// Drop a client that has left. + /// + /// Once nobody is left, the next client to arrive is told about from + /// scratch rather than compared with a room that no longer exists. + pub fn forget(&mut self, client: &K) { + self.stated.remove(client); + if self.stated.is_empty() { + self.sent = None; + } + } + + /// What every client that has spoken can decode. `None` when none has. + /// + /// A client that never states its capabilities is left out rather than + /// read as decoding nothing, which is how the encoder already treats one + /// on its own: it has no opinion, and an empty set would erase everyone + /// else's. + pub fn joint(&self) -> Option { + self.stated + .values() + .copied() + .reduce(|a, b| ClientCaps::from_bits(a.bits() & b.bits())) + } + + /// The joint set, if the encoder has not been given it yet. + pub fn pending(&self) -> Option { + self.joint().filter(|joint| Some(*joint) != self.sent) + } + + /// Note that the encoder now has `caps`. + pub fn mark_sent(&mut self, caps: ClientCaps) { + self.sent = Some(caps); + } +} + +/// A client's capabilities as this hub should count them. +/// +/// A client that decodes some other PyroWave bitstream format cannot read this +/// hub's, so its PyroWave bits are dropped rather than letting it vouch for a +/// stream it would show as garbage. +pub fn effective_caps(caps: ClientCaps, pyrowave_format: Option) -> ClientCaps { + if pyrowave_format == Some(nesprotocol::pyrowave::PYROWAVE_FORMAT) { + return caps; + } + let pyro = ClientCaps::empty() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + ClientCaps::from_bits(caps.bits() & !pyro.bits()) +} + +/// The shared [`Agreement`] for this hub's clients, and the one way of acting +/// on it. +pub struct Negotiator { + agreement: std::sync::Mutex>, +} + +impl Negotiator { + pub fn new() -> Self { + Self { + agreement: std::sync::Mutex::new(Agreement::default()), + } + } + + pub fn state(&self, client: iroh::EndpointId, caps: ClientCaps) { + self.agreement.lock().unwrap().state(client, caps); + } + + pub fn forget(&self, client: &iroh::EndpointId) { + self.agreement.lock().unwrap().forget(client); + } + + /// Give the encoder the joint set if it has changed since it was last + /// given one. + /// + /// Called whenever the set can have changed -- a client stating its + /// capabilities, a client leaving -- and when the controller goes back to + /// automatic. Repeating a set the encoder already has costs nothing there: + /// a change that asks for the current settings is skipped without a + /// rebuild. + /// + /// In manual mode nothing is sent and nothing is marked as sent. A person + /// choosing sets the codec and the depth together, and a client joining + /// must not renegotiate either; the set waits until the mode is automatic + /// again. + pub async fn settle( + &self, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + let manual = controller.lock().await.mode() == ControlMode::Manual; + let mut agreement = self.agreement.lock().unwrap(); + let Some(joint) = agreement.pending() else { + return; + }; + if manual { + info!( + "clients jointly decode {:#010b}, but the encoder is set by hand; leaving it alone", + joint.bits() + ); + return; + } + agreement.mark_sent(joint); + if joint.is_empty() { + // The encoder reads an empty set as no opinion and keeps going, + // which is all that is left to do: whatever it sends, someone + // cannot show it. + warn!( + "the {} connected clients share no codec; at least one of them will not be \ + able to decode the stream", + agreement.stated.len() + ); + return; + } + info!( + "{} client(s) jointly decode {:#010b}", + agreement.stated.len(), + joint.bits() + ); + let mut cmd = vec![nesprotocol::MSG_CLIENT_CAPS]; + nesprotocol::encode_client_caps(&mut cmd, joint); + let _ = cmd_tx.send(cmd); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nesprotocol::{CODEC_AV1, CODEC_H264, CODEC_H265}; + + fn h264_only() -> ClientCaps { + ClientCaps::empty().with(CODEC_H264, DEPTH_8) + } + + fn everything() -> ClientCaps { + ClientCaps::empty() + .with(CODEC_H264, DEPTH_8) + .with(CODEC_H265, DEPTH_8) + .with(CODEC_H265, DEPTH_10) + .with(CODEC_AV1, DEPTH_8) + .with(CODEC_AV1, DEPTH_10) + } + + #[test] + fn a_capable_client_joining_does_not_outvote_a_limited_one() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + a.mark_sent(h264_only()); + + a.state(2, everything()); + assert_eq!(a.joint(), Some(h264_only())); + assert_eq!(a.pending(), None, "the room can still only take H.264"); + } + + #[test] + fn the_limited_client_leaving_frees_the_rest_to_upgrade() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, everything()); + a.mark_sent(h264_only()); + + a.forget(&1); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_limited_client_joining_downgrades_the_room() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + + a.state(2, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + } + + #[test] + fn an_empty_room_starts_over() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.forget(&1); + assert_eq!(a.joint(), None); + + // The encoder may have moved since; the next client is said aloud + // even though it matches what was last sent. + a.state(2, everything()); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_client_that_never_spoke_has_no_vote() { + let a: Agreement = Agreement::default(); + assert_eq!(a.joint(), None); + assert_eq!(a.pending(), None); + } + + #[test] + fn restating_the_same_capabilities_changes_nothing() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.state(1, everything()); + assert_eq!(a.pending(), None); + } + + #[test] + fn clients_sharing_nothing_give_an_empty_set() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, ClientCaps::empty().with(CODEC_AV1, DEPTH_8)); + assert_eq!(a.pending(), Some(ClientCaps::empty())); + } + + #[test] + fn a_foreign_pyrowave_format_does_not_count() { + let caps = everything() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + let foreign = nesprotocol::pyrowave::PYROWAVE_FORMAT.wrapping_add(1); + assert_eq!(effective_caps(caps, Some(foreign)), everything()); + assert_eq!(effective_caps(caps, None), everything()); + assert_eq!( + effective_caps(caps, Some(nesprotocol::pyrowave::PYROWAVE_FORMAT)), + caps + ); + } +} diff --git a/apps/neshub/src/session.rs b/apps/neshub/src/session.rs index 1cb17296..5ec15e5f 100644 --- a/apps/neshub/src/session.rs +++ b/apps/neshub/src/session.rs @@ -16,6 +16,7 @@ use nesprotocol::{ }; use crate::control::{Controller, PathView}; +use crate::negotiation::{Negotiator, effective_caps}; use crate::dgram::{MediaItem, run_datagram_writer}; use bytes::Bytes; @@ -79,6 +80,11 @@ pub struct ClientSession { pending_gamepad_feedback: Option>>, idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, + /// Who this is, for the codec agreement. + client: iroh::EndpointId, + /// Where this client's capabilities go, so the codec is one every client + /// can decode rather than whatever the newest one asked for. + negotiator: Arc, send_video: tokio::sync::mpsc::UnboundedSender, send_audio: tokio::sync::mpsc::UnboundedSender, send_cursor: tokio::sync::mpsc::UnboundedSender>, @@ -115,6 +121,8 @@ impl ClientSession { idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, pyrowave_kbps: u32, + client: iroh::EndpointId, + negotiator: Arc, ) -> Self { let (gamepad_feedback_tx, gamepad_feedback_rx) = tokio::sync::mpsc::unbounded_channel(); let (video_tx, video_rx) = tokio::sync::mpsc::unbounded_channel::(); @@ -138,6 +146,8 @@ impl ClientSession { pending_gamepad_feedback: Some(gamepad_feedback_rx), idr_cmd_tx, controller, + client, + negotiator, send_video: video_tx, send_audio: audio_tx, send_cursor: cursor_tx, @@ -255,8 +265,13 @@ impl ClientSession { ok: self.pyrowave_ok.clone(), default_kbps: self.pyrowave_kbps, }; + let caps = CapsSink { + client: self.client, + negotiator: self.negotiator.clone(), + }; self.tasks.push(tokio::spawn(async move { - run_control_reader(conn, idr, reports, controller, awaiting, pyrowave).await + run_control_reader(conn, idr, reports, controller, awaiting, pyrowave, caps) + .await })); } } @@ -577,6 +592,13 @@ fn pyrowave_request( }) } +/// Where a client's statement of what it decodes is recorded. +#[derive(Clone)] +struct CapsSink { + client: iroh::EndpointId, + negotiator: Arc, +} + /// Everything the client says that is not an input event. async fn run_control_reader( conn: Connection, @@ -585,6 +607,7 @@ async fn run_control_reader( controller: Arc>, awaiting_keyframe: Arc, pyrowave: PyroWaveGate, + caps_sink: CapsSink, ) { run_framed_reader(conn, BIDI_CONTROL, "control", None, |msg_type, payload| { let idr_cmd_tx = idr_cmd_tx.clone(); @@ -592,6 +615,7 @@ async fn run_control_reader( let controller = controller.clone(); let awaiting_keyframe = awaiting_keyframe.clone(); let pyrowave = pyrowave.clone(); + let caps_sink = caps_sink.clone(); async move { match msg_type { MSG_IDR_REQUEST => { @@ -630,25 +654,14 @@ async fn run_control_reader( nesprotocol::pyrowave::PYROWAVE_FORMAT ); } - // Manual means a person is choosing, and the panel - // they chose in sets the codec and the depth - // together. A client joining afterwards must not - // renegotiate either of them: that is the same - // override the controller itself stops doing in - // this mode, and for the same reason. - if controller.lock().await.mode() == ControlMode::Manual { - info!( - "client decodes {:#08b}, but the encoder is set by hand; \ - leaving it alone", - caps.bits() - ); - } else { - info!("client decodes {:#08b}", caps.bits()); - let mut cmd = Vec::with_capacity(1 + payload.len()); - cmd.push(MSG_CLIENT_CAPS); - cmd.extend_from_slice(&payload); - let _ = idr_cmd_tx.send(cmd); - } + // Not forwarded as it stands: the encoder serves + // every client, so it is given what all of them + // decode. See `negotiation`. + info!("client decodes {:#010b}", caps.bits()); + caps_sink + .negotiator + .state(caps_sink.client, effective_caps(caps, format)); + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; } None => debug!("unreadable client capabilities ({} bytes)", payload.len()), } @@ -701,13 +714,20 @@ async fn run_control_reader( }, MSG_CONTROL_MODE => match decode_control_mode(&payload) { Some((mode, ceiling)) => { - let mut controller = controller.lock().await; - controller.set_mode(mode); - controller.set_constant_quality(false); - if let Some(kbps) = ceiling { - controller.set_ceiling(kbps); + { + let mut controller = controller.lock().await; + controller.set_mode(mode); + controller.set_constant_quality(false); + if let Some(kbps) = ceiling { + controller.set_ceiling(kbps); + } } info!("control mode {mode:?}, ceiling {ceiling:?}"); + // Whoever joined or left while a person was choosing + // was not acted on then; now it is. + if mode == ControlMode::Auto { + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; + } } None => debug!("unreadable control mode ({} bytes)", payload.len()), }, @@ -891,6 +911,8 @@ pub struct SessionManager { relay_ms: Arc, // latest relay latency (f32 bits) /// The rate a PyroWave request that names none runs at. pyrowave_kbps: u32, + /// What every connected client decodes, and so what the encoder may pick. + negotiator: Arc, } impl SessionManager { @@ -911,6 +933,7 @@ impl SessionManager { last_audio_bytes: AtomicU64::new(0), relay_ms: Arc::new(AtomicU32::new(0)), pyrowave_kbps, + negotiator: Arc::new(Negotiator::new()), } } @@ -940,6 +963,8 @@ impl SessionManager { idr_cmd_tx, controller, self.pyrowave_kbps, + id, + self.negotiator.clone(), ) }); session.attach(carrier, conn); @@ -950,7 +975,23 @@ impl SessionManager { } } - pub async fn remove_session(&self, id: &iroh::EndpointId) { + /// Drop a client, and let the encoder move to whatever the clients still + /// here can all decode -- a better codec, when the one that left was the + /// one holding the rest back. + pub async fn remove_session( + &self, + id: &iroh::EndpointId, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + if self.drop_session(id).await { + self.negotiator.forget(id); + self.negotiator.settle(controller, cmd_tx).await; + } + } + + /// Whether there was a session to drop. + async fn drop_session(&self, id: &iroh::EndpointId) -> bool { let mut sessions = self.sessions.lock().await; // Said only when something was actually removed. Every carrier of a // client reports its own close, so a client leaving announced its @@ -967,7 +1008,9 @@ impl SessionManager { let mut frame = Vec::with_capacity(7); nesprotocol::gamepad::encode_ipc(&mut frame, session.gamepad_session, &message); let _ = self.gamepad_tx.send(frame); + return true; } + false } /// A receiver for every client's gamepad messages, for the gamepad socket.