From 471d08d20f5e6ef06788b43ff030a90ec6d0daf8 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Mon, 5 Oct 2026 17:47:59 +0300 Subject: [PATCH] fix(neshub): pick a codec every connected client can decode Each client states what it decodes when it connects, and the hub passed each statement to the encoder as it arrived. The newest client decided, so a client that only decodes H.264 lost its picture as soon as one that decodes AV1 or H.265 joined. The hub now keeps every client's statement and gives the encoder their intersection, recomputed when a client joins, when one leaves, and when the controller returns to automatic. A room settles on the best codec all of its clients share, and moves up again once the client holding it back has left. Manual mode still leaves the encoder alone; the change is applied when the mode goes back to automatic. Co-Authored-By: Claude Opus 5.5 --- apps/neshub/src/main.rs | 7 +- apps/neshub/src/negotiation.rs | 269 +++++++++++++++++++++++++++++++++ apps/neshub/src/session.rs | 95 ++++++++---- 3 files changed, 344 insertions(+), 27 deletions(-) create mode 100644 apps/neshub/src/negotiation.rs diff --git a/apps/neshub/src/main.rs b/apps/neshub/src/main.rs index 1e678c17..54e9594f 100644 --- a/apps/neshub/src/main.rs +++ b/apps/neshub/src/main.rs @@ -4,6 +4,7 @@ mod dgram; mod e2e_tests; mod ipc_listener; mod keyframe; +mod negotiation; mod pyro; mod screenshot; mod session; @@ -486,6 +487,8 @@ async fn main() -> Result<()> { ) .await; let mgr_clone = mgr.clone(); + let controller = controller.clone(); + let cmd_tx = cmd_tx.clone(); tokio::spawn(async move { conn.closed().await; // Any one of them going means the session goes. A client @@ -498,7 +501,9 @@ async fn main() -> Result<()> { carrier = carrier.label(), "carrier closed, ending session" ); - mgr_clone.remove_session(&remote_id).await; + mgr_clone + .remove_session(&remote_id, &controller, &cmd_tx) + .await; }); } Err(e) => { diff --git a/apps/neshub/src/negotiation.rs b/apps/neshub/src/negotiation.rs new file mode 100644 index 00000000..7adbbbef --- /dev/null +++ b/apps/neshub/src/negotiation.rs @@ -0,0 +1,269 @@ +//! Which codecs the encoder may pick, given everyone watching. +//! +//! There is one encoder and every client receives its output, so the codec it +//! picks has to be one that every client can decode. Each client states what it +//! decodes when it connects; forwarding each statement as it arrived let the +//! newest client decide, so a client that only decodes H.264 went black the +//! moment one that decodes AV1 joined. The hub is the only place that sees all +//! of them, so it keeps the statements and hands the encoder their +//! intersection instead -- and recomputes it when a client leaves, so the room +//! moves to a better codec once the client holding it back is gone. + +use std::collections::HashMap; +use std::hash::Hash; + +use nesprotocol::{CODEC_PYROWAVE, ClientCaps, ControlMode, DEPTH_8, DEPTH_10}; +use tokio::sync::Mutex; +use tracing::{info, warn}; + +use crate::control::Controller; + +/// Every client's stated capabilities, and the joint set last given to the +/// encoder. +pub struct Agreement { + stated: HashMap, + sent: Option, +} + +impl Default for Agreement { + fn default() -> Self { + Self { + stated: HashMap::new(), + sent: None, + } + } +} + +impl Agreement { + /// Record what `client` decodes, replacing anything it said before. + pub fn state(&mut self, client: K, caps: ClientCaps) { + self.stated.insert(client, caps); + } + + /// Drop a client that has left. + /// + /// Once nobody is left, the next client to arrive is told about from + /// scratch rather than compared with a room that no longer exists. + pub fn forget(&mut self, client: &K) { + self.stated.remove(client); + if self.stated.is_empty() { + self.sent = None; + } + } + + /// What every client that has spoken can decode. `None` when none has. + /// + /// A client that never states its capabilities is left out rather than + /// read as decoding nothing, which is how the encoder already treats one + /// on its own: it has no opinion, and an empty set would erase everyone + /// else's. + pub fn joint(&self) -> Option { + self.stated + .values() + .copied() + .reduce(|a, b| ClientCaps::from_bits(a.bits() & b.bits())) + } + + /// The joint set, if the encoder has not been given it yet. + pub fn pending(&self) -> Option { + self.joint().filter(|joint| Some(*joint) != self.sent) + } + + /// Note that the encoder now has `caps`. + pub fn mark_sent(&mut self, caps: ClientCaps) { + self.sent = Some(caps); + } +} + +/// A client's capabilities as this hub should count them. +/// +/// A client that decodes some other PyroWave bitstream format cannot read this +/// hub's, so its PyroWave bits are dropped rather than letting it vouch for a +/// stream it would show as garbage. +pub fn effective_caps(caps: ClientCaps, pyrowave_format: Option) -> ClientCaps { + if pyrowave_format == Some(nesprotocol::pyrowave::PYROWAVE_FORMAT) { + return caps; + } + let pyro = ClientCaps::empty() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + ClientCaps::from_bits(caps.bits() & !pyro.bits()) +} + +/// The shared [`Agreement`] for this hub's clients, and the one way of acting +/// on it. +pub struct Negotiator { + agreement: std::sync::Mutex>, +} + +impl Negotiator { + pub fn new() -> Self { + Self { + agreement: std::sync::Mutex::new(Agreement::default()), + } + } + + pub fn state(&self, client: iroh::EndpointId, caps: ClientCaps) { + self.agreement.lock().unwrap().state(client, caps); + } + + pub fn forget(&self, client: &iroh::EndpointId) { + self.agreement.lock().unwrap().forget(client); + } + + /// Give the encoder the joint set if it has changed since it was last + /// given one. + /// + /// Called whenever the set can have changed -- a client stating its + /// capabilities, a client leaving -- and when the controller goes back to + /// automatic. Repeating a set the encoder already has costs nothing there: + /// a change that asks for the current settings is skipped without a + /// rebuild. + /// + /// In manual mode nothing is sent and nothing is marked as sent. A person + /// choosing sets the codec and the depth together, and a client joining + /// must not renegotiate either; the set waits until the mode is automatic + /// again. + pub async fn settle( + &self, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + let manual = controller.lock().await.mode() == ControlMode::Manual; + let mut agreement = self.agreement.lock().unwrap(); + let Some(joint) = agreement.pending() else { + return; + }; + if manual { + info!( + "clients jointly decode {:#010b}, but the encoder is set by hand; leaving it alone", + joint.bits() + ); + return; + } + agreement.mark_sent(joint); + if joint.is_empty() { + // The encoder reads an empty set as no opinion and keeps going, + // which is all that is left to do: whatever it sends, someone + // cannot show it. + warn!( + "the {} connected clients share no codec; at least one of them will not be \ + able to decode the stream", + agreement.stated.len() + ); + return; + } + info!( + "{} client(s) jointly decode {:#010b}", + agreement.stated.len(), + joint.bits() + ); + let mut cmd = vec![nesprotocol::MSG_CLIENT_CAPS]; + nesprotocol::encode_client_caps(&mut cmd, joint); + let _ = cmd_tx.send(cmd); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nesprotocol::{CODEC_AV1, CODEC_H264, CODEC_H265}; + + fn h264_only() -> ClientCaps { + ClientCaps::empty().with(CODEC_H264, DEPTH_8) + } + + fn everything() -> ClientCaps { + ClientCaps::empty() + .with(CODEC_H264, DEPTH_8) + .with(CODEC_H265, DEPTH_8) + .with(CODEC_H265, DEPTH_10) + .with(CODEC_AV1, DEPTH_8) + .with(CODEC_AV1, DEPTH_10) + } + + #[test] + fn a_capable_client_joining_does_not_outvote_a_limited_one() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + a.mark_sent(h264_only()); + + a.state(2, everything()); + assert_eq!(a.joint(), Some(h264_only())); + assert_eq!(a.pending(), None, "the room can still only take H.264"); + } + + #[test] + fn the_limited_client_leaving_frees_the_rest_to_upgrade() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, everything()); + a.mark_sent(h264_only()); + + a.forget(&1); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_limited_client_joining_downgrades_the_room() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + + a.state(2, h264_only()); + assert_eq!(a.pending(), Some(h264_only())); + } + + #[test] + fn an_empty_room_starts_over() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.forget(&1); + assert_eq!(a.joint(), None); + + // The encoder may have moved since; the next client is said aloud + // even though it matches what was last sent. + a.state(2, everything()); + assert_eq!(a.pending(), Some(everything())); + } + + #[test] + fn a_client_that_never_spoke_has_no_vote() { + let a: Agreement = Agreement::default(); + assert_eq!(a.joint(), None); + assert_eq!(a.pending(), None); + } + + #[test] + fn restating_the_same_capabilities_changes_nothing() { + let mut a = Agreement::default(); + a.state(1, everything()); + a.mark_sent(everything()); + a.state(1, everything()); + assert_eq!(a.pending(), None); + } + + #[test] + fn clients_sharing_nothing_give_an_empty_set() { + let mut a = Agreement::default(); + a.state(1, h264_only()); + a.state(2, ClientCaps::empty().with(CODEC_AV1, DEPTH_8)); + assert_eq!(a.pending(), Some(ClientCaps::empty())); + } + + #[test] + fn a_foreign_pyrowave_format_does_not_count() { + let caps = everything() + .with(CODEC_PYROWAVE, DEPTH_8) + .with(CODEC_PYROWAVE, DEPTH_10); + let foreign = nesprotocol::pyrowave::PYROWAVE_FORMAT.wrapping_add(1); + assert_eq!(effective_caps(caps, Some(foreign)), everything()); + assert_eq!(effective_caps(caps, None), everything()); + assert_eq!( + effective_caps(caps, Some(nesprotocol::pyrowave::PYROWAVE_FORMAT)), + caps + ); + } +} diff --git a/apps/neshub/src/session.rs b/apps/neshub/src/session.rs index 1cb17296..5ec15e5f 100644 --- a/apps/neshub/src/session.rs +++ b/apps/neshub/src/session.rs @@ -16,6 +16,7 @@ use nesprotocol::{ }; use crate::control::{Controller, PathView}; +use crate::negotiation::{Negotiator, effective_caps}; use crate::dgram::{MediaItem, run_datagram_writer}; use bytes::Bytes; @@ -79,6 +80,11 @@ pub struct ClientSession { pending_gamepad_feedback: Option>>, idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, + /// Who this is, for the codec agreement. + client: iroh::EndpointId, + /// Where this client's capabilities go, so the codec is one every client + /// can decode rather than whatever the newest one asked for. + negotiator: Arc, send_video: tokio::sync::mpsc::UnboundedSender, send_audio: tokio::sync::mpsc::UnboundedSender, send_cursor: tokio::sync::mpsc::UnboundedSender>, @@ -115,6 +121,8 @@ impl ClientSession { idr_cmd_tx: tokio::sync::mpsc::UnboundedSender>, controller: Arc>, pyrowave_kbps: u32, + client: iroh::EndpointId, + negotiator: Arc, ) -> Self { let (gamepad_feedback_tx, gamepad_feedback_rx) = tokio::sync::mpsc::unbounded_channel(); let (video_tx, video_rx) = tokio::sync::mpsc::unbounded_channel::(); @@ -138,6 +146,8 @@ impl ClientSession { pending_gamepad_feedback: Some(gamepad_feedback_rx), idr_cmd_tx, controller, + client, + negotiator, send_video: video_tx, send_audio: audio_tx, send_cursor: cursor_tx, @@ -255,8 +265,13 @@ impl ClientSession { ok: self.pyrowave_ok.clone(), default_kbps: self.pyrowave_kbps, }; + let caps = CapsSink { + client: self.client, + negotiator: self.negotiator.clone(), + }; self.tasks.push(tokio::spawn(async move { - run_control_reader(conn, idr, reports, controller, awaiting, pyrowave).await + run_control_reader(conn, idr, reports, controller, awaiting, pyrowave, caps) + .await })); } } @@ -577,6 +592,13 @@ fn pyrowave_request( }) } +/// Where a client's statement of what it decodes is recorded. +#[derive(Clone)] +struct CapsSink { + client: iroh::EndpointId, + negotiator: Arc, +} + /// Everything the client says that is not an input event. async fn run_control_reader( conn: Connection, @@ -585,6 +607,7 @@ async fn run_control_reader( controller: Arc>, awaiting_keyframe: Arc, pyrowave: PyroWaveGate, + caps_sink: CapsSink, ) { run_framed_reader(conn, BIDI_CONTROL, "control", None, |msg_type, payload| { let idr_cmd_tx = idr_cmd_tx.clone(); @@ -592,6 +615,7 @@ async fn run_control_reader( let controller = controller.clone(); let awaiting_keyframe = awaiting_keyframe.clone(); let pyrowave = pyrowave.clone(); + let caps_sink = caps_sink.clone(); async move { match msg_type { MSG_IDR_REQUEST => { @@ -630,25 +654,14 @@ async fn run_control_reader( nesprotocol::pyrowave::PYROWAVE_FORMAT ); } - // Manual means a person is choosing, and the panel - // they chose in sets the codec and the depth - // together. A client joining afterwards must not - // renegotiate either of them: that is the same - // override the controller itself stops doing in - // this mode, and for the same reason. - if controller.lock().await.mode() == ControlMode::Manual { - info!( - "client decodes {:#08b}, but the encoder is set by hand; \ - leaving it alone", - caps.bits() - ); - } else { - info!("client decodes {:#08b}", caps.bits()); - let mut cmd = Vec::with_capacity(1 + payload.len()); - cmd.push(MSG_CLIENT_CAPS); - cmd.extend_from_slice(&payload); - let _ = idr_cmd_tx.send(cmd); - } + // Not forwarded as it stands: the encoder serves + // every client, so it is given what all of them + // decode. See `negotiation`. + info!("client decodes {:#010b}", caps.bits()); + caps_sink + .negotiator + .state(caps_sink.client, effective_caps(caps, format)); + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; } None => debug!("unreadable client capabilities ({} bytes)", payload.len()), } @@ -701,13 +714,20 @@ async fn run_control_reader( }, MSG_CONTROL_MODE => match decode_control_mode(&payload) { Some((mode, ceiling)) => { - let mut controller = controller.lock().await; - controller.set_mode(mode); - controller.set_constant_quality(false); - if let Some(kbps) = ceiling { - controller.set_ceiling(kbps); + { + let mut controller = controller.lock().await; + controller.set_mode(mode); + controller.set_constant_quality(false); + if let Some(kbps) = ceiling { + controller.set_ceiling(kbps); + } } info!("control mode {mode:?}, ceiling {ceiling:?}"); + // Whoever joined or left while a person was choosing + // was not acted on then; now it is. + if mode == ControlMode::Auto { + caps_sink.negotiator.settle(&controller, &idr_cmd_tx).await; + } } None => debug!("unreadable control mode ({} bytes)", payload.len()), }, @@ -891,6 +911,8 @@ pub struct SessionManager { relay_ms: Arc, // latest relay latency (f32 bits) /// The rate a PyroWave request that names none runs at. pyrowave_kbps: u32, + /// What every connected client decodes, and so what the encoder may pick. + negotiator: Arc, } impl SessionManager { @@ -911,6 +933,7 @@ impl SessionManager { last_audio_bytes: AtomicU64::new(0), relay_ms: Arc::new(AtomicU32::new(0)), pyrowave_kbps, + negotiator: Arc::new(Negotiator::new()), } } @@ -940,6 +963,8 @@ impl SessionManager { idr_cmd_tx, controller, self.pyrowave_kbps, + id, + self.negotiator.clone(), ) }); session.attach(carrier, conn); @@ -950,7 +975,23 @@ impl SessionManager { } } - pub async fn remove_session(&self, id: &iroh::EndpointId) { + /// Drop a client, and let the encoder move to whatever the clients still + /// here can all decode -- a better codec, when the one that left was the + /// one holding the rest back. + pub async fn remove_session( + &self, + id: &iroh::EndpointId, + controller: &Mutex, + cmd_tx: &tokio::sync::mpsc::UnboundedSender>, + ) { + if self.drop_session(id).await { + self.negotiator.forget(id); + self.negotiator.settle(controller, cmd_tx).await; + } + } + + /// Whether there was a session to drop. + async fn drop_session(&self, id: &iroh::EndpointId) -> bool { let mut sessions = self.sessions.lock().await; // Said only when something was actually removed. Every carrier of a // client reports its own close, so a client leaving announced its @@ -967,7 +1008,9 @@ impl SessionManager { let mut frame = Vec::with_capacity(7); nesprotocol::gamepad::encode_ipc(&mut frame, session.gamepad_session, &message); let _ = self.gamepad_tx.send(frame); + return true; } + false } /// A receiver for every client's gamepad messages, for the gamepad socket.