diff --git a/apps/neshub/src/e2e_tests.rs b/apps/neshub/src/e2e_tests.rs index c9220844..915024dd 100644 --- a/apps/neshub/src/e2e_tests.rs +++ b/apps/neshub/src/e2e_tests.rs @@ -317,3 +317,85 @@ async fn a_burst_is_skipped_whole_not_torn() { "frames went missing unaccounted" ); } + +/// A client's first media is live media: nothing produced before a carrier's +/// connection was up reaches it, and a video carrier asks for the keyframe it +/// cannot start without. +/// +/// The session exists from the first connection, and the others follow. Before +/// this, every packet produced in between was queued and delivered at once when +/// the carrier attached: 160 ms of audio at the start of a session, measured. +#[tokio::test(flavor = "multi_thread")] +async fn a_new_carrier_starts_from_live_media() { + use nesprotocol::{ALPN_AUDIO, MSG_IDR_REQUEST, decode_audio_payload, decode_frame}; + + let mgr = Arc::new(SessionManager::new(300_000)); + let hub = iroh::Endpoint::builder(presets::N0) + .alpns(vec![ALPN_VIDEO.to_vec(), ALPN_AUDIO.to_vec()]) + .relay_mode(iroh::endpoint::RelayMode::Disabled) + .transport_config(crate::dgram::media_transport_config()) + .bind() + .await + .expect("hub endpoint"); + let client_ep = iroh::Endpoint::builder(presets::N0) + .relay_mode(iroh::endpoint::RelayMode::Disabled) + .transport_config(crate::dgram::media_transport_config()) + .bind() + .await + .expect("client endpoint"); + let (cmd, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel(); + let controller = Arc::new(tokio::sync::Mutex::new(Controller::new(Limits::new(4_000)))); + + // One connection accepted and attached as the carrier its ALPN names. + let accept = |carrier: Carrier| { + let hub = hub.clone(); + let mgr = mgr.clone(); + let cmd = cmd.clone(); + let controller = controller.clone(); + tokio::spawn(async move { + let conn = hub.accept().await.expect("incoming").await.expect("conn"); + let (input, _) = tokio::sync::broadcast::channel(4); + mgr.attach(conn.remote_id(), carrier, conn, input, cmd, controller) + .await; + }) + }; + + // Video first: the session now exists, and audio has no connection yet. + let attached = accept(Carrier::Video); + let _video = client_ep + .connect(hub.addr(), ALPN_VIDEO) + .await + .expect("dial video"); + attached.await.expect("attach video"); + assert_eq!( + cmd_rx.try_recv().ok(), + Some(vec![MSG_IDR_REQUEST]), + "a new video carrier did not ask for a keyframe" + ); + + let packet = |position: u64| nesprotocol::encode_audio_payload(position, &[0xFC]); + for position in 0..32u64 { + mgr.broadcast_audio(packet(position * 240)).await; + } + + let attached = accept(Carrier::Audio); + let audio = client_ep + .connect(hub.addr(), ALPN_AUDIO) + .await + .expect("dial audio"); + attached.await.expect("attach audio"); + let live = 1_000_000 * 240; + mgr.broadcast_audio(packet(live)).await; + + let datagram = tokio::time::timeout(Duration::from_secs(5), audio.read_datagram()) + .await + .expect("no audio arrived") + .expect("audio connection failed"); + let (_, body) = nesprotocol::datagram::decode_datagram(&datagram).expect("a datagram"); + let (_, _, payload) = decode_frame(body).expect("a frame"); + let (position, _) = decode_audio_payload(payload).expect("an audio payload"); + assert_eq!( + position, live, + "the first audio a new carrier sent was produced before it attached" + ); +} diff --git a/apps/neshub/src/session.rs b/apps/neshub/src/session.rs index 7beba930..8e377516 100644 --- a/apps/neshub/src/session.rs +++ b/apps/neshub/src/session.rs @@ -181,6 +181,13 @@ impl ClientSession { return; }; self.video_conn = Some(conn.clone()); + // A writer that has just started has sent no reference frame, + // so nothing but a keyframe can be decoded from it. Asked for + // now rather than after the client fails to decode a delta and + // says so, a round trip later; and deltas are held back until + // it is sent, as for any client waiting on one. + self.awaiting_keyframe.store(true, Ordering::Relaxed); + let _ = self.idr_cmd_tx.send(vec![MSG_IDR_REQUEST]); let relay_ms = self.relay_ms.clone(); let awaiting = self.awaiting_keyframe.clone(); let withheld = self.withheld.clone(); @@ -339,13 +346,29 @@ impl ClientSession { /// only way it fails is a closed channel, which means the writer is already /// gone, and sixty warnings a second on the way down bury whatever actually /// ended the session. + /// + /// **Nothing is queued for a carrier that has not attached.** A client's + /// connections arrive one after another and its session exists from the + /// first, so a channel created with the session would collect every frame + /// produced before its own connection is up and deliver the lot the moment + /// it is: measured 2026-10-16 as 160 ms of audio landing at once as a + /// session began, and over half a second of video. Media is live or it is + /// nothing, so until the writer exists there is nobody to give it to. pub fn send_video_frame(&self, item: MediaItem) { + if self.pending_video.is_some() { + return; + } if let Err(e) = self.send_video.send(item) { debug!("failed to send video data: {e}"); } } + /// Hand one audio packet to this client's writer, once it has one. See + /// [`send_video_frame`](Self::send_video_frame). pub fn send_audio_packet(&self, data: Bytes) { + if self.pending_audio.is_some() { + return; + } if let Err(e) = self.send_audio.send(MediaItem::Coded(data)) { debug!("failed to send audio data: {e}"); } @@ -1286,14 +1309,18 @@ impl SessionManager { mod pipeline_delay_tests { use super::SessionManager; - /// A video payload with `ts_ms` where the wire format puts it. - fn payload(ts_ms: u32) -> Vec { - let mut p = vec![0u8, 0u8]; - p.extend_from_slice(&ts_ms.to_le_bytes()); - p.extend_from_slice(&1920u16.to_le_bytes()); - p.extend_from_slice(&1080u16.to_le_bytes()); - p.extend_from_slice(&[0u8; 32]); - p + /// A video payload captured `ago_us` before now, on the clock the hub reads. + fn payload(ago_us: u64) -> Vec { + nesprotocol::encode_video_payload( + &nesprotocol::VideoHead { + codec: 0, + flags: 0, + captured_us: nesprotocol::clock::monotonic_us() - ago_us, + width: 1920, + height: 1080, + }, + &[0u8; 32], + ) } #[test] @@ -1304,14 +1331,11 @@ mod pipeline_delay_tests { #[test] fn a_steady_pipeline_reports_no_variation() { - // The encoder's timestamp counts from its own start, so the difference - // to wall clock holds a large unknown constant. A pipeline taking the - // same time on every frame must read as zero, not as that constant -- - // which is what a plain subtraction would have reported, pinned at the - // maximum the wire can carry. + // A pipeline taking the same time on every frame reads as no + // variation, however long that time is. let mgr = SessionManager::new(0); - for i in 0..60u32 { - mgr.note_pipeline_delay(&payload(i * 16)); + for _ in 0..60 { + mgr.note_pipeline_delay(&payload(40_000)); } let (p50, _, max) = mgr.pipeline_delays(); assert_eq!(p50, 0);