mirror of
https://github.com/nestriness/nestri.git
synced 2026-11-03 22:22:39 +02:00
feat(protocol): put time on every media unit (v4)
Step 1 of nescore's 2026-10-16 media timing spec. Media carried no send time and audio no time at all, so every layer had to infer lateness from gaps between arrivals. - clock: one sender clock, CLOCK_MONOTONIC in u64 microseconds, shared by every process that stamps media. Off unix the client uses its own process clock, which is all a receiver needs. - Datagram headers (and PyroWave's) carry sent_us, stamped by neshub at hand-off to QUIC. Keyframe streams open with it too. - Video carries captured_us (from nescapture's present time) in place of milliseconds since the encoder started. Audio carries its first sample's position on the PipeWire graph clock, which neswire already read to fill missed cycles. The IPC stamp is a u64 carrying either. - delay: microseconds. The baseline no longer expires on a 30 s timer: clock skew is measured as the least-squares slope of each second's minimum one-way delay, and only a path change resets the baseline. observe() returns each unit's queueing delay. - ReceiverReport carries video and audio queueing delay summaries for the rate control that will consume them. - Shared encode/decode helpers for video and audio payloads, so the hub and receivers stop hand-building byte offsets. No compatibility kept. STREAM_VERSION is 4. nescore needs a pin and the client side (step 2) before it can talk to this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3aeae631f6
commit
e014c6f1f7
@@ -1080,7 +1080,6 @@ impl PipelineHandle {
|
||||
encode_ms: encode_avg_ms.clone(),
|
||||
dropped: dropped_frames.clone(),
|
||||
idr_requested: idr_requested.clone(),
|
||||
epoch: Instant::now(),
|
||||
reconfig_tx: reconfig_tx.clone(),
|
||||
};
|
||||
let ipc_prog = ipc_progress.clone();
|
||||
@@ -2709,14 +2708,14 @@ fn bgra_to_yuv420(pixels: &[u8], width: u32, height: u32, vk_format: u32) -> Vec
|
||||
/// has: it only has to tell one frame from the next.
|
||||
fn pyro_messages(
|
||||
frame: &nespyro::EncodedFrame,
|
||||
ts_ms: u32,
|
||||
captured_us: u64,
|
||||
width: u16,
|
||||
height: u16,
|
||||
) -> Result<Vec<Vec<u8>>, nesprotocol::pyrowave::ChunkError> {
|
||||
nesprotocol::pyrowave::chunk_frame(
|
||||
frame.index as u32,
|
||||
nesprotocol::pyrowave::FrameMeta {
|
||||
ts_ms,
|
||||
captured_us,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
@@ -2741,8 +2740,6 @@ struct IpcConfig {
|
||||
dropped: Arc<AtomicU32>,
|
||||
/// Shared with the encoder thread, which honours it on the next frame.
|
||||
idr_requested: Arc<AtomicBool>,
|
||||
/// Zero point for wire timestamps.
|
||||
epoch: Instant,
|
||||
/// For the rate probe to command its own steps.
|
||||
///
|
||||
/// The probe has to live where the encoded sizes are, which is here, and
|
||||
@@ -2799,9 +2796,6 @@ fn ipc_send_thread(
|
||||
}
|
||||
};
|
||||
|
||||
// Fixed before the first frame arrives, so every timestamp shares an epoch
|
||||
// even though frames are stamped from their own present.
|
||||
let start_time = cfg.epoch;
|
||||
let mut frame_count: u64 = 0;
|
||||
// So a PyroWave encoder short of scratch space is said once per episode.
|
||||
let mut scratch_overflowing = false;
|
||||
@@ -2920,9 +2914,11 @@ fn ipc_send_thread(
|
||||
// however long the frame spent queued for the encoder into the
|
||||
// timestamp, so the receiver could not tell capture time from
|
||||
// backlog and had nothing honest to pace on.
|
||||
let timestamp_ms = present_time
|
||||
.saturating_duration_since(start_time)
|
||||
.as_millis() as u32;
|
||||
//
|
||||
// On the clock every media stamp on this machine shares, so the hub
|
||||
// can measure capture-to-send exactly and the client can measure
|
||||
// the path from it.
|
||||
let captured_us = nesprotocol::clock::instant_us(present_time);
|
||||
// Set FLAG_RECONFIG on the first frame after an encoder reconfig.
|
||||
// Clear it after setting so only the first frame is marked. Taken
|
||||
// for a PyroWave frame too, which carries no flags, so that the
|
||||
@@ -2948,7 +2944,7 @@ fn ipc_send_thread(
|
||||
STREAM_VIDEO,
|
||||
protocol_codec,
|
||||
flags,
|
||||
timestamp_ms,
|
||||
captured_us,
|
||||
width,
|
||||
height,
|
||||
&pkt.data,
|
||||
@@ -2974,7 +2970,7 @@ fn ipc_send_thread(
|
||||
log::info!("PyroWave encoder has scratch space again");
|
||||
}
|
||||
scratch_overflowing = overflowed;
|
||||
match pyro_messages(frame, timestamp_ms, width, height) {
|
||||
match pyro_messages(frame, captured_us, width, height) {
|
||||
// Every PyroWave frame stands alone, so none is a key
|
||||
// frame in the sense the counters below mean.
|
||||
Ok(messages) => (messages, false, frame.data.len()),
|
||||
@@ -4808,7 +4804,7 @@ mod pyrowave_tests {
|
||||
assert_eq!(got.frame, 41);
|
||||
assert_eq!(got.critical, 4);
|
||||
assert_eq!(
|
||||
(got.meta.ts_ms, got.meta.width, got.meta.height),
|
||||
(got.meta.captured_us, got.meta.width, got.meta.height),
|
||||
(1234, 2560, 1440)
|
||||
);
|
||||
assert_eq!(got.packets.len(), frame.packets.len());
|
||||
|
||||
Reference in New Issue
Block a user