feat: WIP s6-overlay and friends

packages/server/Cargo.toml

@@ -12,17 +12,17 @@ gstreamer = { version = "0.24", features = ["v1_26"] }
 gstreamer-webrtc = { version = "0.24", features = ["v1_26"] }
 gst-plugin-webrtc = { version = "0.14" }
 serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.48", features = ["full"] }
+tokio = { version = "1.49", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["full"] }
 clap = { version = "4.5", features = ["env", "derive"] }
 serde_json = "1.0"
-webrtc = "0.14"
-regex = "1.11"
-rand = "0.9"
+webrtc = "0.17"
+regex = "1.12"
+rand = "0.10"
 rustls = { version = "0.23", features = ["ring"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-vimputti = "0.1.7"
+vimputti = "0.1"
 chrono = "0.4"
 prost = "0.14"
 prost-types = "0.14"

packages/server/rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.91"
+channel = "1.93"

packages/server/src/enc_helper.rs

@@ -301,7 +301,7 @@ pub fn encoder_low_latency_params(
             }
             "svtav1enc" => {
                 encoder_optz.set_parameter("preset", "11");
-                encoder_optz.set_parameter("parameters-string", "lookahead=0");
+                encoder_optz.set_parameter("parameters-string", "lookahead=0:fast-decode=2");
             }
             "av1enc" => {
                 encoder_optz.set_parameter("usage-profile", "realtime");
@@ -351,7 +351,7 @@ pub fn encoder_high_quality_params(
             }
             "svtav1enc" => {
                 encoder_optz.set_parameter("preset", "8");
-                encoder_optz.set_parameter("parameters-string", "lookahead=3");
+                encoder_optz.set_parameter("parameters-string", "lookahead=0:fast-decode=1");
             }
             "av1enc" => {
                 encoder_optz.set_parameter("usage-profile", "realtime");

packages/server/src/main.rs

@@ -209,19 +209,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
     gstreamer::init()?;
     let _ = gstrswebrtc::plugin_register_static(); // Might be already registered, so we'll pass..
 
-    if args.app.zero_copy {
-        if args.encoding.video.encoder_type != EncoderType::HARDWARE {
-            tracing::warn!(
-                "zero-copy is only supported with hardware encoders, disabling zero-copy.."
-            );
-            args.app.zero_copy = false;
-        } else {
-            tracing::warn!(
-                "zero-copy is experimental, it may or may not improve performance, or even work at all."
-            );
-        }
-    }
-
     // Handle GPU selection
     let gpus = match handle_gpus(&args) {
         Ok(gpu) => gpu,
@@ -243,6 +230,20 @@ async fn main() -> Result<(), Box<dyn Error>> {
     // Handle video encoder settings
     video_encoder_info = handle_encoder_video_settings(&args, &video_encoder_info);
 
+    // Deal with zero-copy mismatches
+    if args.app.zero_copy {
+        if video_encoder_info.encoder_type != EncoderType::HARDWARE {
+            tracing::warn!(
+                "zero-copy is only supported with hardware encoders, disabling zero-copy.."
+            );
+            args.app.zero_copy = false;
+        } else {
+            tracing::warn!(
+                "zero-copy is experimental, it may or may not improve performance, or even work at all."
+            );
+        }
+    }
+
     // Handle audio encoder selection
     let audio_encoder = handle_encoder_audio(&args);
 
@@ -338,6 +339,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
         video_source.set_property_from_str("render-node", gpu_info.render_path());
     }
 
+    // videorate to enforce constant framerate during static scenes
+    let video_rate = gstreamer::ElementFactory::make("videorate")
+        .property("drop-only", false) // ensure it duplicates frames, not just drops
+        .property("skip-to-first", true) // helps startup latency sometimes
+        .build()?;
+
     // Caps Filter Element (resolution, fps)
     let caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?;
     let caps = gstreamer::Caps::from_str(&format!(
@@ -417,6 +424,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
                     .build()?,
             );
         }
+        enc_helper::VideoCodec::AV1 if video_encoder_info.encoder_type == EncoderType::SOFTWARE => {
+            video_parser = Some(gstreamer::ElementFactory::make("av1parse").build()?);
+        }
         _ => {
             video_parser = None;
         }
@@ -461,8 +471,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
         &video_sink_queue,
         &audio_sink_queue,
         &video_encoder,
-        &caps_filter,
         &video_source_queue,
+        &caps_filter,
+        &video_rate,
         &video_source,
         &audio_encoder,
         &audio_capsfilter,
@@ -524,6 +535,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         if let (Some(vapostproc), Some(va_caps_filter)) = (&vapostproc, &va_caps_filter) {
             gstreamer::Element::link_many(&[
                 &video_source,
+                &video_rate,
                 &caps_filter,
                 &video_source_queue,
                 &vapostproc,
@@ -532,11 +544,18 @@ async fn main() -> Result<(), Box<dyn Error>> {
             ])?;
         } else if video_encoder_info.encoder_api == EncoderAPI::NVENC {
             // NVENC pipeline
-            gstreamer::Element::link_many(&[&video_source, &caps_filter, &video_encoder])?;
+            gstreamer::Element::link_many(&[
+                &video_source,
+                &video_rate,
+                &caps_filter,
+                &video_source_queue,
+                &video_encoder,
+            ])?;
         }
     } else {
         gstreamer::Element::link_many(&[
             &video_source,
+            &video_rate,
             &caps_filter,
             &video_source_queue,
             &video_converter.unwrap(),
@@ -560,9 +579,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
         ])?;
     }
 
-    video_source.set_property("do-timestamp", &false);
-    audio_source.set_property("do-timestamp", &false);
-
     // Optimize latency of pipeline
     pipeline.set_property("latency", &0u64);
     pipeline.set_property("async-handling", true);

packages/server/src/nestrisink/imp.rs

@@ -398,9 +398,20 @@ fn setup_data_channel(
                     if let Some(message_base) = msg_wrapper.message_base {
                         if message_base.payload_type == "input" {
                             if let Some(input_data) = msg_wrapper.payload {
-                                if let Some(event) = handle_input_message(input_data) {
-                                    // Send the event to wayland source, result bool is ignored
-                                    let _ = wayland_src.send_event(event);
+                                /*if let Payload::Clipboard(data) = &input_data {
+                                    tracing::info!("CLIPBOARD: {:?}", data.content);
+                                    // Make sure Ctrl is unpressed before handling clipboard
+                                    let structure = gstreamer::Structure::builder("SetClipboard")
+                                        .field("content", data.content.clone())
+                                        .build();
+
+                                    let _ = wayland_src.send_event(gstreamer::event::CustomUpstream::new(structure));
+                                } else*/
+                                {
+                                    if let Some(event) = handle_input_message(input_data) {
+                                        // Send the event to wayland source, result bool is ignored
+                                        let _ = wayland_src.send_event(event);
+                                    }
                                 }
                             }
                         } else if message_base.payload_type == "controllerInput" {

packages/server/src/p2p/p2p_protocol_stream.rs

@@ -8,7 +8,8 @@ use std::sync::Arc;
 use tokio::sync::mpsc;
 
 // Cloneable callback type
-pub type CallbackInner = dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static;
+pub type CallbackInner =
+    dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static;
 pub struct Callback(Arc<CallbackInner>);
 impl Callback {
     pub fn new<F>(f: F) -> Self

packages/server/src/proto/proto.rs

@@ -2,68 +2,68 @@
 // This file is @generated by prost-build.
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoTimestampEntry {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub stage: ::prost::alloc::string::String,
-    #[prost(message, optional, tag="2")]
+    #[prost(message, optional, tag = "2")]
     pub time: ::core::option::Option<::prost_types::Timestamp>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoLatencyTracker {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub sequence_id: ::prost::alloc::string::String,
-    #[prost(message, repeated, tag="2")]
+    #[prost(message, repeated, tag = "2")]
     pub timestamps: ::prost::alloc::vec::Vec<ProtoTimestampEntry>,
 }
-// Mouse messages 
+// Mouse messages
 
 /// MouseMove message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoMouseMove {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub x: i32,
-    #[prost(int32, tag="2")]
+    #[prost(int32, tag = "2")]
     pub y: i32,
 }
 /// MouseMoveAbs message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoMouseMoveAbs {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub x: i32,
-    #[prost(int32, tag="2")]
+    #[prost(int32, tag = "2")]
     pub y: i32,
 }
 /// MouseWheel message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoMouseWheel {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub x: i32,
-    #[prost(int32, tag="2")]
+    #[prost(int32, tag = "2")]
     pub y: i32,
 }
 /// MouseKeyDown message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoMouseKeyDown {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub key: i32,
 }
 /// MouseKeyUp message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoMouseKeyUp {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub key: i32,
 }
-// Keyboard messages 
+// Keyboard messages
 
 /// KeyDown message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoKeyDown {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub key: i32,
 }
 /// KeyUp message
 #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoKeyUp {
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub key: i32,
 }
 // Clipboard message
@@ -71,96 +71,96 @@ pub struct ProtoKeyUp {
 //   string content = 1; // Clipboard content
 // }
 
-// Controller messages 
+// Controller messages
 
 /// ControllerAttach message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoControllerAttach {
     /// One of the following enums: "ps", "xbox" or "switch"
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub id: ::prost::alloc::string::String,
     /// Session specific slot number (0-3)
-    #[prost(int32, tag="2")]
+    #[prost(int32, tag = "2")]
     pub session_slot: i32,
     /// Session ID of the client
-    #[prost(string, tag="3")]
+    #[prost(string, tag = "3")]
     pub session_id: ::prost::alloc::string::String,
 }
 /// ControllerDetach message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoControllerDetach {
     /// Session specific slot number (0-3)
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub session_slot: i32,
     /// Session ID of the client
-    #[prost(string, tag="2")]
+    #[prost(string, tag = "2")]
     pub session_id: ::prost::alloc::string::String,
 }
 /// ControllerRumble message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoControllerRumble {
     /// Session specific slot number (0-3)
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub session_slot: i32,
     /// Session ID of the client
-    #[prost(string, tag="2")]
+    #[prost(string, tag = "2")]
     pub session_id: ::prost::alloc::string::String,
     /// Low frequency rumble (0-65535)
-    #[prost(int32, tag="3")]
+    #[prost(int32, tag = "3")]
     pub low_frequency: i32,
     /// High frequency rumble (0-65535)
-    #[prost(int32, tag="4")]
+    #[prost(int32, tag = "4")]
     pub high_frequency: i32,
     /// Duration in milliseconds
-    #[prost(int32, tag="5")]
+    #[prost(int32, tag = "5")]
     pub duration: i32,
 }
 /// ControllerStateBatch - single message containing full or partial controller state
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerStateBatch {
     /// Session specific slot number (0-3)
-    #[prost(int32, tag="1")]
+    #[prost(int32, tag = "1")]
     pub session_slot: i32,
     /// Session ID of the client
-    #[prost(string, tag="2")]
+    #[prost(string, tag = "2")]
     pub session_id: ::prost::alloc::string::String,
-    #[prost(enumeration="proto_controller_state_batch::UpdateType", tag="3")]
+    #[prost(enumeration = "proto_controller_state_batch::UpdateType", tag = "3")]
     pub update_type: i32,
     /// Sequence number for packet loss detection
-    #[prost(uint32, tag="4")]
+    #[prost(uint32, tag = "4")]
     pub sequence: u32,
     /// Button state map (Linux event codes)
-    #[prost(map="int32, bool", tag="5")]
+    #[prost(map = "int32, bool", tag = "5")]
     pub button_changed_mask: ::std::collections::HashMap<i32, bool>,
     /// Analog inputs
     ///
     /// -32768 to 32767
-    #[prost(int32, optional, tag="6")]
+    #[prost(int32, optional, tag = "6")]
     pub left_stick_x: ::core::option::Option<i32>,
     /// -32768 to 32767
-    #[prost(int32, optional, tag="7")]
+    #[prost(int32, optional, tag = "7")]
     pub left_stick_y: ::core::option::Option<i32>,
     /// -32768 to 32767
-    #[prost(int32, optional, tag="8")]
+    #[prost(int32, optional, tag = "8")]
     pub right_stick_x: ::core::option::Option<i32>,
     /// -32768 to 32767
-    #[prost(int32, optional, tag="9")]
+    #[prost(int32, optional, tag = "9")]
     pub right_stick_y: ::core::option::Option<i32>,
     /// -32768 to 32767
-    #[prost(int32, optional, tag="10")]
+    #[prost(int32, optional, tag = "10")]
     pub left_trigger: ::core::option::Option<i32>,
     /// -32768 to 32767
-    #[prost(int32, optional, tag="11")]
+    #[prost(int32, optional, tag = "11")]
     pub right_trigger: ::core::option::Option<i32>,
     /// -1, 0, or 1
-    #[prost(int32, optional, tag="12")]
+    #[prost(int32, optional, tag = "12")]
     pub dpad_x: ::core::option::Option<i32>,
     /// -1, 0, or 1
-    #[prost(int32, optional, tag="13")]
+    #[prost(int32, optional, tag = "13")]
     pub dpad_y: ::core::option::Option<i32>,
     /// Bitmask indicating which fields have changed
     /// Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc.
-    #[prost(uint32, optional, tag="14")]
+    #[prost(uint32, optional, tag = "14")]
     pub changed_fields: ::core::option::Option<u32>,
 }
 /// Nested message and enum types in `ProtoControllerStateBatch`.
@@ -194,78 +194,81 @@ pub mod proto_controller_state_batch {
         }
     }
 }
-// WebRTC + signaling 
+// WebRTC + signaling
 
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct RtcIceCandidateInit {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub candidate: ::prost::alloc::string::String,
-    #[prost(uint32, optional, tag="2")]
+    #[prost(uint32, optional, tag = "2")]
     pub sdp_m_line_index: ::core::option::Option<u32>,
-    #[prost(string, optional, tag="3")]
+    #[prost(string, optional, tag = "3")]
     pub sdp_mid: ::core::option::Option<::prost::alloc::string::String>,
-    #[prost(string, optional, tag="4")]
+    #[prost(string, optional, tag = "4")]
     pub username_fragment: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct RtcSessionDescriptionInit {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub sdp: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
+    #[prost(string, tag = "2")]
     pub r#type: ::prost::alloc::string::String,
 }
 /// ProtoICE message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoIce {
-    #[prost(message, optional, tag="1")]
+    #[prost(message, optional, tag = "1")]
     pub candidate: ::core::option::Option<RtcIceCandidateInit>,
 }
 /// ProtoSDP message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoSdp {
-    #[prost(message, optional, tag="1")]
+    #[prost(message, optional, tag = "1")]
     pub sdp: ::core::option::Option<RtcSessionDescriptionInit>,
 }
 /// ProtoRaw message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoRaw {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub data: ::prost::alloc::string::String,
 }
 /// ProtoClientRequestRoomStream message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoClientRequestRoomStream {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub room_name: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
+    #[prost(string, tag = "2")]
     pub session_id: ::prost::alloc::string::String,
 }
 /// ProtoClientDisconnected message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoClientDisconnected {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub session_id: ::prost::alloc::string::String,
-    #[prost(int32, repeated, tag="2")]
+    #[prost(int32, repeated, tag = "2")]
     pub controller_slots: ::prost::alloc::vec::Vec<i32>,
 }
 /// ProtoServerPushStream message
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ProtoServerPushStream {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub room_name: ::prost::alloc::string::String,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoMessageBase {
-    #[prost(string, tag="1")]
+    #[prost(string, tag = "1")]
     pub payload_type: ::prost::alloc::string::String,
-    #[prost(message, optional, tag="2")]
+    #[prost(message, optional, tag = "2")]
     pub latency: ::core::option::Option<ProtoLatencyTracker>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoMessage {
-    #[prost(message, optional, tag="1")]
+    #[prost(message, optional, tag = "1")]
     pub message_base: ::core::option::Option<ProtoMessageBase>,
-    #[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 20, 21, 22, 23, 24, 25")]
+    #[prost(
+        oneof = "proto_message::Payload",
+        tags = "2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 20, 21, 22, 23, 24, 25"
+    )]
     pub payload: ::core::option::Option<proto_message::Payload>,
 }
 /// Nested message and enum types in `ProtoMessage`.
@@ -273,42 +276,42 @@ pub mod proto_message {
     #[derive(Clone, PartialEq, ::prost::Oneof)]
     pub enum Payload {
         /// Input types
-        #[prost(message, tag="2")]
+        #[prost(message, tag = "2")]
         MouseMove(super::ProtoMouseMove),
-        #[prost(message, tag="3")]
+        #[prost(message, tag = "3")]
         MouseMoveAbs(super::ProtoMouseMoveAbs),
-        #[prost(message, tag="4")]
+        #[prost(message, tag = "4")]
         MouseWheel(super::ProtoMouseWheel),
-        #[prost(message, tag="5")]
+        #[prost(message, tag = "5")]
         MouseKeyDown(super::ProtoMouseKeyDown),
-        #[prost(message, tag="6")]
+        #[prost(message, tag = "6")]
         MouseKeyUp(super::ProtoMouseKeyUp),
-        #[prost(message, tag="7")]
+        #[prost(message, tag = "7")]
         KeyDown(super::ProtoKeyDown),
         /// ProtoClipboard clipboard = 9;
-        #[prost(message, tag="8")]
+        #[prost(message, tag = "8")]
         KeyUp(super::ProtoKeyUp),
         /// Controller input types
-        #[prost(message, tag="9")]
+        #[prost(message, tag = "9")]
         ControllerAttach(super::ProtoControllerAttach),
-        #[prost(message, tag="10")]
+        #[prost(message, tag = "10")]
         ControllerDetach(super::ProtoControllerDetach),
-        #[prost(message, tag="11")]
+        #[prost(message, tag = "11")]
         ControllerRumble(super::ProtoControllerRumble),
-        #[prost(message, tag="12")]
+        #[prost(message, tag = "12")]
         ControllerStateBatch(super::ProtoControllerStateBatch),
         /// Signaling types
-        #[prost(message, tag="20")]
+        #[prost(message, tag = "20")]
         Ice(super::ProtoIce),
-        #[prost(message, tag="21")]
+        #[prost(message, tag = "21")]
         Sdp(super::ProtoSdp),
-        #[prost(message, tag="22")]
+        #[prost(message, tag = "22")]
         Raw(super::ProtoRaw),
-        #[prost(message, tag="23")]
+        #[prost(message, tag = "23")]
         ClientRequestRoomStream(super::ProtoClientRequestRoomStream),
-        #[prost(message, tag="24")]
+        #[prost(message, tag = "24")]
         ClientDisconnected(super::ProtoClientDisconnected),
-        #[prost(message, tag="25")]
+        #[prost(message, tag = "25")]
         ServerPushStream(super::ProtoServerPushStream),
     }
 }