github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 00:35:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix good chunk of packet loss and frame drop issues

Browse Source 
- Also added new latency control parameter, not super visible differences, but it's cool :)
This commit is contained in:
DatCaptainHorse
2025-12-04 02:21:46 +02:00
parent 549a98bc48
commit b743dab332
24 changed files with 813 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/configs/pipewire.conf.d/nestri-low-latency.conf
View File

@@ -2,6 +2,6 @@ context.properties = {
default.clock.rate = 48000
default.clock.allowed-rates = [48000]
default.clock.min-quantum = 128
default.clock.max-quantum = 256
default.clock.quantum = 128
default.clock.max-quantum = 1024
default.clock.quantum = 512
}

2
packages/configs/wireplumber.conf.d/nestri-low-latency.conf
View File

@@ -22,7 +22,7 @@
],
"apply_properties": {
"pulse.min.req": 128,
"pulse.max.req": 256,
"pulse.max.req": 1024,
"pulse.idle.timeout": 0
}
}

2
packages/input/src/proto/latency_tracker_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
// @generated from file latency_tracker.proto (package proto, syntax proto3)
/* eslint-disable */

4
packages/input/src/proto/messages_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
// @generated from file messages.proto (package proto, syntax proto3)
/* eslint-disable */
@@ -90,6 +90,8 @@ export type ProtoMessage = Message<"proto.ProtoMessage"> & {
case: "keyDown";
} | {
/**
* ProtoClipboard clipboard = 9;
*
* @generated from field: proto.ProtoKeyUp key_up = 8;
*/
value: ProtoKeyUp;

2
packages/input/src/proto/types_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
// @generated from file types.proto (package proto, syntax proto3)
/* eslint-disable */

3
packages/play-standalone/src/pages/[room].astro
View File

@@ -99,9 +99,6 @@ if (envs_map.size > 0) {
});
window.addEventListener("gamepaddisconnected", (e) => {
console.log("Gamepad disconnected:", e.gamepad);
if (e.gamepad.id.toLowerCase().includes("nestri"))
return;
let disconnected = nestriControllers.find((c) => c.getSlot() === e.gamepad.index);
if (disconnected) {
disconnected.dispose();

114
packages/relay/internal/common/common.go
View File

@@ -2,6 +2,7 @@ package common
import (
"fmt"
"github.com/pion/interceptor/pkg/nack"
"log/slog"
"strconv"
@@ -30,20 +31,121 @@ func InitWebRTCAPI() error {
return fmt.Errorf("failed to register extensions: %w", err)
}
// Default codecs cover our needs
err = mediaEngine.RegisterDefaultCodecs()
if err != nil {
return err
// Register codecs
for _, codec := range []webrtc.RTPCodecParameters{
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
PayloadType: 111,
},
} {
if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil {
return err
}
}
videoRTCPFeedback := []webrtc.RTCPFeedback{{"nack", ""}, {"nack", "pli"}}
for _, codec := range []webrtc.RTPCodecParameters{
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 102,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 104,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 106,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 108,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 127,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264,
ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 39,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH265,
ClockRate: 90000,
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 116,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeAV1, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback},
PayloadType: 45,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: videoRTCPFeedback},
PayloadType: 98,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=2", RTCPFeedback: videoRTCPFeedback},
PayloadType: 100,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f",
RTCPFeedback: videoRTCPFeedback,
},
PayloadType: 112,
},
} {
if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
return err
}
}
// Interceptor registry
interceptorRegistry := &interceptor.Registry{}
// Use default set
err = webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry)
// Register our interceptors..
nackGenFactory, err := nack.NewGeneratorInterceptor()
if err != nil {
return err
}
interceptorRegistry.Add(nackGenFactory)
nackRespFactory, err := nack.NewResponderInterceptor()
if err != nil {
return err
}
interceptorRegistry.Add(nackRespFactory)
if err = webrtc.ConfigureRTCPReports(interceptorRegistry); err != nil {
return err
}
// Setting engine
settingEngine := webrtc.SettingEngine{}

49
packages/relay/internal/core/protocol_stream.go
View File

@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"log/slog"
"math"
"relay/internal/common"
"relay/internal/connections"
"relay/internal/shared"
@@ -606,52 +605,8 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
}
}
// Calculate differences
var timeDiff int64
var sequenceDiff int
if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastVideoTimestamp)
if !room.VideoTimestampSet {
timeDiff = 0
room.VideoTimestampSet = true
} else if timeDiff < -(math.MaxUint32 / 10) {
timeDiff += math.MaxUint32 + 1
}
sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastVideoSequenceNumber)
if !room.VideoSequenceSet {
sequenceDiff = 0
room.VideoSequenceSet = true
} else if sequenceDiff < -(math.MaxUint16 / 10) {
sequenceDiff += math.MaxUint16 + 1
}
room.LastVideoTimestamp = rtpPacket.Timestamp
room.LastVideoSequenceNumber = rtpPacket.SequenceNumber
} else { // Audio
timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastAudioTimestamp)
if !room.AudioTimestampSet {
timeDiff = 0
room.AudioTimestampSet = true
} else if timeDiff < -(math.MaxUint32 / 10) {
timeDiff += math.MaxUint32 + 1
}
sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastAudioSequenceNumber)
if !room.AudioSequenceSet {
sequenceDiff = 0
room.AudioSequenceSet = true
} else if sequenceDiff < -(math.MaxUint16 / 10) {
sequenceDiff += math.MaxUint16 + 1
}
room.LastAudioTimestamp = rtpPacket.Timestamp
room.LastAudioSequenceNumber = rtpPacket.SequenceNumber
}
// Broadcast with differences
room.BroadcastPacketRetimed(remoteTrack.Kind(), rtpPacket, timeDiff, sequenceDiff)
// Broadcast
room.BroadcastPacket(remoteTrack.Kind(), rtpPacket)
}
slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())

13
packages/relay/internal/shared/participant.go
View File

@@ -106,28 +106,15 @@ func (p *Participant) Close() {
func (p *Participant) packetWriter() {
for pkt := range p.packetQueue {
var track *webrtc.TrackLocalStaticRTP
var sequenceNumber uint16
var timestamp uint32
// No mutex needed - only this goroutine modifies these
if pkt.kind == webrtc.RTPCodecTypeAudio {
track = p.AudioTrack
p.AudioSequenceNumber = uint16(int(p.AudioSequenceNumber) + pkt.sequenceDiff)
p.AudioTimestamp = uint32(int64(p.AudioTimestamp) + pkt.timeDiff)
sequenceNumber = p.AudioSequenceNumber
timestamp = p.AudioTimestamp
} else {
track = p.VideoTrack
p.VideoSequenceNumber = uint16(int(p.VideoSequenceNumber) + pkt.sequenceDiff)
p.VideoTimestamp = uint32(int64(p.VideoTimestamp) + pkt.timeDiff)
sequenceNumber = p.VideoSequenceNumber
timestamp = p.VideoTimestamp
}
if track != nil {
pkt.packet.SequenceNumber = sequenceNumber
pkt.packet.Timestamp = timestamp
if err := track.WriteRTP(pkt.packet); err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("WriteRTP failed", "participant", p.ID, "kind", pkt.kind, "err", err)
}

8
packages/relay/internal/shared/room.go
View File

@@ -21,8 +21,6 @@ var participantPacketPool = sync.Pool{
type participantPacket struct {
kind webrtc.RTPCodecType
packet *rtp.Packet
timeDiff int64
sequenceDiff int
}
type RoomInfo struct {
@@ -141,7 +139,7 @@ func (r *Room) IsOnline() bool {
return r.PeerConnection != nil
}
func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet, timeDiff int64, sequenceDiff int) {
func (r *Room) BroadcastPacket(kind webrtc.RTPCodecType, pkt *rtp.Packet) {
// Lock-free load of channel slice
channels := r.participantChannels.Load()
@@ -155,9 +153,7 @@ func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet,
// Get packet struct from pool
pp := participantPacketPool.Get().(*participantPacket)
pp.kind = kind
pp.packet = pkt.Clone()
pp.timeDiff = timeDiff
pp.sequenceDiff = sequenceDiff
pp.packet = pkt
select {
case ch <- pp:

14
packages/scripts/entrypoint.sh
View File

@@ -348,18 +348,8 @@ main() {
setup_namespaceless
fi
# Make sure /run/udev/ directory exists with /run/udev/control, needed for virtual controller support
if [[ ! -d "/run/udev" || ! -e "/run/udev/control" ]]; then
log "Creating /run/udev directory and control file.."
$ENTCMD_PREFIX mkdir -p /run/udev || {
log "Error: Failed to create /run/udev directory"
exit 1
}
$ENTCMD_PREFIX touch /run/udev/control || {
log "Error: Failed to create /run/udev/control file"
exit 1
}
fi
# Wait for vimputti socket before switching to application startup
wait_for_socket "/tmp/vimputti-0" "vimputti" || exit 1
# Switch to nestri runner entrypoint
log "Switching to application startup entrypoint..."

2
packages/scripts/entrypoint_nestri.sh
View File

@@ -108,7 +108,7 @@ start_compositor() {
# Set default compositor if unset
if [[ -z "${NESTRI_LAUNCH_COMPOSITOR+x}" ]]; then
NESTRI_LAUNCH_COMPOSITOR="gamescope --backend wayland --force-grab-cursor -g -f --rt -W ${WIDTH} -H ${HEIGHT} -r ${FRAMERATE:-60}"
NESTRI_LAUNCH_COMPOSITOR="gamescope --backend wayland -g -f --rt -W ${WIDTH} -H ${HEIGHT} -r ${FRAMERATE:-60}"
fi
# If PRELOAD_SHIM_arch's are set and exist, set LD_PRELOAD for 32/64-bit apps

6
packages/scripts/supervisord.conf
View File

@@ -18,7 +18,7 @@ autorestart=true
autostart=true
startretries=3
priority=3
nice=-10
nice=-2
environment=HOME=%(ENV_NESTRI_HOME)s,XDG_RUNTIME_DIR=%(ENV_NESTRI_XDG_RUNTIME_DIR)s
[program:pipewire-pulse]
@@ -28,7 +28,7 @@ autorestart=true
autostart=true
startretries=3
priority=4
nice=-10
nice=-2
environment=HOME=%(ENV_NESTRI_HOME)s,XDG_RUNTIME_DIR=%(ENV_NESTRI_XDG_RUNTIME_DIR)s
[program:wireplumber]
@@ -38,7 +38,7 @@ autorestart=true
autostart=true
startretries=3
priority=5
nice=-10
nice=-2
environment=HOME=%(ENV_NESTRI_HOME)s,XDG_RUNTIME_DIR=%(ENV_NESTRI_XDG_RUNTIME_DIR)s
[program:vimputti-manager]

739
packages/server/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

4
packages/server/Cargo.toml
View File

@@ -28,7 +28,7 @@ prost = "0.14"
prost-types = "0.14"
parking_lot = "0.12"
byteorder = "1.5"
libp2p = { version = "0.56", features = ["identify", "dns", "tcp", "noise", "ping", "tokio", "serde", "yamux", "macros", "websocket", "autonat"] }
libp2p = { version = "0.56", features = ["identify", "dns", "tcp", "noise", "ping", "tokio", "serde", "yamux", "macros", "autonat", "quic"] }
libp2p-identify = "0.47"
libp2p-ping = "0.47"
libp2p-autonat = { version = "0.15", features = ["v2"] }
@@ -37,7 +37,7 @@ libp2p-yamux = "0.47"
libp2p-noise = "0.46"
libp2p-dns = { version = "0.44", features = ["tokio"] }
libp2p-tcp = { version = "0.44", features = ["tokio"] }
libp2p-websocket = "0.45"
libp2p-quic = { version = "0.13", features = ["tokio"] }
dashmap = "6.1"
anyhow = "1.0"
unsigned-varint = "0.8"

24
packages/server/src/args.rs
View File

@@ -122,6 +122,14 @@ impl Args {
.value_parser(value_parser!(encoding_args::RateControlMethod))
.default_value("cbr"),
)
.arg(
Arg::new("video-latency-control")
.long("video-latency-control")
.env("VIDEO_LATENCY_CONTROL")
.help("Video latency control")
.value_parser(value_parser!(encoding_args::LatencyControl))
.default_value("lowest-latency"),
)
.arg(
Arg::new("video-cqp")
.long("video-cqp")
@@ -165,6 +173,14 @@ impl Args {
)
.default_value("8"),
)
.arg(
Arg::new("keyframe-dist-secs")
.long("keyframe-dist-secs")
.env("KEYFRAME_DIST_SECS")
.help("Distance between keyframes in seconds")
.value_parser(value_parser!(u32).range(1..))
.default_value("1"),
)
.arg(
Arg::new("audio-capture-method")
.long("audio-capture-method")
@@ -195,6 +211,14 @@ impl Args {
.value_parser(value_parser!(encoding_args::RateControlMethod))
.default_value("cbr"),
)
.arg(
Arg::new("audio-latency-control")
.long("audio-latency-control")
.env("AUDIO_LATENCY_CONTROL")
.help("Audio latency control")
.value_parser(value_parser!(encoding_args::LatencyControl))
.default_value("lowest-latency"),
)
.arg(
Arg::new("audio-bitrate")
.long("audio-bitrate")

30
packages/server/src/args/encoding_args.rs
View File

@@ -60,6 +60,12 @@ pub enum RateControl {
CBR(RateControlCBR),
}
#[derive(Debug, PartialEq, Eq, Clone, ValueEnum)]
pub enum LatencyControl {
LowestLatency,
HighestQuality,
}
pub struct EncodingOptionsBase {
/// Codec (e.g. "h264", "opus" etc.)
pub codec: Codec,
@@ -67,6 +73,8 @@ pub struct EncodingOptionsBase {
pub encoder: Option<String>,
/// Rate control method (e.g. "cqp", "vbr", "cbr")
pub rate_control: RateControl,
/// Latency control option, what to tweak settings towards (latency or quality)
pub latency_control: LatencyControl,
}
impl EncodingOptionsBase {
pub fn debug_print(&self) {
@@ -87,6 +95,14 @@ impl EncodingOptionsBase {
tracing::info!("-> Target Bitrate: {}", cbr.target_bitrate);
}
}
match &self.latency_control {
LatencyControl::LowestLatency => {
tracing::info!("> Latency Control: Priorizing lowest latency");
}
LatencyControl::HighestQuality => {
tracing::info!("> Latency Control: Priorizing quality at the cost of latency");
}
}
}
}
@@ -94,6 +110,7 @@ pub struct VideoEncodingOptions {
pub base: EncodingOptionsBase,
pub encoder_type: EncoderType,
pub bit_depth: u32,
pub keyframe_dist_secs: u32,
}
impl VideoEncodingOptions {
pub fn from_matches(matches: &clap::ArgMatches) -> Self {
@@ -125,6 +142,10 @@ impl VideoEncodingOptions {
max_bitrate: matches.get_one::<u32>("video-bitrate-max").unwrap().clone(),
}),
},
latency_control: matches
.get_one::<LatencyControl>("video-latency-control")
.unwrap_or(&LatencyControl::LowestLatency)
.clone(),
},
encoder_type: matches
.get_one::<EncoderType>("video-encoder-type")
@@ -134,6 +155,10 @@ impl VideoEncodingOptions {
.get_one::<u32>("video-bit-depth")
.copied()
.unwrap_or(8),
keyframe_dist_secs: matches
.get_one::<u32>("keyframe-dist-secs")
.copied()
.unwrap_or(1),
}
}
@@ -142,6 +167,7 @@ impl VideoEncodingOptions {
self.base.debug_print();
tracing::info!("> Encoder Type: {}", self.encoder_type.as_str());
tracing::info!("> Bit Depth: {}", self.bit_depth);
tracing::info!("> Keyframe Distance Seconds: {}", self.keyframe_dist_secs);
}
}
impl Deref for VideoEncodingOptions {
@@ -208,6 +234,10 @@ impl AudioEncodingOptions {
}),
wot => panic!("Invalid rate control method for audio: {}", wot.as_str()),
},
latency_control: matches
.get_one::<LatencyControl>("audio-latency-control")
.unwrap_or(&LatencyControl::LowestLatency)
.clone(),
},
capture_method: matches
.get_one::<AudioCaptureMethod>("audio-capture-method")

79
packages/server/src/enc_helper.rs
View File

@@ -74,7 +74,6 @@ pub enum EncoderAPI {
QSV,
VAAPI,
NVENC,
AMF,
SOFTWARE,
UNKNOWN,
}
@@ -85,7 +84,6 @@ impl EncoderAPI {
Self::QSV => "Intel QuickSync Video",
Self::VAAPI => "Video Acceleration API",
Self::NVENC => "NVIDIA NVENC",
Self::AMF => "AMD Media Framework",
Self::SOFTWARE => "Software",
Self::UNKNOWN => "Unknown",
}
@@ -167,8 +165,6 @@ fn get_encoder_api(encoder: &str, encoder_type: &EncoderType) -> EncoderAPI {
EncoderAPI::VAAPI
} else if encoder.starts_with("nv") {
EncoderAPI::NVENC
} else if encoder.starts_with("amf") {
EncoderAPI::AMF
} else {
EncoderAPI::UNKNOWN
}
@@ -275,9 +271,9 @@ pub fn encoder_low_latency_params(
encoder: &VideoEncoderInfo,
_rate_control: &RateControl,
framerate: u32,
keyframe_dist_secs: u32,
) -> VideoEncoderInfo {
// 1 second keyframe interval for fast recovery, is this too taxing?
let mut encoder_optz = encoder_gop_params(encoder, framerate);
let mut encoder_optz = encoder_gop_params(encoder, framerate * keyframe_dist_secs);
match encoder_optz.encoder_api {
EncoderAPI::QSV => {
@@ -293,16 +289,6 @@ pub fn encoder_low_latency_params(
encoder_optz.set_parameter("tune", "ultra-low-latency");
encoder_optz.set_parameter("zerolatency", "true");
}
EncoderAPI::AMF => {
encoder_optz.set_parameter("preset", "speed");
let usage = match encoder_optz.codec {
VideoCodec::H264 | VideoCodec::H265 => "ultra-low-latency",
VideoCodec::AV1 => "low-latency",
};
if !usage.is_empty() {
encoder_optz.set_parameter("usage", usage);
}
}
EncoderAPI::SOFTWARE => match encoder_optz.name.as_str() {
"openh264enc" => {
encoder_optz.set_parameter("complexity", "low");
@@ -330,6 +316,56 @@ pub fn encoder_low_latency_params(
encoder_optz
}
pub fn encoder_high_quality_params(
encoder: &VideoEncoderInfo,
_rate_control: &RateControl,
framerate: u32,
keyframe_dist_secs: u32,
) -> VideoEncoderInfo {
let mut encoder_optz = encoder_gop_params(encoder, framerate * keyframe_dist_secs);
match encoder_optz.encoder_api {
EncoderAPI::QSV => {
encoder_optz.set_parameter("low-latency", "false");
encoder_optz.set_parameter("target-usage", "1");
}
EncoderAPI::VAAPI => {
encoder_optz.set_parameter("target-usage", "1");
}
EncoderAPI::NVENC => {
encoder_optz.set_parameter("multi-pass", "two-pass");
encoder_optz.set_parameter("preset", "p7");
encoder_optz.set_parameter("tune", "high-quality");
encoder_optz.set_parameter("zerolatency", "false");
encoder_optz.set_parameter("spatial-aq", "true");
encoder_optz.set_parameter("rc-lookahead", "3");
}
EncoderAPI::SOFTWARE => match encoder_optz.name.as_str() {
"openh264enc" => {
encoder_optz.set_parameter("complexity", "high");
encoder_optz.set_parameter("usage-type", "screen");
}
"x264enc" => {
encoder_optz.set_parameter("rc-lookahead", "3");
encoder_optz.set_parameter("speed-preset", "medium");
}
"svtav1enc" => {
encoder_optz.set_parameter("preset", "8");
encoder_optz.set_parameter("parameters-string", "lookahead=3");
}
"av1enc" => {
encoder_optz.set_parameter("usage-profile", "realtime");
encoder_optz.set_parameter("cpu-used", "8");
encoder_optz.set_parameter("lag-in-frames", "3");
}
_ => {}
},
_ => {}
}
encoder_optz
}
pub fn get_compatible_encoders(gpus: &Vec<GPUInfo>) -> Vec<VideoEncoderInfo> {
let mut encoders = Vec::new();
let registry = gstreamer::Registry::get();
@@ -427,16 +463,6 @@ pub fn get_compatible_encoders(gpus: &Vec<GPUInfo>) -> Vec<VideoEncoderInfo> {
None
}
}
EncoderAPI::AMF if element.has_property("device") => {
let device_id = match element.property_value("device").get::<u32>() {
Ok(v) => Some(v as usize),
Err(_) => None,
};
device_id.and_then(|id| {
get_gpus_by_vendor(&gpus, GPUVendor::AMD).get(id).cloned()
})
}
_ => None,
}
})
@@ -549,7 +575,6 @@ pub fn get_best_compatible_encoder(
score += match encoder.encoder_api {
EncoderAPI::NVENC => 3,
EncoderAPI::QSV => 3,
EncoderAPI::AMF => 3,
EncoderAPI::VAAPI => 2,
EncoderAPI::SOFTWARE => 1,
EncoderAPI::UNKNOWN => 0,

98
packages/server/src/main.rs
View File

@@ -8,6 +8,7 @@ mod p2p;
mod proto;
use crate::args::encoding_args;
use crate::args::encoding_args::LatencyControl;
use crate::enc_helper::{EncoderAPI, EncoderType};
use crate::gpu::{GPUInfo, GPUVendor};
use crate::input::controller::ControllerManager;
@@ -130,11 +131,20 @@ fn handle_encoder_video_settings(
args: &args::Args,
video_encoder: &enc_helper::VideoEncoderInfo,
) -> enc_helper::VideoEncoderInfo {
let mut optimized_encoder = enc_helper::encoder_low_latency_params(
&video_encoder,
&args.encoding.video.rate_control,
args.app.framerate,
);
let mut optimized_encoder = match args.encoding.video.latency_control {
LatencyControl::LowestLatency => enc_helper::encoder_low_latency_params(
&video_encoder,
&args.encoding.video.rate_control,
args.app.framerate,
args.encoding.video.keyframe_dist_secs,
),
LatencyControl::HighestQuality => enc_helper::encoder_high_quality_params(
&video_encoder,
&args.encoding.video.rate_control,
args.app.framerate,
args.encoding.video.keyframe_dist_secs,
),
};
// Handle rate-control method
match &args.encoding.video.rate_control {
encoding_args::RateControl::CQP(cqp) => {
@@ -429,39 +439,34 @@ async fn main() -> Result<(), Box<dyn Error>> {
webrtcsink.set_property("do-retransmission", false);
/* Queues */
let video_queue = gstreamer::ElementFactory::make("queue")
// Sink queues
let video_sink_queue = gstreamer::ElementFactory::make("queue").build()?;
let audio_sink_queue = gstreamer::ElementFactory::make("queue").build()?;
// Source queues
let video_source_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 2u32)
.property("max-size-time", 0u64)
.property("max-size-bytes", 0u32)
.build()?;
let audio_queue = gstreamer::ElementFactory::make("queue")
let audio_source_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 2u32)
.property("max-size-time", 0u64)
.property("max-size-bytes", 0u32)
.build()?;
/* Clock Sync */
let video_clocksync = gstreamer::ElementFactory::make("clocksync")
.property("sync-to-first", true)
.build()?;
let audio_clocksync = gstreamer::ElementFactory::make("clocksync")
.property("sync-to-first", true)
.build()?;
// Add elements to the pipeline
pipeline.add_many(&[
webrtcsink.upcast_ref(),
&video_sink_queue,
&audio_sink_queue,
&video_encoder,
&caps_filter,
&video_queue,
&video_clocksync,
&video_source_queue,
&video_source,
&audio_encoder,
&audio_capsfilter,
&audio_queue,
&audio_clocksync,
&audio_source_queue,
&audio_rate,
&audio_converter,
&audio_source,
@@ -493,16 +498,24 @@ async fn main() -> Result<(), Box<dyn Error>> {
&audio_converter,
&audio_rate,
&audio_capsfilter,
&audio_queue,
&audio_clocksync,
&audio_source_queue,
&audio_encoder,
])?;
// Link audio parser to audio encoder if present, otherwise just webrtcsink
if let Some(parser) = &audio_parser {
gstreamer::Element::link_many(&[&audio_encoder, parser, webrtcsink.upcast_ref()])?;
gstreamer::Element::link_many(&[
&audio_encoder,
parser,
&audio_sink_queue,
webrtcsink.upcast_ref(),
])?;
} else {
gstreamer::Element::link_many(&[&audio_encoder, webrtcsink.upcast_ref()])?;
gstreamer::Element::link_many(&[
&audio_encoder,
&audio_sink_queue,
webrtcsink.upcast_ref(),
])?;
}
// With zero-copy..
@@ -512,26 +525,20 @@ async fn main() -> Result<(), Box<dyn Error>> {
gstreamer::Element::link_many(&[
&video_source,
&caps_filter,
&video_queue,
&video_clocksync,
&video_source_queue,
&vapostproc,
&va_caps_filter,
&video_encoder,
])?;
} else if video_encoder_info.encoder_api == EncoderAPI::NVENC {
// NVENC pipeline
gstreamer::Element::link_many(&[
&video_source,
&caps_filter,
&video_encoder,
])?;
gstreamer::Element::link_many(&[&video_source, &caps_filter, &video_encoder])?;
}
} else {
gstreamer::Element::link_many(&[
&video_source,
&caps_filter,
&video_queue,
&video_clocksync,
&video_source_queue,
&video_converter.unwrap(),
&video_encoder,
])?;
@@ -539,21 +546,24 @@ async fn main() -> Result<(), Box<dyn Error>> {
// Link video parser if present with webrtcsink, otherwise just link webrtc sink
if let Some(parser) = &video_parser {
gstreamer::Element::link_many(&[&video_encoder, parser, webrtcsink.upcast_ref()])?;
gstreamer::Element::link_many(&[
&video_encoder,
parser,
&video_sink_queue,
webrtcsink.upcast_ref(),
])?;
} else {
gstreamer::Element::link_many(&[&video_encoder, webrtcsink.upcast_ref()])?;
gstreamer::Element::link_many(&[
&video_encoder,
&video_sink_queue,
webrtcsink.upcast_ref(),
])?;
}
// Make sure QOS is disabled to avoid latency
video_encoder.set_property("qos", true);
video_source.set_property("do-timestamp", &false);
audio_source.set_property("do-timestamp", &false);
// Optimize latency of pipeline
video_source
.sync_state_with_parent()
.expect("failed to sync with parent");
video_source.set_property("do-timestamp", &true);
audio_source.set_property("do-timestamp", &true);
pipeline.set_property("latency", &0u64);
pipeline.set_property("async-handling", true);
pipeline.set_property("message-forward", true);

3
packages/server/src/p2p/p2p.rs
View File

@@ -55,9 +55,8 @@ impl NestriP2P {
noise::Config::new,
yamux::Config::default,
)?
.with_quic()
.with_dns()?
.with_websocket(noise::Config::new, yamux::Config::default)
.await?
.with_behaviour(|key| NestriBehaviour::new(key.public()))?
.build(),
));

74
packages/server/src/proto/proto.rs
View File

@@ -1,14 +1,12 @@
// @generated
// This file is @generated by prost-build.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoTimestampEntry {
#[prost(string, tag="1")]
pub stage: ::prost::alloc::string::String,
#[prost(message, optional, tag="2")]
pub time: ::core::option::Option<::prost_types::Timestamp>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoLatencyTracker {
#[prost(string, tag="1")]
@@ -19,8 +17,7 @@ pub struct ProtoLatencyTracker {
// Mouse messages
/// MouseMove message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoMouseMove {
#[prost(int32, tag="1")]
pub x: i32,
@@ -28,8 +25,7 @@ pub struct ProtoMouseMove {
pub y: i32,
}
/// MouseMoveAbs message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoMouseMoveAbs {
#[prost(int32, tag="1")]
pub x: i32,
@@ -37,8 +33,7 @@ pub struct ProtoMouseMoveAbs {
pub y: i32,
}
/// MouseWheel message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoMouseWheel {
#[prost(int32, tag="1")]
pub x: i32,
@@ -46,15 +41,13 @@ pub struct ProtoMouseWheel {
pub y: i32,
}
/// MouseKeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoMouseKeyDown {
#[prost(int32, tag="1")]
pub key: i32,
}
/// MouseKeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoMouseKeyUp {
#[prost(int32, tag="1")]
pub key: i32,
@@ -62,24 +55,26 @@ pub struct ProtoMouseKeyUp {
// Keyboard messages
/// KeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoKeyDown {
#[prost(int32, tag="1")]
pub key: i32,
}
/// KeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoKeyUp {
#[prost(int32, tag="1")]
pub key: i32,
}
// Clipboard message
// message ProtoClipboard {
// string content = 1; // Clipboard content
// }
// Controller messages
/// ControllerAttach message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoControllerAttach {
/// One of the following enums: "ps", "xbox" or "switch"
#[prost(string, tag="1")]
@@ -92,8 +87,7 @@ pub struct ProtoControllerAttach {
pub session_id: ::prost::alloc::string::String,
}
/// ControllerDetach message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoControllerDetach {
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
@@ -103,8 +97,7 @@ pub struct ProtoControllerDetach {
pub session_id: ::prost::alloc::string::String,
}
/// ControllerRumble message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoControllerRumble {
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
@@ -123,7 +116,6 @@ pub struct ProtoControllerRumble {
pub duration: i32,
}
/// ControllerStateBatch - single message containing full or partial controller state
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerStateBatch {
/// Session specific slot number (0-3)
@@ -188,8 +180,8 @@ pub mod proto_controller_state_batch {
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
pub fn as_str_name(&self) -> &'static str {
match self {
UpdateType::FullState => "FULL_STATE",
UpdateType::Delta => "DELTA",
Self::FullState => "FULL_STATE",
Self::Delta => "DELTA",
}
}
/// Creates an enum from field names used in the ProtoBuf definition.
@@ -204,8 +196,7 @@ pub mod proto_controller_state_batch {
}
// WebRTC + signaling
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RtcIceCandidateInit {
#[prost(string, tag="1")]
pub candidate: ::prost::alloc::string::String,
@@ -216,8 +207,7 @@ pub struct RtcIceCandidateInit {
#[prost(string, optional, tag="4")]
pub username_fragment: ::core::option::Option<::prost::alloc::string::String>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct RtcSessionDescriptionInit {
#[prost(string, tag="1")]
pub sdp: ::prost::alloc::string::String,
@@ -225,29 +215,25 @@ pub struct RtcSessionDescriptionInit {
pub r#type: ::prost::alloc::string::String,
}
/// ProtoICE message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoIce {
#[prost(message, optional, tag="1")]
pub candidate: ::core::option::Option<RtcIceCandidateInit>,
}
/// ProtoSDP message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoSdp {
#[prost(message, optional, tag="1")]
pub sdp: ::core::option::Option<RtcSessionDescriptionInit>,
}
/// ProtoRaw message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoRaw {
#[prost(string, tag="1")]
pub data: ::prost::alloc::string::String,
}
/// ProtoClientRequestRoomStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoClientRequestRoomStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
@@ -255,8 +241,7 @@ pub struct ProtoClientRequestRoomStream {
pub session_id: ::prost::alloc::string::String,
}
/// ProtoClientDisconnected message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoClientDisconnected {
#[prost(string, tag="1")]
pub session_id: ::prost::alloc::string::String,
@@ -264,13 +249,11 @@ pub struct ProtoClientDisconnected {
pub controller_slots: ::prost::alloc::vec::Vec<i32>,
}
/// ProtoServerPushStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ProtoServerPushStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoMessageBase {
#[prost(string, tag="1")]
@@ -278,7 +261,6 @@ pub struct ProtoMessageBase {
#[prost(message, optional, tag="2")]
pub latency: ::core::option::Option<ProtoLatencyTracker>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoMessage {
#[prost(message, optional, tag="1")]
@@ -288,8 +270,7 @@ pub struct ProtoMessage {
}
/// Nested message and enum types in `ProtoMessage`.
pub mod proto_message {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Payload {
/// Input types
#[prost(message, tag="2")]
@@ -304,6 +285,7 @@ pub mod proto_message {
MouseKeyUp(super::ProtoMouseKeyUp),
#[prost(message, tag="7")]
KeyDown(super::ProtoKeyDown),
/// ProtoClipboard clipboard = 9;
#[prost(message, tag="8")]
KeyUp(super::ProtoKeyUp),
/// Controller input types