fix: Fix good chunk of packet loss and frame drop issues

- Also added new latency control parameter, not super visible differences, but it's cool :)

packages/relay/internal/common/common.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"fmt"
+	"github.com/pion/interceptor/pkg/nack"
 	"log/slog"
 	"strconv"
 
@@ -30,20 +31,121 @@ func InitWebRTCAPI() error {
 		return fmt.Errorf("failed to register extensions: %w", err)
 	}
 
-	// Default codecs cover our needs
-	err = mediaEngine.RegisterDefaultCodecs()
-	if err != nil {
-		return err
+	// Register codecs
+	for _, codec := range []webrtc.RTPCodecParameters{
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
+			PayloadType:        111,
+		},
+	} {
+		if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil {
+			return err
+		}
+	}
+
+	videoRTCPFeedback := []webrtc.RTCPFeedback{{"nack", ""}, {"nack", "pli"}}
+	for _, codec := range []webrtc.RTPCodecParameters{
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 102,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 104,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 106,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 108,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 127,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeH264,
+				ClockRate:    90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 39,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeH265,
+				ClockRate:    90000,
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 116,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeAV1, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback},
+			PayloadType:        45,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: videoRTCPFeedback},
+			PayloadType:        98,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=2", RTCPFeedback: videoRTCPFeedback},
+			PayloadType:        100,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f",
+				RTCPFeedback: videoRTCPFeedback,
+			},
+			PayloadType: 112,
+		},
+	} {
+		if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+			return err
+		}
 	}
 
 	// Interceptor registry
 	interceptorRegistry := &interceptor.Registry{}
 
-	// Use default set
-	err = webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry)
+	// Register our interceptors..
+	nackGenFactory, err := nack.NewGeneratorInterceptor()
 	if err != nil {
 		return err
 	}
+	interceptorRegistry.Add(nackGenFactory)
+	nackRespFactory, err := nack.NewResponderInterceptor()
+	if err != nil {
+		return err
+	}
+	interceptorRegistry.Add(nackRespFactory)
+
+	if err = webrtc.ConfigureRTCPReports(interceptorRegistry); err != nil {
+		return err
+	}
 
 	// Setting engine
 	settingEngine := webrtc.SettingEngine{}

packages/relay/internal/core/protocol_stream.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
-	"math"
 	"relay/internal/common"
 	"relay/internal/connections"
 	"relay/internal/shared"
@@ -606,52 +605,8 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 							}
 						}
 
-						// Calculate differences
-						var timeDiff int64
-						var sequenceDiff int
-
-						if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
-							timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastVideoTimestamp)
-							if !room.VideoTimestampSet {
-								timeDiff = 0
-								room.VideoTimestampSet = true
-							} else if timeDiff < -(math.MaxUint32 / 10) {
-								timeDiff += math.MaxUint32 + 1
-							}
-
-							sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastVideoSequenceNumber)
-							if !room.VideoSequenceSet {
-								sequenceDiff = 0
-								room.VideoSequenceSet = true
-							} else if sequenceDiff < -(math.MaxUint16 / 10) {
-								sequenceDiff += math.MaxUint16 + 1
-							}
-
-							room.LastVideoTimestamp = rtpPacket.Timestamp
-							room.LastVideoSequenceNumber = rtpPacket.SequenceNumber
-						} else { // Audio
-							timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastAudioTimestamp)
-							if !room.AudioTimestampSet {
-								timeDiff = 0
-								room.AudioTimestampSet = true
-							} else if timeDiff < -(math.MaxUint32 / 10) {
-								timeDiff += math.MaxUint32 + 1
-							}
-
-							sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastAudioSequenceNumber)
-							if !room.AudioSequenceSet {
-								sequenceDiff = 0
-								room.AudioSequenceSet = true
-							} else if sequenceDiff < -(math.MaxUint16 / 10) {
-								sequenceDiff += math.MaxUint16 + 1
-							}
-
-							room.LastAudioTimestamp = rtpPacket.Timestamp
-							room.LastAudioSequenceNumber = rtpPacket.SequenceNumber
-						}
-
-						// Broadcast with differences
-						room.BroadcastPacketRetimed(remoteTrack.Kind(), rtpPacket, timeDiff, sequenceDiff)
+						// Broadcast
+						room.BroadcastPacket(remoteTrack.Kind(), rtpPacket)
 					}
 
 					slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())

packages/relay/internal/shared/participant.go

@@ -106,28 +106,15 @@ func (p *Participant) Close() {
 func (p *Participant) packetWriter() {
 	for pkt := range p.packetQueue {
 		var track *webrtc.TrackLocalStaticRTP
-		var sequenceNumber uint16
-		var timestamp uint32
 
 		// No mutex needed - only this goroutine modifies these
 		if pkt.kind == webrtc.RTPCodecTypeAudio {
 			track = p.AudioTrack
-			p.AudioSequenceNumber = uint16(int(p.AudioSequenceNumber) + pkt.sequenceDiff)
-			p.AudioTimestamp = uint32(int64(p.AudioTimestamp) + pkt.timeDiff)
-			sequenceNumber = p.AudioSequenceNumber
-			timestamp = p.AudioTimestamp
 		} else {
 			track = p.VideoTrack
-			p.VideoSequenceNumber = uint16(int(p.VideoSequenceNumber) + pkt.sequenceDiff)
-			p.VideoTimestamp = uint32(int64(p.VideoTimestamp) + pkt.timeDiff)
-			sequenceNumber = p.VideoSequenceNumber
-			timestamp = p.VideoTimestamp
 		}
 
 		if track != nil {
-			pkt.packet.SequenceNumber = sequenceNumber
-			pkt.packet.Timestamp = timestamp
-
 			if err := track.WriteRTP(pkt.packet); err != nil && !errors.Is(err, io.ErrClosedPipe) {
 				slog.Error("WriteRTP failed", "participant", p.ID, "kind", pkt.kind, "err", err)
 			}

packages/relay/internal/shared/room.go

@@ -21,8 +21,6 @@ var participantPacketPool = sync.Pool{
 type participantPacket struct {
 	kind         webrtc.RTPCodecType
 	packet       *rtp.Packet
-	timeDiff     int64
-	sequenceDiff int
 }
 
 type RoomInfo struct {
@@ -141,7 +139,7 @@ func (r *Room) IsOnline() bool {
 	return r.PeerConnection != nil
 }
 
-func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet, timeDiff int64, sequenceDiff int) {
+func (r *Room) BroadcastPacket(kind webrtc.RTPCodecType, pkt *rtp.Packet) {
 	// Lock-free load of channel slice
 	channels := r.participantChannels.Load()
 
@@ -155,9 +153,7 @@ func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet,
 		// Get packet struct from pool
 		pp := participantPacketPool.Get().(*participantPacket)
 		pp.kind = kind
-		pp.packet = pkt.Clone()
-		pp.timeDiff = timeDiff
-		pp.sequenceDiff = sequenceDiff
+		pp.packet = pkt
 
 		select {
 		case ch <- pp: