github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

⭐ feat: Migrate from WebSocket to libp2p for peer-to-peer connectivity (#286)

Browse Source 
## Description
Whew, some stuff is still not re-implemented, but it's working!

Rabbit's gonna explode with the amount of changes I reckon 😅



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Introduced a peer-to-peer relay system using libp2p with enhanced
stream forwarding, room state synchronization, and mDNS peer discovery.
- Added decentralized room and participant management, metrics
publishing, and safe, size-limited, concurrent message streaming with
robust framing and callback dispatching.
- Implemented asynchronous, callback-driven message handling over custom
libp2p streams replacing WebSocket signaling.
- **Improvements**
- Migrated signaling and stream protocols from WebSocket to libp2p,
improving reliability and scalability.
- Simplified configuration and environment variables, removing
deprecated flags and adding persistent data support.
- Enhanced logging, error handling, and connection management for better
observability and robustness.
- Refined RTP header extension registration and NAT IP handling for
improved WebRTC performance.
- **Bug Fixes**
- Improved ICE candidate buffering and SDP negotiation in WebRTC
connections.
  - Fixed NAT IP and UDP port range configuration issues.
- **Refactor**
- Modularized codebase, reorganized relay and server logic, and removed
deprecated WebSocket-based components.
- Streamlined message structures, removed obsolete enums and message
types, and simplified SafeMap concurrency.
- Replaced WebSocket signaling with libp2p stream protocols in server
and relay components.
- **Chores**
- Updated and cleaned dependencies across Go, Rust, and JavaScript
packages.
  - Added `.gitignore` for persistent data directory in relay package.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Philipp Neumann <3daquawolf@gmail.com>
This commit is contained in:
Kristian Ollikainen
2025-06-06 16:48:49 +03:00
committed by GitHub
parent e67a8d2b32
commit 6e82eff9e2
48 changed files with 4741 additions and 2787 deletions
Download Patch File Download Diff File Expand all files Collapse all files

305
packages/input/src/webrtc-stream.ts
View File

@@ -1,18 +1,27 @@
import {
MessageBase,
MessageICE,
MessageJoin,
MessageSDP,
MessageAnswer,
JoinerType,
AnswerType,
NewMessageRaw,
NewMessageSDP,
NewMessageICE,
SafeStream,
} from "./messages";
import { webSockets } from "@libp2p/websockets";
import { createLibp2p, Libp2p } from "libp2p";
import { noise } from "@chainsafe/libp2p-noise";
import { yamux } from "@chainsafe/libp2p-yamux";
import { identify } from "@libp2p/identify";
import { multiaddr } from "@multiformats/multiaddr";
import { Connection } from "@libp2p/interface";
import { ping } from "@libp2p/ping";
//FIXME: Sometimes the room will wait to say offline, then appear to be online after retrying :D
// This works for me, with my trashy internet, does it work for you as well?
const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
export class WebRTCStream {
private _ws: WebSocket | undefined = undefined;
private _p2p: Libp2p | undefined = undefined;
private _p2pConn: Connection | undefined = undefined;
private _p2pSafeStream: SafeStream | undefined = undefined;
private _pc: RTCPeerConnection | undefined = undefined;
private _audioTrack: MediaStreamTrack | undefined = undefined;
private _videoTrack: MediaStreamTrack | undefined = undefined;
@@ -24,7 +33,11 @@ export class WebRTCStream {
private _isConnected: boolean = false; // Add flag to track connection state
currentFrameRate: number = 60;
constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) {
constructor(
serverURL: string,
roomName: string,
connectedCallback: (stream: MediaStream | null) => void,
) {
if (roomName.length <= 0) {
console.error("Room name not provided");
return;
@@ -33,120 +46,114 @@ export class WebRTCStream {
this._onConnected = connectedCallback;
this._serverURL = serverURL;
this._roomName = roomName;
this._setup(serverURL, roomName);
this._setup(serverURL, roomName).catch(console.error);
}
private _setup(serverURL: string, roomName: string) {
private async _setup(serverURL: string, roomName: string) {
// Don't setup new connection if already connected
if (this._isConnected) {
console.log("Already connected, skipping setup");
return;
}
console.log("Setting up WebSocket");
const wsURL = serverURL.replace(/^http/, "ws");
this._ws = new WebSocket(`${wsURL}/api/ws/${roomName}`);
this._ws.onopen = async () => {
console.log("WebSocket opened");
// Send join message
const joinMessage: MessageJoin = {
payload_type: "join",
joiner_type: JoinerType.JoinerClient
};
this._ws!.send(JSON.stringify(joinMessage));
}
console.log("Setting up libp2p");
let iceHolder: RTCIceCandidateInit[] = [];
this._p2p = await createLibp2p({
transports: [webSockets()],
connectionEncrypters: [noise()],
streamMuxers: [yamux()],
connectionGater: {
denyDialMultiaddr: () => {
return false;
},
},
services: {
identify: identify(),
ping: ping(),
},
});
this._ws.onmessage = async (e) => {
// allow only JSON
if (typeof e.data === "object") return;
if (!e.data) return;
const message = JSON.parse(e.data) as MessageBase;
switch (message.payload_type) {
case "sdp":
this._p2p.addEventListener("peer:connect", async (e) => {
console.debug("Peer connected:", e.detail);
});
this._p2p.addEventListener("peer:disconnect", (e) => {
console.debug("Peer disconnected:", e.detail);
});
const ma = multiaddr(serverURL);
console.debug("Dialing peer at:", ma.toString());
this._p2pConn = await this._p2p.dial(ma);
if (this._p2pConn) {
console.log("Stream is being established");
let stream = await this._p2pConn
.newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
.catch(console.error);
if (stream) {
this._p2pSafeStream = new SafeStream(stream);
console.log("Stream opened with peer");
let iceHolder: RTCIceCandidateInit[] = [];
this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
if (this._pc) {
if (this._pc.remoteDescription) {
this._pc.addIceCandidate(data.candidate).catch((err) => {
console.error("Error adding ICE candidate:", err);
});
// Add held candidates
iceHolder.forEach((candidate) => {
this._pc!.addIceCandidate(candidate).catch((err) => {
console.error("Error adding held ICE candidate:", err);
});
});
iceHolder = [];
} else {
iceHolder.push(data.candidate);
}
} else {
iceHolder.push(data.candidate);
}
});
this._p2pSafeStream.registerCallback("offer", async (data) => {
if (!this._pc) {
// Setup peer connection now
this._setupPeerConnection();
}
console.log("Received SDP: ", (message as MessageSDP).sdp);
await this._pc!.setRemoteDescription((message as MessageSDP).sdp);
await this._pc!.setRemoteDescription(data.sdp);
// Create our answer
const answer = await this._pc!.createAnswer();
// Force stereo in Chromium browsers
answer.sdp = this.forceOpusStereo(answer.sdp!);
await this._pc!.setLocalDescription(answer);
this._ws!.send(JSON.stringify({
payload_type: "sdp",
sdp: answer
}));
break;
case "ice":
if (!this._pc) break;
if (this._pc.remoteDescription) {
try {
await this._pc.addIceCandidate((message as MessageICE).candidate);
// Add held ICE candidates
for (const ice of iceHolder) {
try {
await this._pc.addIceCandidate(ice);
} catch (e) {
console.error("Error adding held ICE candidate: ", e);
}
}
iceHolder = [];
} catch (e) {
console.error("Error adding ICE candidate: ", e);
}
} else {
iceHolder.push((message as MessageICE).candidate);
}
break;
case "answer":
switch ((message as MessageAnswer).answer_type) {
case AnswerType.AnswerOffline:
console.log("Room is offline");
// Call callback with null stream
if (this._onConnected)
this._onConnected(null);
// Send answer back
const answerMsg = NewMessageSDP("answer", answer);
await this._p2pSafeStream?.writeMessage(answerMsg);
});
break;
case AnswerType.AnswerInUse:
console.warn("Room is in use, we shouldn't even be getting this message");
break;
case AnswerType.AnswerOK:
console.log("Joining Room was successful");
break;
}
break;
default:
console.error("Unknown message type: ", message);
this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
console.warn("Stream is offline for room:", data.roomName);
this._onConnected?.(null);
});
// Send stream request
// marshal room name into json
const request = NewMessageRaw(
"request-stream-room",
roomName,
);
await this._p2pSafeStream.writeMessage(request);
}
}
this._ws.onclose = () => {
console.log("WebSocket closed, reconnecting in 3 seconds");
if (this._onConnected)
this._onConnected(null);
// Clear PeerConnection
this._cleanupPeerConnection()
this._handleConnectionFailure()
// setTimeout(() => {
// this._setup(serverURL, roomName);
// }, this._connectionTimeout);
}
this._ws.onerror = (e) => {
console.error("WebSocket error: ", e);
}
}
// Forces opus to stereo in Chromium browsers, because of course
private forceOpusStereo(SDP: string): string {
// Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
return SDP.replace(/(minptime=10;useinbandfec=1)/, "$1;stereo=1;sprop-stereo=1;");
return SDP.replace(
/(minptime=10;useinbandfec=1)/,
"$1;stereo=1;sprop-stereo=1;",
);
}
private _setupPeerConnection() {
@@ -158,43 +165,50 @@ export class WebRTCStream {
this._pc = new RTCPeerConnection({
iceServers: [
{
urls: "stun:stun.l.google.com:19302"
}
urls: "stun:stun.l.google.com:19302",
},
],
});
this._pc.ontrack = (e) => {
console.log("Track received: ", e.track);
if (e.track.kind === "audio")
this._audioTrack = e.track;
else if (e.track.kind === "video")
this._videoTrack = e.track;
console.debug("Track received: ", e.track);
if (e.track.kind === "audio") this._audioTrack = e.track;
else if (e.track.kind === "video") this._videoTrack = e.track;
this._checkConnectionState();
};
this._pc.onconnectionstatechange = () => {
console.log("Connection state changed to: ", this._pc!.connectionState);
console.debug("Connection state changed to: ", this._pc!.connectionState);
this._checkConnectionState();
};
this._pc.oniceconnectionstatechange = () => {
console.log("ICE connection state changed to: ", this._pc!.iceConnectionState);
console.debug(
"ICE connection state changed to: ",
this._pc!.iceConnectionState,
);
this._checkConnectionState();
};
this._pc.onicegatheringstatechange = () => {
console.log("ICE gathering state changed to: ", this._pc!.iceGatheringState);
console.debug(
"ICE gathering state changed to: ",
this._pc!.iceGatheringState,
);
this._checkConnectionState();
};
this._pc.onicecandidate = (e) => {
if (e.candidate) {
const message: MessageICE = {
payload_type: "ice",
candidate: e.candidate
};
this._ws!.send(JSON.stringify(message));
const iceMsg = NewMessageICE("ice-candidate", e.candidate);
if (this._p2pSafeStream) {
this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
console.error("Error sending ICE candidate:", err),
);
} else {
console.warn("P2P stream not established, cannot send ICE candidate");
}
}
};
@@ -207,26 +221,35 @@ export class WebRTCStream {
private _checkConnectionState() {
if (!this._pc) return;
console.log("Checking connection state:", {
console.debug("Checking connection state:", {
connectionState: this._pc.connectionState,
iceConnectionState: this._pc.iceConnectionState,
hasAudioTrack: !!this._audioTrack,
hasVideoTrack: !!this._videoTrack,
isConnected: this._isConnected
isConnected: this._isConnected,
});
if (this._pc.connectionState === "connected" && this._audioTrack !== undefined && this._videoTrack !== undefined) {
if (
this._pc.connectionState === "connected" &&
this._audioTrack !== undefined &&
this._videoTrack !== undefined
) {
this._clearConnectionTimer();
if (!this._isConnected) {
// Only trigger callback if not already connected
this._isConnected = true;
if (this._onConnected !== undefined) {
this._onConnected(new MediaStream([this._audioTrack, this._videoTrack]));
this._onConnected(
new MediaStream([this._audioTrack, this._videoTrack]),
);
// Continuously set low-latency target
this._pc.getReceivers().forEach((receiver: RTCRtpReceiver) => {
let intervalLoop = setInterval(async () => {
if (receiver.track.readyState !== "live" || (receiver.transport && receiver.transport.state !== "connected")) {
if (
receiver.track.readyState !== "live" ||
(receiver.transport && receiver.transport.state !== "connected")
) {
clearInterval(intervalLoop);
return;
} else {
@@ -239,9 +262,11 @@ export class WebRTCStream {
}
this._gatherFrameRate();
} else if (this._pc.connectionState === "failed" ||
} else if (
this._pc.connectionState === "failed" ||
this._pc.connectionState === "closed" ||
this._pc.iceConnectionState === "failed") {
this._pc.iceConnectionState === "failed"
) {
console.log("Connection failed or closed, attempting reconnect");
this._isConnected = false; // Reset connected state
this._handleConnectionFailure();
@@ -250,7 +275,8 @@ export class WebRTCStream {
private _handleConnectionFailure() {
this._clearConnectionTimer();
if (this._isConnected) { // Only notify if previously connected
if (this._isConnected) {
// Only notify if previously connected
this._isConnected = false;
if (this._onConnected) {
this._onConnected(null);
@@ -260,7 +286,7 @@ export class WebRTCStream {
// Attempt to reconnect only if not already connected
if (!this._isConnected && this._serverURL && this._roomName) {
this._setup(this._serverURL, this._roomName);
this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
}
}
@@ -276,10 +302,8 @@ export class WebRTCStream {
if (this._audioTrack || this._videoTrack) {
try {
if (this._audioTrack)
this._audioTrack.stop();
if (this._videoTrack)
this._videoTrack.stop();
if (this._audioTrack) this._audioTrack.stop();
if (this._videoTrack) this._videoTrack.stop();
} catch (err) {
console.error("Error stopping media tracks:", err);
}
@@ -308,16 +332,18 @@ export class WebRTCStream {
private _setupDataChannelEvents() {
if (!this._dataChannel) return;
this._dataChannel.onclose = () => console.log('sendChannel has closed')
this._dataChannel.onopen = () => console.log('sendChannel has opened')
this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`)
this._dataChannel.onclose = () => console.log("sendChannel has closed");
this._dataChannel.onopen = () => console.log("sendChannel has opened");
this._dataChannel.onmessage = (e) =>
console.log(
`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`,
);
}
private _gatherFrameRate() {
if (this._pc === undefined || this._videoTrack === undefined)
return;
if (this._pc === undefined || this._videoTrack === undefined) return;
const videoInfoPromise = new Promise<{ fps: number}>((resolve) => {
const videoInfoPromise = new Promise<{ fps: number }>((resolve) => {
// Keep trying to get fps until it's found
const interval = setInterval(async () => {
if (this._pc === undefined) {
@@ -329,7 +355,7 @@ export class WebRTCStream {
stats.forEach((report) => {
if (report.type === "inbound-rtp") {
clearInterval(interval);
resolve({ fps: report.framesPerSecond });
}
});
@@ -337,25 +363,26 @@ export class WebRTCStream {
});
videoInfoPromise.then((value) => {
this.currentFrameRate = value.fps
})
this.currentFrameRate = value.fps;
});
}
// Send binary message through the data channel
public sendBinary(data: Uint8Array) {
if (this._dataChannel && this._dataChannel.readyState === "open")
this._dataChannel.send(data);
else
console.log("Data channel not open or not established.");
else console.log("Data channel not open or not established.");
}
public disconnect() {
this._clearConnectionTimer();
this._cleanupPeerConnection();
if (this._ws) {
this._ws.close();
this._ws = undefined;
if (this._p2pConn) {
this._p2pConn
.close()
.catch((err) => console.error("Error closing P2P connection:", err));
this._p2pConn = undefined;
}
this._isConnected = false;
}
}
}