github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Restructure protobufs and use them everywhere

Browse Source
This commit is contained in:
DatCaptainHorse
2025-10-21 18:41:45 +03:00
parent 32341574dc
commit 67f9a7d0a0
37 changed files with 3455 additions and 3074 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/input/package.json
View File

@@ -7,24 +7,22 @@
".": "./src/index.ts"
},
"devDependencies": {
"@bufbuild/buf": "^1.57.2",
"@bufbuild/protoc-gen-es": "^2.9.0"
"@bufbuild/buf": "^1.59.0",
"@bufbuild/protoc-gen-es": "^2.10.0"
},
"dependencies": {
"@bufbuild/protobuf": "^2.9.0",
"@chainsafe/libp2p-noise": "^16.1.4",
"@bufbuild/protobuf": "^2.10.0",
"@chainsafe/libp2p-noise": "^17.0.0",
"@chainsafe/libp2p-quic": "^1.1.3",
"@chainsafe/libp2p-yamux": "^7.0.4",
"@libp2p/identify": "^3.0.39",
"@libp2p/interface": "^2.11.0",
"@libp2p/ping": "^2.0.37",
"@libp2p/websockets": "^9.2.19",
"@libp2p/webtransport": "^5.0.51",
"@multiformats/multiaddr": "^12.5.1",
"it-length-prefixed": "^10.0.1",
"it-pipe": "^3.0.1",
"libp2p": "^2.10.0",
"uint8arraylist": "^2.4.8",
"uint8arrays": "^5.1.0"
"@chainsafe/libp2p-yamux": "^8.0.1",
"@libp2p/identify": "^4.0.5",
"@libp2p/interface": "^3.0.2",
"@libp2p/ping": "^3.0.5",
"@libp2p/websockets": "^10.0.6",
"@libp2p/webtransport": "^6.0.7",
"@libp2p/utils": "^7.0.5",
"@multiformats/multiaddr": "^13.0.1",
"libp2p": "^3.0.6",
"uint8arraylist": "^2.4.8"
}
}

387
packages/input/src/controller.ts
View File

@@ -1,12 +1,6 @@
import { controllerButtonToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import {
ProtoMessageBase,
ProtoMessageInput,
ProtoMessageInputSchema,
} from "./proto/messages_pb";
import {
ProtoInputSchema,
ProtoControllerAttachSchema,
ProtoControllerDetachSchema,
ProtoControllerButtonSchema,
@@ -16,6 +10,8 @@ import {
ProtoControllerRumble,
} from "./proto/types_pb";
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -36,7 +32,7 @@ interface GamepadState {
export class Controller {
protected wrtc: WebRTCStream;
protected slot: number;
protected slotMap: Map<number, number> = new Map(); // local slot to server slot
protected connected: boolean = false;
protected gamepad: Gamepad | null = null;
protected lastState: GamepadState = {
@@ -54,17 +50,13 @@ export class Controller {
protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range)
private updateInterval = 10.0; // 100 updates per second
private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null;
private _dcHandler: ((data: ArrayBuffer) => void) | null = null;
constructor({ webrtc, e }: Props) {
this.wrtc = webrtc;
this.slot = e.gamepad.index;
this.updateInterval = 1000 / webrtc.currentFrameRate;
// Gamepad connected
this.gamepad = e.gamepad;
// Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc")
const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/);
const vendorId = vendorMatch ? vendorMatch[1].toLowerCase() : "unknown";
@@ -72,30 +64,40 @@ export class Controller {
const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/);
const productId = productMatch ? productMatch[1].toLowerCase() : "unknown";
const attachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAttach",
value: create(ProtoControllerAttachSchema, {
type: "ControllerAttach",
id: this.vendor_id_to_controller(vendorId, productId),
slot: this.slot,
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: attachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
// Listen to datachannel events from server
this._dcHandler = (data: ArrayBuffer) => {
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data);
// Listen to feedback rumble events from server
this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer);
this.wrtc.addDataChannelCallback(this._dcRumbleHandler);
if (messageWrapper.payload.case === "controllerRumble") {
this.rumbleCallback(messageWrapper.payload.value);
} else if (messageWrapper.payload.case === "controllerAttach") {
if (this.gamepad) return; // already attached
const attachMsg = messageWrapper.payload.value;
// Gamepad connected succesfully
this.gamepad = e.gamepad;
this.slotMap.set(e.gamepad.index, attachMsg.slot);
console.log(
`Gamepad connected: ${e.gamepad.id} assigned to slot ${attachMsg.slot} on server, local slot ${e.gamepad.index}`,
);
}
} catch (err) {
console.error("Error decoding datachannel message:", err);
}
};
this.wrtc.addDataChannelCallback(this._dcHandler);
const attachMsg = createMessage(
create(ProtoControllerAttachSchema, {
id: this.vendor_id_to_controller(vendorId, productId),
sessionId: this.wrtc.getSessionID(),
}),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg));
this.run();
}
@@ -150,12 +152,13 @@ export class Controller {
}
private pollGamepad() {
// Get updated gamepad state
const gamepads = navigator.getGamepads();
if (this.slot < gamepads.length) {
const gamepad = gamepads[this.slot];
if (gamepad) {
if (this.gamepad) {
if (gamepads[this.gamepad.index]) {
this.gamepad = gamepads[this.gamepad!.index];
/* Button handling */
gamepad.buttons.forEach((button, index) => {
this.gamepad.buttons.forEach((button, index) => {
// Ignore d-pad buttons (12-15) as we handle those as axis
if (index >= 12 && index <= 15) return;
// ignore trigger buttons (6-7) as we handle those as axis
@@ -169,29 +172,15 @@ export class Controller {
return;
}
const buttonProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerButton",
value: create(ProtoControllerButtonSchema, {
type: "ControllerButton",
slot: this.slot,
const buttonMessage = createMessage(
create(ProtoControllerButtonSchema, {
slot: this.getServerSlot(),
button: linuxCode,
pressed: button.pressed,
}),
},
});
const buttonMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: buttonProto,
};
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, buttonMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage));
// Store button state
this.lastState.buttonState.set(index, button.pressed);
}
@@ -200,128 +189,100 @@ export class Controller {
/* Trigger handling */
// map trigger value from 0.0 to 1.0 to -32768 to 32767
const leftTrigger = Math.round(
this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767),
this.remapFromTo(
this.gamepad.buttons[6]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
// If state differs, send
if (leftTrigger !== this.lastState.leftTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
const triggerMessage = createMessage(
create(ProtoControllerTriggerSchema, {
slot: this.getServerSlot(),
trigger: 0, // 0 = left, 1 = right
value: leftTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.leftTrigger = leftTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
this.lastState.leftTrigger = leftTrigger;
}
const rightTrigger = Math.round(
this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767),
this.remapFromTo(
this.gamepad.buttons[7]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
// If state differs, send
if (rightTrigger !== this.lastState.rightTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
const triggerMessage = createMessage(
create(ProtoControllerTriggerSchema, {
slot: this.getServerSlot(),
trigger: 1, // 0 = left, 1 = right
value: rightTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.rightTrigger = rightTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
this.lastState.rightTrigger = rightTrigger;
}
/* DPad handling */
// We send dpad buttons as axis values -1 to 1 for left/up, right/down
const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0;
const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0;
const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0;
const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0;
const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0;
if (dpadX !== this.lastState.dpadX) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
const dpadMessage = createMessage(
create(ProtoControllerAxisSchema, {
slot: this.getServerSlot(),
axis: 0, // 0 = dpadX, 1 = dpadY
value: dpadX,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
"controllerInput",
);
this.lastState.dpadX = dpadX;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
}
const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0;
const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0;
const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0;
const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0;
const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0;
if (dpadY !== this.lastState.dpadY) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
const dpadMessage = createMessage(
create(ProtoControllerAxisSchema, {
slot: this.getServerSlot(),
axis: 1, // 0 = dpadX, 1 = dpadY
value: dpadY,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
this.lastState.dpadY = dpadY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
}
/* Stick handling */
// stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767
const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767);
const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767);
const leftX = this.remapFromTo(
this.gamepad.axes[0] ?? 0,
-1,
1,
-32768,
32767,
);
const leftY = this.remapFromTo(
this.gamepad.axes[1] ?? 0,
-1,
1,
-32768,
32767,
);
// Apply deadzone
const sendLeftX =
Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
@@ -333,35 +294,33 @@ export class Controller {
sendLeftX !== this.lastState.leftX ||
sendLeftY !== this.lastState.leftY
) {
// console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY);
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
const stickMessage = createMessage(
create(ProtoControllerStickSchema, {
stick: 0, // 0 = left, 1 = right
x: sendLeftX,
y: sendLeftY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
this.lastState.leftX = sendLeftX;
this.lastState.leftY = sendLeftY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767);
const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767);
const rightX = this.remapFromTo(
this.gamepad.axes[2] ?? 0,
-1,
1,
-32768,
32767,
);
const rightY = this.remapFromTo(
this.gamepad.axes[3] ?? 0,
-1,
1,
-32768,
32767,
);
// Apply deadzone
const sendRightX =
Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
@@ -371,30 +330,17 @@ export class Controller {
sendRightX !== this.lastState.rightX ||
sendRightY !== this.lastState.rightY
) {
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
const stickMessage = createMessage(
create(ProtoControllerStickSchema, {
stick: 1, // 0 = left, 1 = right
x: sendRightX,
y: sendRightY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
this.lastState.rightX = sendRightX;
this.lastState.rightY = sendRightY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
}
}
@@ -403,8 +349,7 @@ export class Controller {
private loopInterval: any = null;
public run() {
if (this.connected)
this.stop();
if (this.connected) this.stop();
this.connected = true;
// Poll gamepads in setInterval loop
@@ -421,69 +366,57 @@ export class Controller {
this.connected = false;
}
public getSlot() {
return this.slot;
public getLocalSlot(): number {
if (this.gamepad) {
return this.gamepad.index;
}
return -1;
}
public getServerSlot(): number {
if (this.gamepad) {
const slot = this.slotMap.get(this.gamepad.index);
if (slot !== undefined) return slot;
}
return -1;
}
public dispose() {
this.stop();
// Remove callback
if (this._dcRumbleHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcRumbleHandler);
this._dcRumbleHandler = null;
if (this._dcHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcHandler);
this._dcHandler = null;
}
// Gamepad disconnected
const detachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerDetach",
value: create(ProtoControllerDetachSchema, {
type: "ControllerDetach",
slot: this.slot,
const detachMsg = createMessage(
create(ProtoControllerDetachSchema, {
slot: this.getServerSlot(),
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: detachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg));
}
private controllerButtonToVirtualKeyCode(code: number) {
return controllerButtonToLinuxEventCode[code] || undefined;
}
private rumbleCallback(data: ArrayBuffer) {
private rumbleCallback(rumbleMsg: ProtoControllerRumble) {
// If not connected, ignore
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data);
// Check if it contains controller rumble data
if (messageWrapper.data?.inputType?.case === "controllerRumble") {
const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble;
// Check if aimed at this controller slot
if (rumbleMsg.slot !== this.slot) return;
if (rumbleMsg.slot !== this.getServerSlot()) return;
// Trigger actual rumble
// Need to remap from 0-65535 to 0.0-1.0 ranges
const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
const rumbleLowFreq = this.remapFromTo(
clampedLowFreq,
const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0);
const clampedHighFreq = Math.max(
0,
65535,
0.0,
1.0,
Math.min(65535, rumbleMsg.highFrequency),
);
const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency));
const rumbleHighFreq = this.remapFromTo(
clampedHighFreq,
0,
@@ -494,16 +427,14 @@ export class Controller {
// Cap to valid range (max 5000)
const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
if (this.gamepad.vibrationActuator) {
this.gamepad.vibrationActuator.playEffect("dual-rumble", {
this.gamepad.vibrationActuator
.playEffect("dual-rumble", {
startDelay: 0,
duration: rumbleDuration,
weakMagnitude: rumbleLowFreq,
strongMagnitude: rumbleHighFreq,
}).catch(console.error);
}
}
} catch (error) {
console.error("Failed to decode rumble message:", error);
})
.catch(console.error);
}
}
}

95
packages/input/src/keyboard.ts
View File

@@ -1,16 +1,9 @@
import {keyCodeToLinuxEventCode} from "./codes"
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
import {
ProtoInput,
ProtoInputSchema,
ProtoKeyDownSchema,
ProtoKeyUpSchema,
} from "./proto/types_pb";
import {create, toBinary} from "@bufbuild/protobuf";
import { keyCodeToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,38 +17,29 @@ export class Keyboard {
private readonly keydownListener: (e: KeyboardEvent) => void;
private readonly keyupListener: (e: KeyboardEvent) => void;
constructor({webrtc}: Props) {
constructor({ webrtc }: Props) {
this.wrtc = webrtc;
this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyDown",
value: create(ProtoKeyDownSchema, {
type: "KeyDown",
key: this.keyToVirtualKeyCode(e.code)
this.keydownListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
}
}));
this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyUp",
value: create(ProtoKeyUpSchema, {
type: "KeyUp",
key: this.keyToVirtualKeyCode(e.code)
);
this.keyupListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
}
}));
this.run()
);
this.run();
}
private run() {
if (this.connected)
this.stop()
if (this.connected) this.stop();
this.connected = true
document.addEventListener("keydown", this.keydownListener, {passive: false});
document.addEventListener("keyup", this.keyupListener, {passive: false});
this.connected = true;
document.addEventListener("keydown", this.keydownListener, {
passive: false,
});
document.addEventListener("keyup", this.keyupListener, { passive: false });
}
private stop() {
@@ -65,42 +49,19 @@ export class Keyboard {
}
// Helper function to create and return mouse listeners
private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createKeyboardListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
// Prevent repeated key events from being sent (important for games)
if ((e as any).repeat)
return;
if ((e as any).repeat) return;
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-keyboard");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}

305
packages/input/src/messages.ts
View File

@@ -1,305 +0,0 @@
import { LatencyTracker } from "./latency";
import { Uint8ArrayList } from "uint8arraylist";
import { allocUnsafe } from "uint8arrays/alloc";
import { pipe } from "it-pipe";
import { decode, encode } from "it-length-prefixed";
import { Stream } from "@libp2p/interface";
export interface MessageBase {
payload_type: string;
latency?: LatencyTracker;
}
export interface MessageRaw extends MessageBase {
data: any;
}
export function NewMessageRaw(type: string, data: any): Uint8Array {
const msg = {
payload_type: type,
data: data,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageICE extends MessageBase {
candidate: RTCIceCandidateInit;
}
export function NewMessageICE(
type: string,
candidate: RTCIceCandidateInit,
): Uint8Array {
const msg = {
payload_type: type,
candidate: candidate,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageSDP extends MessageBase {
sdp: RTCSessionDescriptionInit;
}
export function NewMessageSDP(
type: string,
sdp: RTCSessionDescriptionInit,
): Uint8Array {
const msg = {
payload_type: type,
sdp: sdp,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
const MAX_SIZE = 1024 * 1024; // 1MB
const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue
// Custom 4-byte length encoder
export const length4ByteEncoder = (length: number) => {
const buf = allocUnsafe(4);
// Write the length as a 32-bit unsigned integer (4 bytes)
buf[0] = length >>> 24;
buf[1] = (length >>> 16) & 0xff;
buf[2] = (length >>> 8) & 0xff;
buf[3] = length & 0xff;
// Set the bytes property to 4
length4ByteEncoder.bytes = 4;
return buf;
};
length4ByteEncoder.bytes = 4;
// Custom 4-byte length decoder
export const length4ByteDecoder = (data: Uint8ArrayList) => {
if (data.byteLength < 4) {
// Not enough bytes to read the length
return -1;
}
// Read the length from the first 4 bytes
let length = 0;
length =
(data.subarray(0, 1)[0] >>> 0) * 0x1000000 +
(data.subarray(1, 2)[0] >>> 0) * 0x10000 +
(data.subarray(2, 3)[0] >>> 0) * 0x100 +
(data.subarray(3, 4)[0] >>> 0);
// Set bytes read to 4
length4ByteDecoder.bytes = 4;
return length;
};
length4ByteDecoder.bytes = 4;
interface PromiseMessage {
data: Uint8Array;
resolve: () => void;
reject: (error: Error) => void;
}
export class SafeStream {
private stream: Stream;
private callbacks: Map<string, ((data: any) => void)[]> = new Map();
private isReading: boolean = false;
private isWriting: boolean = false;
private closed: boolean = false;
private messageQueue: PromiseMessage[] = [];
private writeLock = false;
private readRetries = 0;
private writeRetries = 0;
private readonly MAX_RETRIES = 5;
constructor(stream: Stream) {
this.stream = stream;
this.startReading();
this.startWriting();
}
private async startReading(): Promise<void> {
if (this.isReading || this.closed) return;
this.isReading = true;
try {
const source = this.stream.source;
const decodedSource = decode(source, {
maxDataLength: MAX_SIZE,
lengthDecoder: length4ByteDecoder,
});
for await (const chunk of decodedSource) {
if (this.closed) break;
this.readRetries = 0;
try {
const data = chunk.slice();
const message = JSON.parse(
new TextDecoder().decode(data),
) as MessageBase;
const msgType = message.payload_type;
if (this.callbacks.has(msgType)) {
const handlers = this.callbacks.get(msgType)!;
for (const handler of handlers) {
try {
handler(message);
} catch (err) {
console.error(`Error in message handler for ${msgType}:`, err);
}
}
}
} catch (err) {
console.error("Error processing message:", err);
}
}
} catch (err) {
console.error("Stream reading error:", err);
} finally {
this.isReading = false;
this.readRetries++;
// If not closed, try to restart reading
if (!this.closed && this.readRetries < this.MAX_RETRIES)
setTimeout(() => this.startReading(), 100);
else if (this.readRetries >= this.MAX_RETRIES)
console.error(
"Max retries reached for reading stream, stopping attempts",
);
}
}
public registerCallback(
msgType: string,
callback: (data: any) => void,
): void {
if (!this.callbacks.has(msgType)) {
this.callbacks.set(msgType, []);
}
this.callbacks.get(msgType)!.push(callback);
}
public removeCallback(msgType: string, callback: (data: any) => void): void {
if (this.callbacks.has(msgType)) {
const callbacks = this.callbacks.get(msgType)!;
const index = callbacks.indexOf(callback);
if (index !== -1) {
callbacks.splice(index, 1);
}
if (callbacks.length === 0) {
this.callbacks.delete(msgType);
}
}
}
private async startWriting(): Promise<void> {
if (this.isWriting || this.closed) return;
this.isWriting = true;
try {
// Create an async generator for real-time message processing
const messageSource = async function* (this: SafeStream) {
while (!this.closed) {
// Check if we have messages to send
if (this.messageQueue.length > 0) {
this.writeLock = true;
try {
const message = this.messageQueue[0];
// Encode the message
const encoded = encode([message.data], {
maxDataLength: MAX_SIZE,
lengthEncoder: length4ByteEncoder,
});
for await (const chunk of encoded) {
yield chunk;
}
// Remove message after successful sending
this.writeRetries = 0;
const sentMessage = this.messageQueue.shift();
if (sentMessage)
sentMessage.resolve();
} catch (err) {
console.error("Error encoding or sending message:", err);
const failedMessage = this.messageQueue.shift();
if (failedMessage)
failedMessage.reject(new Error(`Failed to send message: ${err}`));
} finally {
this.writeLock = false;
}
} else {
// No messages to send, wait for a short period
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}.bind(this);
await pipe(messageSource(), this.stream.sink).catch((err) => {
console.error("Sink error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing to stream sink, stopping attempts");
}
});
} catch (err) {
console.error("Stream writing error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing stream, stopping attempts");
}
}
}
public async writeMessage(message: Uint8Array): Promise<void> {
if (this.closed) {
throw new Error("Cannot write to closed stream");
}
// Validate message size before queuing
if (message.length > MAX_SIZE) {
throw new Error("Message size exceeds maximum size limit");
}
// Check if the message queue is too large
if (this.messageQueue.length >= MAX_QUEUE_SIZE) {
throw new Error("Message queue is full, cannot write message");
}
// Create a promise to resolve when the message is sent
return new Promise((resolve, reject) => {
this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage);
});
}
public close(): void {
this.closed = true;
this.callbacks.clear();
// Reject pending messages
for (const msg of this.messageQueue)
msg.reject(new Error("Stream closed"));
this.messageQueue = [];
this.readRetries = 0;
this.writeRetries = 0;
}
}

158
packages/input/src/mouse.ts
View File

@@ -1,18 +1,14 @@
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
import { WebRTCStream } from "./webrtc-stream";
import {
ProtoInput, ProtoInputSchema,
ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
ProtoMouseMove,
ProtoMouseKeyDownSchema,
ProtoMouseKeyUpSchema,
ProtoMouseMoveSchema,
ProtoMouseWheel, ProtoMouseWheelSchema
ProtoMouseWheelSchema,
} from "./proto/types_pb";
import {mouseButtonToLinuxEventCode} from "./codes";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {create, toBinary} from "@bufbuild/protobuf";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import { mouseButtonToLinuxEventCode } from "./codes";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,7 +20,7 @@ export class Mouse {
protected canvas: HTMLCanvasElement;
protected connected!: boolean;
private sendInterval = 10 // 100 updates per second
private sendInterval = 10; // 100 updates per second
// Store references to event listeners
private readonly mousemoveListener: (e: MouseEvent) => void;
@@ -35,7 +31,7 @@ export class Mouse {
private readonly mouseupListener: (e: MouseEvent) => void;
private readonly mousewheelListener: (e: WheelEvent) => void;
constructor({webrtc, canvas}: Props) {
constructor({ webrtc, canvas }: Props) {
this.wrtc = webrtc;
this.canvas = canvas;
@@ -48,65 +44,56 @@ export class Mouse {
this.movementY += e.movementY;
};
this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyDown",
value: create(ProtoMouseKeyDownSchema, {
type: "MouseKeyDown",
key: this.keyToVirtualKeyCode(e.button)
this.mousedownListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
}
}));
this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyUp",
value: create(ProtoMouseKeyUpSchema, {
type: "MouseKeyUp",
key: this.keyToVirtualKeyCode(e.button)
);
this.mouseupListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
}
}));
this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseWheel",
value: create(ProtoMouseWheelSchema, {
type: "MouseWheel",
);
this.mousewheelListener = this.createMouseListener((e: any) =>
create(ProtoMouseWheelSchema, {
x: Math.round(e.deltaX),
y: Math.round(e.deltaY),
}),
}
}));
);
this.run()
this.run();
this.startProcessing();
}
private run() {
//calls all the other functions
if (!document.pointerLockElement) {
console.log("no pointerlock")
console.log("no pointerlock");
if (this.connected) {
this.stop()
this.stop();
}
return;
}
if (document.pointerLockElement == this.canvas) {
this.connected = true
this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
this.connected = true;
this.canvas.addEventListener("mousemove", this.mousemoveListener, {
passive: false,
});
this.canvas.addEventListener("mousedown", this.mousedownListener, {
passive: false,
});
this.canvas.addEventListener("mouseup", this.mouseupListener, {
passive: false,
});
this.canvas.addEventListener("wheel", this.mousewheelListener, {
passive: false,
});
} else {
if (this.connected) {
this.stop()
this.stop();
}
}
}
private stop() {
@@ -128,79 +115,26 @@ export class Mouse {
}
private sendAggregatedMouseMove() {
const data = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseMove",
value: create(ProtoMouseMoveSchema, {
type: "MouseMove",
const data = create(ProtoMouseMoveSchema, {
x: Math.round(this.movementX),
y: Math.round(this.movementY),
}),
},
});
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
}
// Helper function to create and return mouse listeners
private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createMouseListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}

142
packages/input/src/proto/messages_pb.ts
View File

@@ -4,7 +4,7 @@
import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
import type { ProtoInput } from "./types_pb";
import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerAxis, ProtoControllerButton, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStick, ProtoControllerTrigger, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb";
import { file_types } from "./types_pb";
import type { ProtoLatencyTracker } from "./latency_tracker_pb";
import { file_latency_tracker } from "./latency_tracker_pb";
@@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf";
* Describes the file messages.proto.
*/
export const file_messages: GenFile = /*@__PURE__*/
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiyQgKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9idXR0b24YCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJCdXR0b25IABI7ChJjb250cm9sbGVyX3RyaWdnZXIYDCABKAsyHS5wcm90by5Qcm90b0NvbnRyb2xsZXJUcmlnZ2VySAASNwoQY29udHJvbGxlcl9zdGljaxgNIAEoCzIbLnByb3RvLlByb3RvQ29udHJvbGxlclN0aWNrSAASNQoPY29udHJvbGxlcl9heGlzGA4gASgLMhoucHJvdG8uUHJvdG9Db250cm9sbGVyQXhpc0gAEjkKEWNvbnRyb2xsZXJfcnVtYmxlGA8gASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyUnVtYmxlSAASHgoDaWNlGBQgASgLMg8ucHJvdG8uUHJvdG9JQ0VIABIeCgNzZHAYFSABKAsyDy5wcm90by5Qcm90b1NEUEgAEh4KA3JhdxgWIAEoCzIPLnByb3RvLlByb3RvUmF3SAASSQoaY2xpZW50X3JlcXVlc3Rfcm9vbV9zdHJlYW0YFyABKAsyIy5wcm90by5Qcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtSAASPQoTY2xpZW50X2Rpc2Nvbm5lY3RlZBgYIAEoCzIeLnByb3RvLlByb3RvQ2xpZW50RGlzY29ubmVjdGVkSAASOgoSc2VydmVyX3B1c2hfc3RyZWFtGBkgASgLMhwucHJvdG8uUHJvdG9TZXJ2ZXJQdXNoU3RyZWFtSABCCQoHcGF5bG9hZEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
/**
* @generated from message proto.ProtoMessageBase
@@ -39,24 +39,148 @@ export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*
messageDesc(file_messages, 0);
/**
* @generated from message proto.ProtoMessageInput
* @generated from message proto.ProtoMessage
*/
export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
export type ProtoMessage = Message<"proto.ProtoMessage"> & {
/**
* @generated from field: proto.ProtoMessageBase message_base = 1;
*/
messageBase?: ProtoMessageBase;
/**
* @generated from field: proto.ProtoInput data = 2;
* @generated from oneof proto.ProtoMessage.payload
*/
data?: ProtoInput;
payload: {
/**
* Input types
*
* @generated from field: proto.ProtoMouseMove mouse_move = 2;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 4;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 7;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 8;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* @generated from field: proto.ProtoControllerAttach controller_attach = 9;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 10;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerButton controller_button = 11;
*/
value: ProtoControllerButton;
case: "controllerButton";
} | {
/**
* @generated from field: proto.ProtoControllerTrigger controller_trigger = 12;
*/
value: ProtoControllerTrigger;
case: "controllerTrigger";
} | {
/**
* @generated from field: proto.ProtoControllerStick controller_stick = 13;
*/
value: ProtoControllerStick;
case: "controllerStick";
} | {
/**
* @generated from field: proto.ProtoControllerAxis controller_axis = 14;
*/
value: ProtoControllerAxis;
case: "controllerAxis";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 15;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | {
/**
* Signaling types
*
* @generated from field: proto.ProtoICE ice = 20;
*/
value: ProtoICE;
case: "ice";
} | {
/**
* @generated from field: proto.ProtoSDP sdp = 21;
*/
value: ProtoSDP;
case: "sdp";
} | {
/**
* @generated from field: proto.ProtoRaw raw = 22;
*/
value: ProtoRaw;
case: "raw";
} | {
/**
* @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23;
*/
value: ProtoClientRequestRoomStream;
case: "clientRequestRoomStream";
} | {
/**
* @generated from field: proto.ProtoClientDisconnected client_disconnected = 24;
*/
value: ProtoClientDisconnected;
case: "clientDisconnected";
} | {
/**
* @generated from field: proto.ProtoServerPushStream server_push_stream = 25;
*/
value: ProtoServerPushStream;
case: "serverPushStream";
} | { case: undefined; value?: undefined };
};
/**
* Describes the message proto.ProtoMessageInput.
* Use `create(ProtoMessageInputSchema)` to create a new message.
* Describes the message proto.ProtoMessage.
* Use `create(ProtoMessageSchema)` to create a new message.
*/
export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
export const ProtoMessageSchema: GenMessage<ProtoMessage> = /*@__PURE__*/
messageDesc(file_messages, 1);

416
packages/input/src/proto/types_pb.ts
View File

@@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf";
* Describes the file types.proto.
*/
export const file_types: GenFile = /*@__PURE__*/
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiRQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEgwKBHNsb3QYAiABKAUSEgoKc2Vzc2lvbl9pZBgDIAEoCSIlChVQcm90b0NvbnRyb2xsZXJEZXRhY2gSDAoEc2xvdBgBIAEoBSJGChVQcm90b0NvbnRyb2xsZXJCdXR0b24SDAoEc2xvdBgBIAEoBRIOCgZidXR0b24YAiABKAUSDwoHcHJlc3NlZBgDIAEoCCJGChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHNsb3QYASABKAUSDwoHdHJpZ2dlchgCIAEoBRINCgV2YWx1ZRgDIAEoBSJJChRQcm90b0NvbnRyb2xsZXJTdGljaxIMCgRzbG90GAEgASgFEg0KBXN0aWNrGAIgASgFEgkKAXgYAyABKAUSCQoBeRgEIAEoBSJAChNQcm90b0NvbnRyb2xsZXJBeGlzEgwKBHNsb3QYASABKAUSDAoEYXhpcxgCIAEoBRINCgV2YWx1ZRgDIAEoBSJmChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEc2xvdBgBIAEoBRIVCg1sb3dfZnJlcXVlbmN5GAIgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAMgASgFEhAKCGR1cmF0aW9uGAQgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
/**
* MouseMove message
@@ -19,19 +19,12 @@ export const file_types: GenFile = /*@__PURE__*/
*/
export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
/**
* Fixed value "MouseMove"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -50,19 +43,12 @@ export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
*/
export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
/**
* Fixed value "MouseMoveAbs"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -81,19 +67,12 @@ export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE_
*/
export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
/**
* Fixed value "MouseWheel"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -112,14 +91,7 @@ export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
*/
export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
/**
* Fixed value "MouseKeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -138,14 +110,7 @@ export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE_
*/
export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
/**
* Fixed value "MouseKeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -164,14 +129,7 @@ export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
*/
export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
/**
* Fixed value "KeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -190,14 +148,7 @@ export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
*/
export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
/**
* Fixed value "KeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -215,26 +166,26 @@ export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
* @generated from message proto.ProtoControllerAttach
*/
export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & {
/**
* Fixed value "ControllerAttach"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* One of the following enums: "ps", "xbox" or "switch"
*
* @generated from field: string id = 2;
* @generated from field: string id = 1;
*/
id: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 3;
* @generated from field: int32 slot = 2;
*/
slot: number;
/**
* Session ID of the client attaching the controller
*
* @generated from field: string session_id = 3;
*/
sessionId: string;
};
/**
@@ -250,17 +201,10 @@ export const ProtoControllerAttachSchema: GenMessage<ProtoControllerAttach> = /*
* @generated from message proto.ProtoControllerDetach
*/
export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
/**
* Fixed value "ControllerDetach"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
};
@@ -278,31 +222,24 @@ export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*
* @generated from message proto.ProtoControllerButton
*/
export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & {
/**
* Fixed value "ControllerButtons"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
/**
* Button code (linux input event code)
*
* @generated from field: int32 button = 3;
* @generated from field: int32 button = 2;
*/
button: number;
/**
* true if pressed, false if released
*
* @generated from field: bool pressed = 4;
* @generated from field: bool pressed = 3;
*/
pressed: boolean;
};
@@ -320,31 +257,24 @@ export const ProtoControllerButtonSchema: GenMessage<ProtoControllerButton> = /*
* @generated from message proto.ProtoControllerTrigger
*/
export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & {
/**
* Fixed value "ControllerTriggers"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
/**
* Trigger number (0 for left, 1 for right)
*
* @generated from field: int32 trigger = 3;
* @generated from field: int32 trigger = 2;
*/
trigger: number;
/**
* trigger value (-32768 to 32767)
*
* @generated from field: int32 value = 4;
* @generated from field: int32 value = 3;
*/
value: number;
};
@@ -362,38 +292,31 @@ export const ProtoControllerTriggerSchema: GenMessage<ProtoControllerTrigger> =
* @generated from message proto.ProtoControllerStick
*/
export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & {
/**
* Fixed value "ControllerStick"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
/**
* Stick number (0 for left, 1 for right)
*
* @generated from field: int32 stick = 3;
* @generated from field: int32 stick = 2;
*/
stick: number;
/**
* X axis value (-32768 to 32767)
*
* @generated from field: int32 x = 4;
* @generated from field: int32 x = 3;
*/
x: number;
/**
* Y axis value (-32768 to 32767)
*
* @generated from field: int32 y = 5;
* @generated from field: int32 y = 4;
*/
y: number;
};
@@ -411,31 +334,24 @@ export const ProtoControllerStickSchema: GenMessage<ProtoControllerStick> = /*@_
* @generated from message proto.ProtoControllerAxis
*/
export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & {
/**
* Fixed value "ControllerAxis"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
/**
* Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
*
* @generated from field: int32 axis = 3;
* @generated from field: int32 axis = 2;
*/
axis: number;
/**
* axis value (-1 to 1)
*
* @generated from field: int32 value = 4;
* @generated from field: int32 value = 3;
*/
value: number;
};
@@ -453,38 +369,31 @@ export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__P
* @generated from message proto.ProtoControllerRumble
*/
export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
/**
* Fixed value "ControllerRumble"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
* @generated from field: int32 slot = 1;
*/
slot: number;
/**
* Low frequency rumble (0-65535)
*
* @generated from field: int32 low_frequency = 3;
* @generated from field: int32 low_frequency = 2;
*/
lowFrequency: number;
/**
* High frequency rumble (0-65535)
*
* @generated from field: int32 high_frequency = 4;
* @generated from field: int32 high_frequency = 3;
*/
highFrequency: number;
/**
* Duration in milliseconds
*
* @generated from field: int32 duration = 5;
* @generated from field: int32 duration = 4;
*/
duration: number;
};
@@ -497,105 +406,180 @@ export const ProtoControllerRumbleSchema: GenMessage<ProtoControllerRumble> = /*
messageDesc(file_types, 13);
/**
* Union of all Input types
*
* @generated from message proto.ProtoInput
* @generated from message proto.RTCIceCandidateInit
*/
export type ProtoInput = Message<"proto.ProtoInput"> & {
export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & {
/**
* @generated from oneof proto.ProtoInput.input_type
* @generated from field: string candidate = 1;
*/
inputType: {
candidate: string;
/**
* @generated from field: proto.ProtoMouseMove mouse_move = 1;
* @generated from field: optional uint32 sdpMLineIndex = 2;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
sdpMLineIndex?: number;
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
* @generated from field: optional string sdpMid = 3;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
sdpMid?: string;
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
* @generated from field: optional string usernameFragment = 4;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 6;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 7;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* @generated from field: proto.ProtoControllerAttach controller_attach = 8;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 9;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerButton controller_button = 10;
*/
value: ProtoControllerButton;
case: "controllerButton";
} | {
/**
* @generated from field: proto.ProtoControllerTrigger controller_trigger = 11;
*/
value: ProtoControllerTrigger;
case: "controllerTrigger";
} | {
/**
* @generated from field: proto.ProtoControllerStick controller_stick = 12;
*/
value: ProtoControllerStick;
case: "controllerStick";
} | {
/**
* @generated from field: proto.ProtoControllerAxis controller_axis = 13;
*/
value: ProtoControllerAxis;
case: "controllerAxis";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 14;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | { case: undefined; value?: undefined };
usernameFragment?: string;
};
/**
* Describes the message proto.ProtoInput.
* Use `create(ProtoInputSchema)` to create a new message.
* Describes the message proto.RTCIceCandidateInit.
* Use `create(RTCIceCandidateInitSchema)` to create a new message.
*/
export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
export const RTCIceCandidateInitSchema: GenMessage<RTCIceCandidateInit> = /*@__PURE__*/
messageDesc(file_types, 14);
/**
* @generated from message proto.RTCSessionDescriptionInit
*/
export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & {
/**
* @generated from field: string sdp = 1;
*/
sdp: string;
/**
* @generated from field: string type = 2;
*/
type: string;
};
/**
* Describes the message proto.RTCSessionDescriptionInit.
* Use `create(RTCSessionDescriptionInitSchema)` to create a new message.
*/
export const RTCSessionDescriptionInitSchema: GenMessage<RTCSessionDescriptionInit> = /*@__PURE__*/
messageDesc(file_types, 15);
/**
* ProtoICE message
*
* @generated from message proto.ProtoICE
*/
export type ProtoICE = Message<"proto.ProtoICE"> & {
/**
* @generated from field: proto.RTCIceCandidateInit candidate = 1;
*/
candidate?: RTCIceCandidateInit;
};
/**
* Describes the message proto.ProtoICE.
* Use `create(ProtoICESchema)` to create a new message.
*/
export const ProtoICESchema: GenMessage<ProtoICE> = /*@__PURE__*/
messageDesc(file_types, 16);
/**
* ProtoSDP message
*
* @generated from message proto.ProtoSDP
*/
export type ProtoSDP = Message<"proto.ProtoSDP"> & {
/**
* @generated from field: proto.RTCSessionDescriptionInit sdp = 1;
*/
sdp?: RTCSessionDescriptionInit;
};
/**
* Describes the message proto.ProtoSDP.
* Use `create(ProtoSDPSchema)` to create a new message.
*/
export const ProtoSDPSchema: GenMessage<ProtoSDP> = /*@__PURE__*/
messageDesc(file_types, 17);
/**
* ProtoRaw message
*
* @generated from message proto.ProtoRaw
*/
export type ProtoRaw = Message<"proto.ProtoRaw"> & {
/**
* @generated from field: string data = 1;
*/
data: string;
};
/**
* Describes the message proto.ProtoRaw.
* Use `create(ProtoRawSchema)` to create a new message.
*/
export const ProtoRawSchema: GenMessage<ProtoRaw> = /*@__PURE__*/
messageDesc(file_types, 18);
/**
* ProtoClientRequestRoomStream message
*
* @generated from message proto.ProtoClientRequestRoomStream
*/
export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
/**
* @generated from field: string session_id = 2;
*/
sessionId: string;
};
/**
* Describes the message proto.ProtoClientRequestRoomStream.
* Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message.
*/
export const ProtoClientRequestRoomStreamSchema: GenMessage<ProtoClientRequestRoomStream> = /*@__PURE__*/
messageDesc(file_types, 19);
/**
* ProtoClientDisconnected message
*
* @generated from message proto.ProtoClientDisconnected
*/
export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & {
/**
* @generated from field: string session_id = 1;
*/
sessionId: string;
/**
* @generated from field: repeated int32 controller_slots = 2;
*/
controllerSlots: number[];
};
/**
* Describes the message proto.ProtoClientDisconnected.
* Use `create(ProtoClientDisconnectedSchema)` to create a new message.
*/
export const ProtoClientDisconnectedSchema: GenMessage<ProtoClientDisconnected> = /*@__PURE__*/
messageDesc(file_types, 20);
/**
* ProtoServerPushStream message
*
* @generated from message proto.ProtoServerPushStream
*/
export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
};
/**
* Describes the message proto.ProtoServerPushStream.
* Use `create(ProtoServerPushStreamSchema)` to create a new message.
*/
export const ProtoServerPushStreamSchema: GenMessage<ProtoServerPushStream> = /*@__PURE__*/
messageDesc(file_types, 21);

81
packages/input/src/streamwrapper.ts Normal file
View File

@@ -0,0 +1,81 @@
import { pbStream, type ProtobufStream } from "@libp2p/utils";
import type { Stream } from "@libp2p/interface";
import { bufbuildAdapter } from "./utils";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBase,
} from "./proto/messages_pb";
type MessageHandler = (
data: any,
base: ProtoMessageBase,
) => void | Promise<void>;
export class P2PMessageStream {
private pb: ProtobufStream;
private handlers = new Map<string, MessageHandler[]>();
private closed = false;
private readLoopRunning = false;
constructor(stream: Stream) {
this.pb = pbStream(stream);
}
public on(payloadType: string, handler: MessageHandler): void {
if (!this.handlers.has(payloadType)) {
this.handlers.set(payloadType, []);
}
this.handlers.get(payloadType)!.push(handler);
if (!this.readLoopRunning) this.startReading().catch(console.error);
}
private async startReading(): Promise<void> {
if (this.readLoopRunning || this.closed) return;
this.readLoopRunning = true;
while (!this.closed) {
try {
const msg: ProtoMessage = await this.pb.read(
bufbuildAdapter(ProtoMessageSchema),
);
const payloadType = msg.messageBase?.payloadType;
if (payloadType && this.handlers.has(payloadType)) {
const handlers = this.handlers.get(payloadType)!;
if (msg.payload.value) {
for (const handler of handlers) {
try {
await handler(msg.payload.value, msg.messageBase);
} catch (err) {
console.error(`Error in handler for ${payloadType}:`, err);
}
}
}
}
} catch (err) {
if (this.closed) break;
console.error("Stream read error:", err);
this.close();
}
}
this.readLoopRunning = false;
}
public async write(
message: ProtoMessage,
options?: { signal?: AbortSignal },
): Promise<void> {
if (this.closed)
throw new Error("Cannot write to closed stream");
await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options);
}
public close(): void {
this.closed = true;
this.handlers.clear();
}
}

95
packages/input/src/utils.ts Normal file
View File

@@ -0,0 +1,95 @@
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import type { Message } from "@bufbuild/protobuf";
import { Uint8ArrayList } from "uint8arraylist";
import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
import { timestampFromDate } from "@bufbuild/protobuf/wkt";
import {
ProtoLatencyTracker,
ProtoLatencyTrackerSchema,
ProtoTimestampEntrySchema,
} from "./proto/latency_tracker_pb";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBaseSchema,
} from "./proto/messages_pb";
export function bufbuildAdapter<T extends Message>(schema: GenMessage<T>) {
return {
encode: (data: T): Uint8Array => {
return toBinary(schema, data);
},
decode: (data: Uint8Array | Uint8ArrayList): T => {
// Convert Uint8ArrayList to Uint8Array if needed
const bytes = data instanceof Uint8ArrayList ? data.subarray() : data;
return fromBinary(schema, bytes);
},
};
}
// Latency tracker helpers
export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker {
return create(ProtoLatencyTrackerSchema, {
sequenceId: sequenceId || crypto.randomUUID(),
timestamps: [],
});
}
export function addLatencyTimestamp(
tracker: ProtoLatencyTracker,
stage: string,
): ProtoLatencyTracker {
const entry = create(ProtoTimestampEntrySchema, {
stage,
time: timestampFromDate(new Date()),
});
return {
...tracker,
timestamps: [...tracker.timestamps, entry],
};
}
interface CreateMessageOptions {
sequenceId?: string;
}
function derivePayloadCase(data: Message): string {
// Extract case from $typeName: "proto.ProtoICE" -> "ice"
// "proto.ProtoControllerAttach" -> "controllerAttach"
const typeName = data.$typeName;
if (!typeName)
throw new Error("Message has no $typeName");
// Remove "proto.Proto" prefix and convert first char to lowercase
const caseName = typeName.replace(/^proto\.Proto/, "");
// Convert PascalCase to camelCase
// If it's all caps (like SDP, ICE), lowercase everything
// Otherwise, just lowercase the first character
if (caseName === caseName.toUpperCase()) {
return caseName.toLowerCase();
}
return caseName.charAt(0).toLowerCase() + caseName.slice(1);
}
export function createMessage(
data: Message,
payloadType: string,
options?: CreateMessageOptions,
): ProtoMessage {
const payloadCase = derivePayloadCase(data);
return create(ProtoMessageSchema, {
messageBase: create(ProtoMessageBaseSchema, {
payloadType,
latency: options?.sequenceId
? createLatencyTracker(options.sequenceId)
: undefined,
}),
payload: {
case: payloadCase,
value: data,
} as any, // Type assertion needed for dynamic case
});
}

114
packages/input/src/webrtc-stream.ts
View File

@@ -1,9 +1,3 @@
import {
NewMessageRaw,
NewMessageSDP,
NewMessageICE,
SafeStream,
} from "./messages";
import { webSockets } from "@libp2p/websockets";
import { webTransport } from "@libp2p/webtransport";
import { createLibp2p, Libp2p } from "libp2p";
@@ -13,19 +7,32 @@ import { identify } from "@libp2p/identify";
import { multiaddr } from "@multiformats/multiaddr";
import { Connection } from "@libp2p/interface";
import { ping } from "@libp2p/ping";
import { createMessage } from "./utils";
import { create } from "@bufbuild/protobuf";
import {
ProtoClientRequestRoomStream,
ProtoClientRequestRoomStreamSchema,
ProtoICE,
ProtoICESchema, ProtoRaw,
ProtoSDP,
ProtoSDPSchema
} from "./proto/types_pb";
import { P2PMessageStream } from "./streamwrapper";
const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
export class WebRTCStream {
private _p2p: Libp2p | undefined = undefined;
private _p2pConn: Connection | undefined = undefined;
private _p2pSafeStream: SafeStream | undefined = undefined;
private _msgStream: P2PMessageStream | undefined = undefined;
private _pc: RTCPeerConnection | undefined = undefined;
private _audioTrack: MediaStreamTrack | undefined = undefined;
private _videoTrack: MediaStreamTrack | undefined = undefined;
private _dataChannel: RTCDataChannel | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined =
undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined =
undefined;
private _serverURL: string | undefined = undefined;
private _roomName: string | undefined = undefined;
private _isConnected: boolean = false;
@@ -89,14 +96,20 @@ export class WebRTCStream {
.newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
.catch(console.error);
if (stream) {
this._p2pSafeStream = new SafeStream(stream);
this._msgStream = new P2PMessageStream(stream);
console.log("Stream opened with peer");
let iceHolder: RTCIceCandidateInit[] = [];
this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
this._msgStream.on("ice-candidate", (data: ProtoICE) => {
const cand: RTCIceCandidateInit = {
candidate: data.candidate.candidate,
sdpMLineIndex: data.candidate.sdpMLineIndex,
sdpMid: data.candidate.sdpMid,
usernameFragment: data.candidate.usernameFragment,
};
if (this._pc) {
if (this._pc.remoteDescription) {
this._pc.addIceCandidate(data.candidate).catch((err) => {
this._pc.addIceCandidate(cand).catch((err) => {
console.error("Error adding ICE candidate:", err);
});
// Add held candidates
@@ -107,45 +120,70 @@ export class WebRTCStream {
});
iceHolder = [];
} else {
iceHolder.push(data.candidate);
iceHolder.push(cand);
}
} else {
iceHolder.push(data.candidate);
iceHolder.push(cand);
}
});
this._p2pSafeStream.registerCallback("offer", async (data) => {
this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => {
const sessionId = data.sessionId;
localStorage.setItem("nestri-session-id", sessionId);
console.log("Session ID assigned:", sessionId, "for room:", data.roomName);
});
this._msgStream.on("offer", async (data: ProtoSDP) => {
if (!this._pc) {
// Setup peer connection now
this._setupPeerConnection();
}
await this._pc!.setRemoteDescription(data.sdp);
await this._pc!.setRemoteDescription({
sdp: data.sdp.sdp,
type: data.sdp.type as RTCSdpType,
});
// Create our answer
const answer = await this._pc!.createAnswer();
// Force stereo in Chromium browsers
answer.sdp = this.forceOpusStereo(answer.sdp!);
await this._pc!.setLocalDescription(answer);
// Send answer back
const answerMsg = NewMessageSDP("answer", answer);
await this._p2pSafeStream?.writeMessage(answerMsg);
const answerMsg = createMessage(
create(ProtoSDPSchema, {
sdp: answer,
}),
"answer",
);
await this._msgStream?.write(answerMsg);
});
this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
console.warn("Stream is offline for room:", data.roomName);
this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => {
console.warn("Stream is offline for room:", msg.data);
this._onConnected?.(null);
});
const clientId = localStorage.getItem("nestri-session-id");
if (clientId) {
console.debug("Using existing session ID:", clientId);
}
// Send stream request
// marshal room name into json
const request = NewMessageRaw(
const requestMsg = createMessage(
create(ProtoClientRequestRoomStreamSchema, {
roomName: roomName,
sessionId: clientId,
}),
"request-stream-room",
roomName,
);
await this._p2pSafeStream.writeMessage(request);
await this._msgStream.write(requestMsg);
}
}
}
public getSessionID(): string {
return localStorage.getItem("nestri-session-id") || "";
}
// Forces opus to stereo in Chromium browsers, because of course
private forceOpusStereo(SDP: string): string {
// Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
@@ -200,11 +238,16 @@ export class WebRTCStream {
this._pc.onicecandidate = (e) => {
if (e.candidate) {
const iceMsg = NewMessageICE("ice-candidate", e.candidate);
if (this._p2pSafeStream) {
this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
console.error("Error sending ICE candidate:", err),
const iceMsg = createMessage(
create(ProtoICESchema, {
candidate: e.candidate,
}),
"ice-candidate",
);
if (this._msgStream) {
this._msgStream
.write(iceMsg)
.catch((err) => console.error("Error sending ICE candidate:", err));
} else {
console.warn("P2P stream not established, cannot send ICE candidate");
}
@@ -218,8 +261,7 @@ export class WebRTCStream {
}
private _checkConnectionState() {
if (!this._pc || !this._p2p || !this._p2pConn)
return;
if (!this._pc || !this._p2p || !this._p2pConn) return;
console.debug("Checking connection state:", {
connectionState: this._pc.connectionState,
@@ -286,7 +328,9 @@ export class WebRTCStream {
// Attempt to reconnect only if not already connected
if (!this._isConnected && this._serverURL && this._roomName) {
this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
this._setup(this._serverURL, this._roomName).catch((err) =>
console.error("Reconnection failed:", err),
);
}
}
@@ -335,7 +379,9 @@ export class WebRTCStream {
}
public removeDataChannelCallback(callback: (data: any) => void) {
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback);
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(
(cb) => cb !== callback,
);
}
private _setupDataChannelEvents() {
@@ -343,7 +389,7 @@ export class WebRTCStream {
this._dataChannel.onclose = () => console.log("sendChannel has closed");
this._dataChannel.onopen = () => console.log("sendChannel has opened");
this._dataChannel.onmessage = (event => {
this._dataChannel.onmessage = (event) => {
// Parse as ProtoBuf message
const data = event.data;
// Call registered callback if exists
@@ -354,7 +400,7 @@ export class WebRTCStream {
console.error("Error in data channel callback:", err);
}
});
});
};
}
private _gatherFrameRate() {

2
packages/play-standalone/src/pages/[room].astro
View File

@@ -106,7 +106,7 @@ if (envs_map.size > 0) {
if (e.gamepad.id.toLowerCase().includes("nestri"))
return;
let disconnected = nestriControllers.find((c) => c.getSlot() === e.gamepad.index);
let disconnected = nestriControllers.find((c) => c.getLocalSlot() === e.gamepad.index);
if (disconnected) {
disconnected.dispose();
nestriControllers = nestriControllers.filter((c) => c !== disconnected);

2
packages/relay/go.mod
View File

@@ -33,7 +33,7 @@ require (
github.com/ipfs/go-cid v0.5.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/koron/go-ssdp v0.1.0 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect

4
packages/relay/go.sum
View File

@@ -82,8 +82,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/koron/go-ssdp v0.1.0 h1:ckl5x5H6qSNFmi+wCuROvvGUu2FQnMbQrU95IHCcv3Y=

182
packages/relay/internal/common/safebufio.go
View File

@@ -3,16 +3,28 @@ package common
import (
"bufio"
"encoding/binary"
"encoding/json"
"errors"
"io"
gen "relay/internal/proto"
"sync"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/timestamppb"
)
// MaxSize is the maximum allowed data size (1MB)
const MaxSize = 1024 * 1024
// readUvarint reads an unsigned varint from the reader
func readUvarint(r io.ByteReader) (uint64, error) {
return binary.ReadUvarint(r)
}
// writeUvarint writes an unsigned varint to the writer
func writeUvarint(w io.Writer, x uint64) error {
buf := make([]byte, binary.MaxVarintLen64)
n := binary.PutUvarint(buf, x)
_, err := w.Write(buf[:n])
return err
}
// SafeBufioRW wraps a bufio.ReadWriter for sending and receiving JSON and protobufs safely
type SafeBufioRW struct {
@@ -24,83 +36,6 @@ func NewSafeBufioRW(brw *bufio.ReadWriter) *SafeBufioRW {
return &SafeBufioRW{brw: brw}
}
// SendJSON serializes the given data as JSON and sends it with a 4-byte length prefix
func (bu *SafeBufioRW) SendJSON(data interface{}) error {
bu.mutex.Lock()
defer bu.mutex.Unlock()
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
if len(jsonData) > MaxSize {
return errors.New("JSON data exceeds maximum size")
}
// Write the 4-byte length prefix
if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(jsonData))); err != nil {
return err
}
// Write the JSON data
if _, err = bu.brw.Write(jsonData); err != nil {
return err
}
// Flush the writer to ensure data is sent
return bu.brw.Flush()
}
// ReceiveJSON reads a 4-byte length prefix, then reads and unmarshals the JSON
func (bu *SafeBufioRW) ReceiveJSON(dest interface{}) error {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
return err
}
if length > MaxSize {
return errors.New("received JSON data exceeds maximum size")
}
// Read the JSON data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
return err
}
return json.Unmarshal(data, dest)
}
// Receive reads a 4-byte length prefix, then reads the raw data
func (bu *SafeBufioRW) Receive() ([]byte, error) {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
return nil, err
}
if length > MaxSize {
return nil, errors.New("received data exceeds maximum size")
}
// Read the raw data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
return nil, err
}
return data, nil
}
// SendProto serializes the given protobuf message and sends it with a 4-byte length prefix
func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
bu.mutex.Lock()
defer bu.mutex.Unlock()
@@ -110,12 +45,8 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
return err
}
if len(protoData) > MaxSize {
return errors.New("protobuf data exceeds maximum size")
}
// Write the 4-byte length prefix
if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(protoData))); err != nil {
// Write varint length prefix
if err := writeUvarint(bu.brw, uint64(len(protoData))); err != nil {
return err
}
@@ -124,25 +55,19 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
return err
}
// Flush the writer to ensure data is sent
return bu.brw.Flush()
}
// ReceiveProto reads a 4-byte length prefix, then reads and unmarshals the protobuf
func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
// Read varint length prefix
length, err := readUvarint(bu.brw)
if err != nil {
return err
}
if length > MaxSize {
return errors.New("received Protobuf data exceeds maximum size")
}
// Read the Protobuf data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
@@ -152,24 +77,51 @@ func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
return proto.Unmarshal(data, msg)
}
// Write writes raw data to the underlying buffer
func (bu *SafeBufioRW) Write(data []byte) (int, error) {
bu.mutex.Lock()
defer bu.mutex.Unlock()
if len(data) > MaxSize {
return 0, errors.New("data exceeds maximum size")
}
n, err := bu.brw.Write(data)
if err != nil {
return n, err
}
// Flush the writer to ensure data is sent
if err = bu.brw.Flush(); err != nil {
return n, err
}
return n, nil
type CreateMessageOptions struct {
SequenceID string
Latency *gen.ProtoLatencyTracker
}
func CreateMessage(payload proto.Message, payloadType string, opts *CreateMessageOptions) (*gen.ProtoMessage, error) {
msg := &gen.ProtoMessage{
MessageBase: &gen.ProtoMessageBase{
PayloadType: payloadType,
},
}
if opts != nil {
if opts.Latency != nil {
msg.MessageBase.Latency = opts.Latency
} else if opts.SequenceID != "" {
msg.MessageBase.Latency = &gen.ProtoLatencyTracker{
SequenceId: opts.SequenceID,
Timestamps: []*gen.ProtoTimestampEntry{
{
Stage: "created",
Time: timestamppb.Now(),
},
},
}
}
}
// Use reflection to set the oneof field automatically
msgReflect := msg.ProtoReflect()
payloadReflect := payload.ProtoReflect()
oneofDesc := msgReflect.Descriptor().Oneofs().ByName("payload")
if oneofDesc == nil {
return nil, errors.New("payload oneof not found")
}
fields := oneofDesc.Fields()
for i := 0; i < fields.Len(); i++ {
field := fields.Get(i)
if field.Message() != nil && field.Message().FullName() == payloadReflect.Descriptor().FullName() {
msgReflect.Set(field, protoreflect.ValueOfMessage(payloadReflect))
return msg, nil
}
}
return nil, errors.New("payload type not found in oneof")
}

10
packages/relay/internal/connections/datachannel.go
View File

@@ -31,16 +31,18 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
}
// Decode message
var base gen.ProtoMessageInput
var base gen.ProtoMessage
if err := proto.Unmarshal(msg.Data, &base); err != nil {
slog.Error("failed to decode binary DataChannel message", "err", err)
return
}
// Handle message type callback
if callback, ok := ndc.callbacks["input"]; ok {
// Route based on PayloadType
if base.MessageBase != nil && len(base.MessageBase.PayloadType) > 0 {
if callback, ok := ndc.callbacks[base.MessageBase.PayloadType]; ok {
go callback(msg.Data)
} // We don't care about unhandled messages
}
}
})
return ndc

94
packages/relay/internal/connections/messages.go
View File

@@ -1,94 +0,0 @@
package connections
import (
"encoding/json"
"relay/internal/common"
"github.com/pion/webrtc/v4"
)
// MessageBase is the base type for any JSON message
type MessageBase struct {
Type string `json:"payload_type"`
Latency *common.LatencyTracker `json:"latency,omitempty"`
}
type MessageRaw struct {
MessageBase
Data json.RawMessage `json:"data"`
}
func NewMessageRaw(t string, data json.RawMessage) *MessageRaw {
return &MessageRaw{
MessageBase: MessageBase{
Type: t,
},
Data: data,
}
}
type MessageLog struct {
MessageBase
Level string `json:"level"`
Message string `json:"message"`
Time string `json:"time"`
}
func NewMessageLog(t string, level, message, time string) *MessageLog {
return &MessageLog{
MessageBase: MessageBase{
Type: t,
},
Level: level,
Message: message,
Time: time,
}
}
type MessageMetrics struct {
MessageBase
UsageCPU float64 `json:"usage_cpu"`
UsageMemory float64 `json:"usage_memory"`
Uptime uint64 `json:"uptime"`
PipelineLatency float64 `json:"pipeline_latency"`
}
func NewMessageMetrics(t string, usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) *MessageMetrics {
return &MessageMetrics{
MessageBase: MessageBase{
Type: t,
},
UsageCPU: usageCPU,
UsageMemory: usageMemory,
Uptime: uptime,
PipelineLatency: pipelineLatency,
}
}
type MessageICE struct {
MessageBase
Candidate webrtc.ICECandidateInit `json:"candidate"`
}
func NewMessageICE(t string, candidate webrtc.ICECandidateInit) *MessageICE {
return &MessageICE{
MessageBase: MessageBase{
Type: t,
},
Candidate: candidate,
}
}
type MessageSDP struct {
MessageBase
SDP webrtc.SessionDescription `json:"sdp"`
}
func NewMessageSDP(t string, sdp webrtc.SessionDescription) *MessageSDP {
return &MessageSDP{
MessageBase: MessageBase{
Type: t,
},
SDP: sdp,
}
}

13
packages/relay/internal/core/core.go
View File

@@ -10,6 +10,7 @@ import (
"os"
"relay/internal/common"
"relay/internal/shared"
"time"
"github.com/libp2p/go-libp2p"
pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -37,6 +38,16 @@ var globalRelay *Relay
// -- Structs --
// ClientSession tracks browser client connections
type ClientSession struct {
PeerID peer.ID
SessionID string
RoomName string
ConnectedAt time.Time
LastActivity time.Time
ControllerSlots []int32 // Track which controller slots this client owns
}
// Relay structure enhanced with metrics and state
type Relay struct {
*PeerInfo
@@ -48,6 +59,7 @@ type Relay struct {
// Local
LocalRooms *common.SafeMap[ulid.ULID, *shared.Room] // room ID -> local Room struct (hosted by this relay)
LocalMeshConnections *common.SafeMap[peer.ID, *webrtc.PeerConnection] // peer ID -> PeerConnection (connected to this relay)
ClientSessions *common.SafeMap[peer.ID, *ClientSession] // peer ID -> ClientSession
// Protocols
ProtocolRegistry
@@ -144,6 +156,7 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay
PingService: pingSvc,
LocalRooms: common.NewSafeMap[ulid.ULID, *shared.Room](),
LocalMeshConnections: common.NewSafeMap[peer.ID, *webrtc.PeerConnection](),
ClientSessions: common.NewSafeMap[peer.ID, *ClientSession](),
}
// Add network notifier after relay is initialized

698
packages/relay/internal/core/protocol_stream.go
View File

@@ -3,7 +3,6 @@ package core
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -11,6 +10,11 @@ import (
"relay/internal/common"
"relay/internal/connections"
"relay/internal/shared"
"time"
gen "relay/internal/proto"
"google.golang.org/protobuf/proto"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
@@ -69,7 +73,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
var currentRoomName string // Track the current room for this stream
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
var msgWrapper gen.ProtoMessage
err := safeBRW.ReceiveProto(&msgWrapper)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Stream request connection closed by peer", "peer", stream.Conn().RemotePeer())
@@ -82,80 +87,216 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message", "err", err)
continue
if msgWrapper.MessageBase == nil {
slog.Error("No MessageBase in stream request")
_ = stream.Reset()
return
}
switch baseMsg.Type {
switch msgWrapper.MessageBase.PayloadType {
case "request-stream-room":
var rawMsg connections.MessageRaw
if err = json.Unmarshal(data, &rawMsg); err != nil {
slog.Error("Failed to unmarshal raw message for room stream request", "err", err)
reqMsg := msgWrapper.GetClientRequestRoomStream()
if reqMsg != nil {
currentRoomName = reqMsg.RoomName
// Generate session ID if not provided (first connection)
sessionID := reqMsg.SessionId
if sessionID == "" {
ulid, err := common.NewULID()
if err != nil {
slog.Error("Failed to generate session ID", "err", err)
continue
}
var roomName string
if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
slog.Error("Failed to unmarshal room name from raw message", "err", err)
continue
sessionID = ulid.String()
}
currentRoomName = roomName // Store the room name
slog.Info("Received stream request for room", "room", roomName)
session := &ClientSession{
PeerID: stream.Conn().RemotePeer(),
SessionID: sessionID,
RoomName: reqMsg.RoomName,
ConnectedAt: time.Now(),
LastActivity: time.Now(),
}
sp.relay.ClientSessions.Set(stream.Conn().RemotePeer(), session)
room := sp.relay.GetRoomByName(roomName)
slog.Info("Client session established", "peer", session.PeerID, "session", sessionID, "room", reqMsg.RoomName)
// Send session ID back to client
sesMsg, err := common.CreateMessage(
&gen.ProtoClientRequestRoomStream{SessionId: sessionID, RoomName: reqMsg.RoomName},
"session-assigned", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(sesMsg); err != nil {
slog.Error("Failed to send session assignment", "err", err)
}
slog.Info("Received stream request for room", "room", reqMsg.RoomName)
room := sp.relay.GetRoomByName(reqMsg.RoomName)
if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID {
// TODO: Allow forward requests to other relays from here?
slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", roomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", reqMsg.RoomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
// Respond with "request-stream-offline" message with room name
// TODO: Store the peer and send "online" message when the room comes online
roomNameData, err := json.Marshal(roomName)
rawMsg, err := common.CreateMessage(
&gen.ProtoRaw{
Data: reqMsg.RoomName,
},
"request-stream-offline", nil,
)
if err != nil {
slog.Error("Failed to marshal room name for request stream offline", "room", roomName, "err", err)
slog.Error("Failed to create proto message", "err", err)
continue
} else {
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"request-stream-offline",
roomNameData,
)); err != nil {
slog.Error("Failed to send request stream offline message", "room", roomName, "err", err)
}
if err = safeBRW.SendProto(rawMsg); err != nil {
slog.Error("Failed to send request stream offline message", "room", reqMsg.RoomName, "err", err)
}
continue
}
pc, err := common.CreatePeerConnection(func() {
slog.Info("PeerConnection closed for requested stream", "room", roomName)
slog.Info("PeerConnection closed for requested stream", "room", reqMsg.RoomName)
// Cleanup the stream connection
if roomMap, ok := sp.servedConns.Get(roomName); ok {
if roomMap, ok := sp.servedConns.Get(reqMsg.RoomName); ok {
roomMap.Delete(stream.Conn().RemotePeer())
// If the room map is empty, delete it
if roomMap.Len() == 0 {
sp.servedConns.Delete(roomName)
sp.servedConns.Delete(reqMsg.RoomName)
}
}
})
if err != nil {
slog.Error("Failed to create PeerConnection for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create PeerConnection for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
// Add tracks
if room.AudioTrack != nil {
if _, err = pc.AddTrack(room.AudioTrack); err != nil {
slog.Error("Failed to add audio track for requested stream", "room", roomName, "err", err)
// Create participant for this viewer
participant, err := shared.NewParticipant(
"", // session ID will be set if this is a client session
stream.Conn().RemotePeer(),
)
if err != nil {
slog.Error("Failed to create participant", "room", reqMsg.RoomName, "err", err)
continue
}
// If this is a client session, link it
if session, ok := sp.relay.ClientSessions.Get(stream.Conn().RemotePeer()); ok {
participant.SessionID = session.SessionID
}
participant.PeerConnection = pc
// Create per-participant tracks
if room.VideoTrack != nil {
if _, err = pc.AddTrack(room.VideoTrack); err != nil {
slog.Error("Failed to add video track for requested stream", "room", roomName, "err", err)
participant.VideoTrack, err = webrtc.NewTrackLocalStaticRTP(
room.VideoTrack.Codec(),
"video-"+participant.ID.String(),
"nestri-"+reqMsg.RoomName+"-video",
)
if err != nil {
slog.Error("Failed to create participant video track", "room", reqMsg.RoomName, "err", err)
continue
}
rtpSender, err := pc.AddTrack(participant.VideoTrack)
if err != nil {
slog.Error("Failed to add participant video track", "room", reqMsg.RoomName, "err", err)
continue
}
slog.Info("Added video track for participant",
"room", reqMsg.RoomName,
"participant", participant.ID,
"sender_id", fmt.Sprintf("%p", rtpSender))
// Relay packets from channel to track (VIDEO)
go func() {
for pkt := range participant.VideoChan {
// Use a context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
done := make(chan error, 1)
go func() {
done <- participant.VideoTrack.WriteRTP(pkt)
}()
select {
case err := <-done:
cancel()
if err != nil {
if !errors.Is(err, io.ErrClosedPipe) {
slog.Debug("Failed to write video", "room", reqMsg.RoomName, "err", err)
}
return
}
case <-ctx.Done():
cancel()
slog.Error("WriteRTP BLOCKED for >100ms!",
"participant", participant.ID,
"room", reqMsg.RoomName)
// Don't return, continue processing
}
}
}()
}
if room.AudioTrack != nil {
participant.AudioTrack, err = webrtc.NewTrackLocalStaticRTP(
room.AudioTrack.Codec(),
"audio-"+participant.ID.String(),
"nestri-"+reqMsg.RoomName+"-audio",
)
if err != nil {
slog.Error("Failed to create participant audio track", "room", reqMsg.RoomName, "err", err)
continue
}
_, err := pc.AddTrack(participant.AudioTrack)
if err != nil {
slog.Error("Failed to add participant audio track", "room", reqMsg.RoomName, "err", err)
continue
}
// Relay packets from channel to track (AUDIO)
go func() {
for pkt := range participant.AudioChan {
start := time.Now()
if err := participant.AudioTrack.WriteRTP(pkt); err != nil {
if !errors.Is(err, io.ErrClosedPipe) {
slog.Debug("Failed to write audio to participant", "room", reqMsg.RoomName, "err", err)
}
return
}
duration := time.Since(start)
if duration > 50*time.Millisecond {
slog.Warn("Slow audio WriteRTP detected",
"duration", duration,
"participant", participant.ID,
"room", reqMsg.RoomName)
}
}
}()
}
// Add participant to room
room.AddParticipant(participant)
// Cleanup on disconnect
cleanupParticipantID := participant.ID
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
if state == webrtc.PeerConnectionStateClosed ||
state == webrtc.PeerConnectionStateFailed ||
state == webrtc.PeerConnectionStateDisconnected {
slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID)
room.RemoveParticipantByID(cleanupParticipantID)
participant.Close()
}
})
// DataChannel setup
settingOrdered := true
settingMaxRetransmits := uint16(2)
@@ -164,21 +305,84 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
MaxRetransmits: &settingMaxRetransmits,
})
if err != nil {
slog.Error("Failed to create DataChannel for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create DataChannel for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
ndc := connections.NewNestriDataChannel(dc)
ndc.RegisterOnOpen(func() {
slog.Debug("Relay DataChannel opened for requested stream", "room", roomName)
slog.Debug("Relay DataChannel opened for requested stream", "room", reqMsg.RoomName)
})
ndc.RegisterOnClose(func() {
slog.Debug("Relay DataChannel closed for requested stream", "room", roomName)
slog.Debug("Relay DataChannel closed for requested stream", "room", reqMsg.RoomName)
})
ndc.RegisterMessageCallback("input", func(data []byte) {
if room.DataChannel != nil {
if err = room.DataChannel.SendBinary(data); err != nil {
slog.Error("Failed to forward input message from mesh to upstream room", "room", roomName, "err", err)
slog.Error("Failed to forward input message from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
}
}
})
// Track controller input separately
ndc.RegisterMessageCallback("controllerInput", func(data []byte) {
// Parse the message to track controller slots for client sessions
var msgWrapper gen.ProtoMessage
if err = proto.Unmarshal(data, &msgWrapper); err != nil {
slog.Error("Failed to unmarshal controller input", "err", err)
} else if msgWrapper.Payload != nil {
// Get the peer ID for this connection
peerID := stream.Conn().RemotePeer()
// Check if it's a controller attach with assigned slot
if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.Slot >= 0 {
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
// Check if slot already tracked
hasSlot := false
for _, slot := range session.ControllerSlots {
if slot == attach.Slot {
hasSlot = true
break
}
}
if !hasSlot {
session.ControllerSlots = append(session.ControllerSlots, attach.Slot)
session.LastActivity = time.Now()
slog.Info("Controller slot assigned to client session",
"session", session.SessionID,
"slot", attach.Slot,
"total_slots", len(session.ControllerSlots))
}
}
}
// Check if it's a controller detach
if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.Slot >= 0 {
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
newSlots := make([]int32, 0, len(session.ControllerSlots))
for _, slot := range session.ControllerSlots {
if slot != detach.Slot {
newSlots = append(newSlots, slot)
}
}
session.ControllerSlots = newSlots
session.LastActivity = time.Now()
slog.Info("Controller slot removed from client session",
"session", session.SessionID,
"slot", detach.Slot,
"remaining_slots", len(session.ControllerSlots))
}
}
// Update last activity on any controller input
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
session.LastActivity = time.Now()
}
}
// Forward to upstream room
if room.DataChannel != nil {
if err = room.DataChannel.SendBinary(data); err != nil {
slog.Error("Failed to forward controller input from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
}
}
})
@@ -189,8 +393,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE("ice-candidate", candidate.ToJSON())); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", roomName, "err", err)
candInit := candidate.ToJSON()
biggified := uint32(*candInit.SDPMLineIndex)
iceMsg, err := common.CreateMessage(
&gen.ProtoICE{
Candidate: &gen.RTCIceCandidateInit{
Candidate: candInit.Candidate,
SdpMLineIndex: &biggified,
SdpMid: candInit.SDPMid,
},
},
"ice-candidate", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
return
}
if err = safeBRW.SendProto(iceMsg); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", reqMsg.RoomName, "err", err)
return
}
})
@@ -198,23 +418,36 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
// Create offer
offer, err := pc.CreateOffer(nil)
if err != nil {
slog.Error("Failed to create offer for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create offer for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
if err = pc.SetLocalDescription(offer); err != nil {
slog.Error("Failed to set local description for requested stream", "room", roomName, "err", err)
slog.Error("Failed to set local description for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageSDP("offer", offer)); err != nil {
slog.Error("Failed to send offer for requested stream", "room", roomName, "err", err)
offerMsg, err := common.CreateMessage(
&gen.ProtoSDP{
Sdp: &gen.RTCSessionDescriptionInit{
Sdp: offer.SDP,
Type: offer.Type.String(),
},
},
"offer", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(offerMsg); err != nil {
slog.Error("Failed to send offer for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
// Store the connection
roomMap, ok := sp.servedConns.Get(roomName)
roomMap, ok := sp.servedConns.Get(reqMsg.RoomName)
if !ok {
roomMap = common.NewSafeMap[peer.ID, *StreamConnection]()
sp.servedConns.Set(roomName, roomMap)
sp.servedConns.Set(reqMsg.RoomName, roomMap)
}
roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{
pc: pc,
@@ -222,17 +455,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
})
slog.Debug("Sent offer for requested stream")
} else {
slog.Error("Could not get ClientRequestRoomStream for stream request")
}
case "ice-candidate":
var iceMsg connections.MessageICE
if err := json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE message", "err", err)
continue
iceMsg := msgWrapper.GetIce()
if iceMsg != nil {
smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
cand := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
SDPMid: iceMsg.Candidate.SdpMid,
SDPMLineIndex: &smollified,
UsernameFragment: iceMsg.Candidate.UsernameFragment,
}
// Use currentRoomName to get the connection from nested map
if len(currentRoomName) > 0 {
if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil {
if err := conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
if err = conn.pc.AddICECandidate(cand); err != nil {
slog.Error("Failed to add ICE candidate", "err", err)
}
for _, heldIce := range iceHolder {
@@ -244,24 +484,28 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
}
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
} else {
slog.Error("Could not GetIce from ice-candidate")
}
case "answer":
var answerMsg connections.MessageSDP
if err := json.Unmarshal(data, &answerMsg); err != nil {
slog.Error("Failed to unmarshal answer from signaling message", "err", err)
continue
answerMsg := msgWrapper.GetSdp()
if answerMsg != nil {
ansSdp := webrtc.SessionDescription{
SDP: answerMsg.Sdp.Sdp,
Type: webrtc.NewSDPType(answerMsg.Sdp.Type),
}
// Use currentRoomName to get the connection from nested map
if len(currentRoomName) > 0 {
if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok {
if err := conn.pc.SetRemoteDescription(answerMsg.SDP); err != nil {
if err = conn.pc.SetRemoteDescription(ansSdp); err != nil {
slog.Error("Failed to set remote description for answer", "err", err)
continue
}
@@ -273,199 +517,11 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
} else {
slog.Warn("Received answer without active PeerConnection")
}
}
}
}
// requestStream manages the internals of the stream request
func (sp *StreamProtocol) requestStream(stream network.Stream, room *shared.Room) error {
brw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream))
safeBRW := common.NewSafeBufioRW(brw)
slog.Debug("Requesting room stream from peer", "room", room.Name, "peer", stream.Conn().RemotePeer())
// Send room name to the remote peer
roomData, err := json.Marshal(room.Name)
if err != nil {
_ = stream.Close()
return fmt.Errorf("failed to marshal room name: %w", err)
}
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"request-stream-room",
roomData,
)); err != nil {
_ = stream.Close()
return fmt.Errorf("failed to send room request: %w", err)
}
pc, err := common.CreatePeerConnection(func() {
slog.Info("Relay PeerConnection closed for requested stream", "room", room.Name)
_ = stream.Close() // ignore error as may be closed already
// Cleanup the stream connection
if ok := sp.requestedConns.Has(room.Name); ok {
sp.requestedConns.Delete(room.Name)
}
})
if err != nil {
_ = stream.Close()
return fmt.Errorf("failed to create PeerConnection: %w", err)
}
pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
localTrack, _ := webrtc.NewTrackLocalStaticRTP(track.Codec().RTPCodecCapability, track.ID(), "relay-"+room.Name+"-"+track.Kind().String())
slog.Debug("Received track for requested stream", "room", room.Name, "track_kind", track.Kind().String())
room.SetTrack(track.Kind(), localTrack)
go func() {
for {
rtpPacket, _, err := track.ReadRTP()
if err != nil {
if !errors.Is(err, io.EOF) {
slog.Error("Failed to read RTP packet for requested stream room", "room", room.Name, "err", err)
}
break
}
err = localTrack.WriteRTP(rtpPacket)
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("Failed to write RTP to local track for requested stream room", "room", room.Name, "err", err)
break
}
}
}()
})
pc.OnDataChannel(func(dc *webrtc.DataChannel) {
ndc := connections.NewNestriDataChannel(dc)
ndc.RegisterOnOpen(func() {
slog.Debug("Relay DataChannel opened for requested stream", "room", room.Name)
})
ndc.RegisterOnClose(func() {
slog.Debug("Relay DataChannel closed for requested stream", "room", room.Name)
})
// Set the DataChannel in the requestedConns map
if conn, ok := sp.requestedConns.Get(room.Name); ok {
conn.ndc = ndc
} else {
sp.requestedConns.Set(room.Name, &StreamConnection{
pc: pc,
ndc: ndc,
})
}
// We do not handle any messages from upstream here
})
pc.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE(
"ice-candidate",
candidate.ToJSON(),
)); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", room.Name, "err", err)
return
}
})
// Handle incoming messages (offer and candidates)
go func() {
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Connection for requested stream closed by peer", "room", room.Name)
return
}
slog.Error("Failed to receive data for requested stream", "room", room.Name, "err", err)
_ = stream.Reset()
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message for requested stream", "room", room.Name, "err", err)
return
}
switch baseMsg.Type {
case "ice-candidate":
var iceMsg connections.MessageICE
if err = json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE candidate for requested stream", "room", room.Name, "err", err)
continue
}
if conn, ok := sp.requestedConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
slog.Error("Failed to add ICE candidate for requested stream", "room", room.Name, "err", err)
}
// Add held candidates
for _, heldCandidate := range iceHolder {
if err = conn.pc.AddICECandidate(heldCandidate); err != nil {
slog.Error("Failed to add held ICE candidate for requested stream", "room", room.Name, "err", err)
slog.Warn("Could not GetSdp from answer")
}
}
// Clear the held candidates
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
}
case "offer":
var offerMsg connections.MessageSDP
if err = json.Unmarshal(data, &offerMsg); err != nil {
slog.Error("Failed to unmarshal offer for requested stream", "room", room.Name, "err", err)
continue
}
if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
slog.Error("Failed to set remote description for requested stream", "room", room.Name, "err", err)
continue
}
answer, err := pc.CreateAnswer(nil)
if err != nil {
slog.Error("Failed to create answer for requested stream", "room", room.Name, "err", err)
if err = stream.Reset(); err != nil {
slog.Error("Failed to reset stream for requested stream", "err", err)
}
return
}
if err = pc.SetLocalDescription(answer); err != nil {
slog.Error("Failed to set local description for requested stream", "room", room.Name, "err", err)
if err = stream.Reset(); err != nil {
slog.Error("Failed to reset stream for requested stream", "err", err)
}
return
}
if err = safeBRW.SendJSON(connections.NewMessageSDP(
"answer",
answer,
)); err != nil {
slog.Error("Failed to send answer for requested stream", "room", room.Name, "err", err)
continue
}
// Store the connection
sp.requestedConns.Set(room.Name, &StreamConnection{
pc: pc,
ndc: nil,
})
slog.Debug("Sent answer for requested stream", "room", room.Name)
default:
slog.Warn("Unknown signaling message type", "room", room.Name, "type", baseMsg.Type)
}
}
}()
return nil
}
// handleStreamPush manages a stream push from a node (nestri-server)
@@ -476,7 +532,8 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
var room *shared.Room
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
var msgWrapper gen.ProtoMessage
err := safeBRW.ReceiveProto(&msgWrapper)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err)
@@ -489,29 +546,19 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message from base message", "err", err)
continue
if msgWrapper.MessageBase == nil {
slog.Error("No MessageBase in stream push")
_ = stream.Reset()
return
}
switch baseMsg.Type {
switch msgWrapper.MessageBase.PayloadType {
case "push-stream-room":
var rawMsg connections.MessageRaw
if err = json.Unmarshal(data, &rawMsg); err != nil {
slog.Error("Failed to unmarshal room name from data", "err", err)
continue
}
pushMsg := msgWrapper.GetServerPushStream()
if pushMsg != nil {
slog.Info("Received stream push request for room", "room", pushMsg.RoomName)
var roomName string
if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
slog.Error("Failed to unmarshal room name from raw message", "err", err)
continue
}
slog.Info("Received stream push request for room", "room", roomName)
room = sp.relay.GetRoomByName(roomName)
room = sp.relay.GetRoomByName(pushMsg.RoomName)
if room != nil {
if room.OwnerID != sp.relay.ID {
slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID)
@@ -523,30 +570,39 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
}
} else {
// Create a new room if it doesn't exist
room = sp.relay.CreateRoom(roomName)
room = sp.relay.CreateRoom(pushMsg.RoomName)
}
// Respond with an OK with the room name
roomData, err := json.Marshal(room.Name)
resMsg, err := common.CreateMessage(
&gen.ProtoServerPushStream{
RoomName: pushMsg.RoomName,
},
"push-stream-ok", nil,
)
if err != nil {
slog.Error("Failed to marshal room name for push stream response", "err", err)
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"push-stream-ok",
roomData,
)); err != nil {
if err = safeBRW.SendProto(resMsg); err != nil {
slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err)
continue
}
} else {
slog.Error("Failed to GetServerPushStream in push-stream-room")
}
case "ice-candidate":
var iceMsg connections.MessageICE
if err = json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE candidate from data", "err", err)
continue
iceMsg := msgWrapper.GetIce()
if iceMsg != nil {
smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
cand := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
SDPMid: iceMsg.Candidate.SdpMid,
SDPMLineIndex: &smollified,
UsernameFragment: iceMsg.Candidate.UsernameFragment,
}
if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
if err = conn.pc.AddICECandidate(cand); err != nil {
slog.Error("Failed to add ICE candidate for pushed stream", "err", err)
}
for _, heldIce := range iceHolder {
@@ -558,7 +614,10 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
} else {
slog.Error("Failed to GetIce in pushed stream ice-candidate")
}
case "offer":
// Make sure we have room set to push to (set by "push-stream-room")
@@ -567,12 +626,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
continue
}
var offerMsg connections.MessageSDP
if err = json.Unmarshal(data, &offerMsg); err != nil {
slog.Error("Failed to unmarshal offer from data", "err", err)
continue
offerMsg := msgWrapper.GetSdp()
if offerMsg != nil {
offSdp := webrtc.SessionDescription{
SDP: offerMsg.Sdp.Sdp,
Type: webrtc.NewSDPType(offerMsg.Sdp.Type),
}
// Create PeerConnection for the incoming stream
pc, err := common.CreatePeerConnection(func() {
slog.Info("PeerConnection closed for pushed stream", "room", room.Name)
@@ -595,20 +654,19 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
room.DataChannel.RegisterOnClose(func() {
slog.Debug("DataChannel closed for pushed stream", "room", room.Name)
})
room.DataChannel.RegisterMessageCallback("input", func(data []byte) {
if room.DataChannel != nil {
// Pass to servedConns DataChannels for this specific room
// Handle controller feedback reverse-flow (like rumble events coming from game to client)
room.DataChannel.RegisterMessageCallback("controllerInput", func(data []byte) {
// Forward controller input to all viewers
if roomMap, ok := sp.servedConns.Get(room.Name); ok {
roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool {
if conn.ndc != nil {
if err = conn.ndc.SendBinary(data); err != nil {
slog.Error("Failed to forward input message from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
}
}
return true // Continue iteration
return true
})
}
}
})
// Set the DataChannel in the incomingConns map
@@ -627,10 +685,23 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE(
"ice-candidate",
candidate.ToJSON(),
)); err != nil {
candInit := candidate.ToJSON()
biggified := uint32(*candInit.SDPMLineIndex)
iceMsg, err := common.CreateMessage(
&gen.ProtoICE{
Candidate: &gen.RTCIceCandidateInit{
Candidate: candInit.Candidate,
SdpMLineIndex: &biggified,
SdpMid: candInit.SDPMid,
},
},
"ice-candidate", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
return
}
if err = safeBRW.SendProto(iceMsg); err != nil {
slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err)
return
}
@@ -676,11 +747,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
}
}
err = localTrack.WriteRTP(rtpPacket)
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("Failed to write RTP to local track for room", "room", room.Name, "err", err)
break
}
room.BroadcastPacket(remoteTrack.Kind(), rtpPacket)
}
slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())
@@ -690,7 +757,7 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
})
// Set the remote description
if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
if err = pc.SetRemoteDescription(offSdp); err != nil {
slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err)
continue
}
@@ -706,10 +773,20 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageSDP(
"answer",
answer,
)); err != nil {
answerMsg, err := common.CreateMessage(
&gen.ProtoSDP{
Sdp: &gen.RTCSessionDescriptionInit{
Sdp: answer.SDP,
Type: answer.Type.String(),
},
},
"answer", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(answerMsg); err != nil {
slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err)
}
@@ -721,16 +798,17 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
slog.Debug("Sent answer for pushed stream", "room", room.Name)
}
}
}
}
// --- Public Usable Methods ---
// RequestStream sends a request to get room stream from another relay
func (sp *StreamProtocol) RequestStream(ctx context.Context, room *shared.Room, peerID peer.ID) error {
stream, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
_, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
if err != nil {
return fmt.Errorf("failed to create stream: %w", err)
}
return sp.requestStream(stream, room)
return nil /* TODO: This? */
}

48
packages/relay/internal/core/state.go
View File

@@ -5,9 +5,14 @@ import (
"encoding/json"
"errors"
"log/slog"
"relay/internal/common"
"relay/internal/shared"
"time"
gen "relay/internal/proto"
"google.golang.org/protobuf/proto"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
@@ -129,12 +134,51 @@ func (r *Relay) onPeerConnected(peerID peer.ID) {
// onPeerDisconnected marks a peer as disconnected in our status view and removes latency info
func (r *Relay) onPeerDisconnected(peerID peer.ID) {
// Check if this was a client session disconnect
if session, ok := r.ClientSessions.Get(peerID); ok {
slog.Info("Client session disconnected",
"peer", peerID,
"session", session.SessionID,
"room", session.RoomName,
"controller_slots", session.ControllerSlots)
// Send cleanup message to nestri-server if client had active controllers
if len(session.ControllerSlots) > 0 {
room := r.GetRoomByName(session.RoomName)
if room != nil && room.DataChannel != nil {
// Create disconnect notification
disconnectMsg, err := common.CreateMessage(&gen.ProtoClientDisconnected{
SessionId: session.SessionID,
ControllerSlots: session.ControllerSlots,
}, "client-disconnected", nil)
if err != nil {
slog.Error("Failed to create client disconnect message", "err", err)
}
disMarshal, err := proto.Marshal(disconnectMsg)
if err != nil {
slog.Error("Failed to marshal client disconnect message", "err", err)
} else {
if err = room.DataChannel.SendBinary(disMarshal); err != nil {
slog.Error("Failed to send client disconnect notification", "err", err)
} else {
slog.Info("Sent controller cleanup notification to nestri-server",
"session", session.SessionID,
"slots", session.ControllerSlots)
}
}
}
}
r.ClientSessions.Delete(peerID)
return
}
// Relay peer disconnect handling
slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID)
// Remove peer from local mesh peers
if r.Peers.Has(peerID) {
r.Peers.Delete(peerID)
}
// Remove any rooms associated with this peer
if r.Rooms.Has(peerID.String()) {
r.Rooms.Delete(peerID.String())
}

463
packages/relay/internal/proto/messages.pb.go
View File

@@ -73,28 +73,50 @@ func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker {
return nil
}
type ProtoMessageInput struct {
type ProtoMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
MessageBase *ProtoMessageBase `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"`
Data *ProtoInput `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
// Types that are valid to be assigned to Payload:
//
// *ProtoMessage_MouseMove
// *ProtoMessage_MouseMoveAbs
// *ProtoMessage_MouseWheel
// *ProtoMessage_MouseKeyDown
// *ProtoMessage_MouseKeyUp
// *ProtoMessage_KeyDown
// *ProtoMessage_KeyUp
// *ProtoMessage_ControllerAttach
// *ProtoMessage_ControllerDetach
// *ProtoMessage_ControllerButton
// *ProtoMessage_ControllerTrigger
// *ProtoMessage_ControllerStick
// *ProtoMessage_ControllerAxis
// *ProtoMessage_ControllerRumble
// *ProtoMessage_Ice
// *ProtoMessage_Sdp
// *ProtoMessage_Raw
// *ProtoMessage_ClientRequestRoomStream
// *ProtoMessage_ClientDisconnected
// *ProtoMessage_ServerPushStream
Payload isProtoMessage_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ProtoMessageInput) Reset() {
*x = ProtoMessageInput{}
func (x *ProtoMessage) Reset() {
*x = ProtoMessage{}
mi := &file_messages_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ProtoMessageInput) String() string {
func (x *ProtoMessage) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ProtoMessageInput) ProtoMessage() {}
func (*ProtoMessage) ProtoMessage() {}
func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
func (x *ProtoMessage) ProtoReflect() protoreflect.Message {
mi := &file_messages_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -106,25 +128,331 @@ func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead.
func (*ProtoMessageInput) Descriptor() ([]byte, []int) {
// Deprecated: Use ProtoMessage.ProtoReflect.Descriptor instead.
func (*ProtoMessage) Descriptor() ([]byte, []int) {
return file_messages_proto_rawDescGZIP(), []int{1}
}
func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase {
func (x *ProtoMessage) GetMessageBase() *ProtoMessageBase {
if x != nil {
return x.MessageBase
}
return nil
}
func (x *ProtoMessageInput) GetData() *ProtoInput {
func (x *ProtoMessage) GetPayload() isProtoMessage_Payload {
if x != nil {
return x.Data
return x.Payload
}
return nil
}
func (x *ProtoMessage) GetMouseMove() *ProtoMouseMove {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseMove); ok {
return x.MouseMove
}
}
return nil
}
func (x *ProtoMessage) GetMouseMoveAbs() *ProtoMouseMoveAbs {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseMoveAbs); ok {
return x.MouseMoveAbs
}
}
return nil
}
func (x *ProtoMessage) GetMouseWheel() *ProtoMouseWheel {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseWheel); ok {
return x.MouseWheel
}
}
return nil
}
func (x *ProtoMessage) GetMouseKeyDown() *ProtoMouseKeyDown {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseKeyDown); ok {
return x.MouseKeyDown
}
}
return nil
}
func (x *ProtoMessage) GetMouseKeyUp() *ProtoMouseKeyUp {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseKeyUp); ok {
return x.MouseKeyUp
}
}
return nil
}
func (x *ProtoMessage) GetKeyDown() *ProtoKeyDown {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_KeyDown); ok {
return x.KeyDown
}
}
return nil
}
func (x *ProtoMessage) GetKeyUp() *ProtoKeyUp {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_KeyUp); ok {
return x.KeyUp
}
}
return nil
}
func (x *ProtoMessage) GetControllerAttach() *ProtoControllerAttach {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerAttach); ok {
return x.ControllerAttach
}
}
return nil
}
func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerDetach); ok {
return x.ControllerDetach
}
}
return nil
}
func (x *ProtoMessage) GetControllerButton() *ProtoControllerButton {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerButton); ok {
return x.ControllerButton
}
}
return nil
}
func (x *ProtoMessage) GetControllerTrigger() *ProtoControllerTrigger {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerTrigger); ok {
return x.ControllerTrigger
}
}
return nil
}
func (x *ProtoMessage) GetControllerStick() *ProtoControllerStick {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerStick); ok {
return x.ControllerStick
}
}
return nil
}
func (x *ProtoMessage) GetControllerAxis() *ProtoControllerAxis {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerAxis); ok {
return x.ControllerAxis
}
}
return nil
}
func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok {
return x.ControllerRumble
}
}
return nil
}
func (x *ProtoMessage) GetIce() *ProtoICE {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Ice); ok {
return x.Ice
}
}
return nil
}
func (x *ProtoMessage) GetSdp() *ProtoSDP {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Sdp); ok {
return x.Sdp
}
}
return nil
}
func (x *ProtoMessage) GetRaw() *ProtoRaw {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Raw); ok {
return x.Raw
}
}
return nil
}
func (x *ProtoMessage) GetClientRequestRoomStream() *ProtoClientRequestRoomStream {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ClientRequestRoomStream); ok {
return x.ClientRequestRoomStream
}
}
return nil
}
func (x *ProtoMessage) GetClientDisconnected() *ProtoClientDisconnected {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ClientDisconnected); ok {
return x.ClientDisconnected
}
}
return nil
}
func (x *ProtoMessage) GetServerPushStream() *ProtoServerPushStream {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ServerPushStream); ok {
return x.ServerPushStream
}
}
return nil
}
type isProtoMessage_Payload interface {
isProtoMessage_Payload()
}
type ProtoMessage_MouseMove struct {
// Input types
MouseMove *ProtoMouseMove `protobuf:"bytes,2,opt,name=mouse_move,json=mouseMove,proto3,oneof"`
}
type ProtoMessage_MouseMoveAbs struct {
MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,3,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"`
}
type ProtoMessage_MouseWheel struct {
MouseWheel *ProtoMouseWheel `protobuf:"bytes,4,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"`
}
type ProtoMessage_MouseKeyDown struct {
MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,5,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"`
}
type ProtoMessage_MouseKeyUp struct {
MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,6,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"`
}
type ProtoMessage_KeyDown struct {
KeyDown *ProtoKeyDown `protobuf:"bytes,7,opt,name=key_down,json=keyDown,proto3,oneof"`
}
type ProtoMessage_KeyUp struct {
KeyUp *ProtoKeyUp `protobuf:"bytes,8,opt,name=key_up,json=keyUp,proto3,oneof"`
}
type ProtoMessage_ControllerAttach struct {
ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"`
}
type ProtoMessage_ControllerDetach struct {
ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"`
}
type ProtoMessage_ControllerButton struct {
ControllerButton *ProtoControllerButton `protobuf:"bytes,11,opt,name=controller_button,json=controllerButton,proto3,oneof"`
}
type ProtoMessage_ControllerTrigger struct {
ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,12,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"`
}
type ProtoMessage_ControllerStick struct {
ControllerStick *ProtoControllerStick `protobuf:"bytes,13,opt,name=controller_stick,json=controllerStick,proto3,oneof"`
}
type ProtoMessage_ControllerAxis struct {
ControllerAxis *ProtoControllerAxis `protobuf:"bytes,14,opt,name=controller_axis,json=controllerAxis,proto3,oneof"`
}
type ProtoMessage_ControllerRumble struct {
ControllerRumble *ProtoControllerRumble `protobuf:"bytes,15,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"`
}
type ProtoMessage_Ice struct {
// Signaling types
Ice *ProtoICE `protobuf:"bytes,20,opt,name=ice,proto3,oneof"`
}
type ProtoMessage_Sdp struct {
Sdp *ProtoSDP `protobuf:"bytes,21,opt,name=sdp,proto3,oneof"`
}
type ProtoMessage_Raw struct {
Raw *ProtoRaw `protobuf:"bytes,22,opt,name=raw,proto3,oneof"`
}
type ProtoMessage_ClientRequestRoomStream struct {
ClientRequestRoomStream *ProtoClientRequestRoomStream `protobuf:"bytes,23,opt,name=client_request_room_stream,json=clientRequestRoomStream,proto3,oneof"`
}
type ProtoMessage_ClientDisconnected struct {
ClientDisconnected *ProtoClientDisconnected `protobuf:"bytes,24,opt,name=client_disconnected,json=clientDisconnected,proto3,oneof"`
}
type ProtoMessage_ServerPushStream struct {
ServerPushStream *ProtoServerPushStream `protobuf:"bytes,25,opt,name=server_push_stream,json=serverPushStream,proto3,oneof"`
}
func (*ProtoMessage_MouseMove) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseMoveAbs) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseWheel) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseKeyDown) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseKeyUp) isProtoMessage_Payload() {}
func (*ProtoMessage_KeyDown) isProtoMessage_Payload() {}
func (*ProtoMessage_KeyUp) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerButton) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerTrigger) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerStick) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerAxis) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {}
func (*ProtoMessage_Ice) isProtoMessage_Payload() {}
func (*ProtoMessage_Sdp) isProtoMessage_Payload() {}
func (*ProtoMessage_Raw) isProtoMessage_Payload() {}
func (*ProtoMessage_ClientRequestRoomStream) isProtoMessage_Payload() {}
func (*ProtoMessage_ClientDisconnected) isProtoMessage_Payload() {}
func (*ProtoMessage_ServerPushStream) isProtoMessage_Payload() {}
var File_messages_proto protoreflect.FileDescriptor
const file_messages_proto_rawDesc = "" +
@@ -132,10 +460,35 @@ const file_messages_proto_rawDesc = "" +
"\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" +
"\x10ProtoMessageBase\x12!\n" +
"\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" +
"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"v\n" +
"\x11ProtoMessageInput\x12:\n" +
"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x12%\n" +
"\x04data\x18\x02 \x01(\v2\x11.proto.ProtoInputR\x04dataB\x16Z\x14relay/internal/protob\x06proto3"
"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\xef\n" +
"\n" +
"\fProtoMessage\x12:\n" +
"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" +
"\n" +
"mouse_move\x18\x02 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" +
"\x0emouse_move_abs\x18\x03 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" +
"\vmouse_wheel\x18\x04 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" +
"mouseWheel\x12@\n" +
"\x0emouse_key_down\x18\x05 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" +
"\fmouse_key_up\x18\x06 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" +
"mouseKeyUp\x120\n" +
"\bkey_down\x18\a \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" +
"\x06key_up\x18\b \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" +
"\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" +
"\x11controller_detach\x18\n" +
" \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" +
"\x11controller_button\x18\v \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" +
"\x12controller_trigger\x18\f \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" +
"\x10controller_stick\x18\r \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" +
"\x0fcontroller_axis\x18\x0e \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" +
"\x11controller_rumble\x18\x0f \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12#\n" +
"\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" +
"\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" +
"\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" +
"\x1aclient_request_room_stream\x18\x17 \x01(\v2#.proto.ProtoClientRequestRoomStreamH\x00R\x17clientRequestRoomStream\x12Q\n" +
"\x13client_disconnected\x18\x18 \x01(\v2\x1e.proto.ProtoClientDisconnectedH\x00R\x12clientDisconnected\x12L\n" +
"\x12server_push_stream\x18\x19 \x01(\v2\x1c.proto.ProtoServerPushStreamH\x00R\x10serverPushStreamB\t\n" +
"\apayloadB\x16Z\x14relay/internal/protob\x06proto3"
var (
file_messages_proto_rawDescOnce sync.Once
@@ -152,19 +505,57 @@ func file_messages_proto_rawDescGZIP() []byte {
var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_messages_proto_goTypes = []any{
(*ProtoMessageBase)(nil), // 0: proto.ProtoMessageBase
(*ProtoMessageInput)(nil), // 1: proto.ProtoMessageInput
(*ProtoMessage)(nil), // 1: proto.ProtoMessage
(*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker
(*ProtoInput)(nil), // 3: proto.ProtoInput
(*ProtoMouseMove)(nil), // 3: proto.ProtoMouseMove
(*ProtoMouseMoveAbs)(nil), // 4: proto.ProtoMouseMoveAbs
(*ProtoMouseWheel)(nil), // 5: proto.ProtoMouseWheel
(*ProtoMouseKeyDown)(nil), // 6: proto.ProtoMouseKeyDown
(*ProtoMouseKeyUp)(nil), // 7: proto.ProtoMouseKeyUp
(*ProtoKeyDown)(nil), // 8: proto.ProtoKeyDown
(*ProtoKeyUp)(nil), // 9: proto.ProtoKeyUp
(*ProtoControllerAttach)(nil), // 10: proto.ProtoControllerAttach
(*ProtoControllerDetach)(nil), // 11: proto.ProtoControllerDetach
(*ProtoControllerButton)(nil), // 12: proto.ProtoControllerButton
(*ProtoControllerTrigger)(nil), // 13: proto.ProtoControllerTrigger
(*ProtoControllerStick)(nil), // 14: proto.ProtoControllerStick
(*ProtoControllerAxis)(nil), // 15: proto.ProtoControllerAxis
(*ProtoControllerRumble)(nil), // 16: proto.ProtoControllerRumble
(*ProtoICE)(nil), // 17: proto.ProtoICE
(*ProtoSDP)(nil), // 18: proto.ProtoSDP
(*ProtoRaw)(nil), // 19: proto.ProtoRaw
(*ProtoClientRequestRoomStream)(nil), // 20: proto.ProtoClientRequestRoomStream
(*ProtoClientDisconnected)(nil), // 21: proto.ProtoClientDisconnected
(*ProtoServerPushStream)(nil), // 22: proto.ProtoServerPushStream
}
var file_messages_proto_depIdxs = []int32{
2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker
0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase
3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
0, // 1: proto.ProtoMessage.message_base:type_name -> proto.ProtoMessageBase
3, // 2: proto.ProtoMessage.mouse_move:type_name -> proto.ProtoMouseMove
4, // 3: proto.ProtoMessage.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs
5, // 4: proto.ProtoMessage.mouse_wheel:type_name -> proto.ProtoMouseWheel
6, // 5: proto.ProtoMessage.mouse_key_down:type_name -> proto.ProtoMouseKeyDown
7, // 6: proto.ProtoMessage.mouse_key_up:type_name -> proto.ProtoMouseKeyUp
8, // 7: proto.ProtoMessage.key_down:type_name -> proto.ProtoKeyDown
9, // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp
10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach
11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach
12, // 11: proto.ProtoMessage.controller_button:type_name -> proto.ProtoControllerButton
13, // 12: proto.ProtoMessage.controller_trigger:type_name -> proto.ProtoControllerTrigger
14, // 13: proto.ProtoMessage.controller_stick:type_name -> proto.ProtoControllerStick
15, // 14: proto.ProtoMessage.controller_axis:type_name -> proto.ProtoControllerAxis
16, // 15: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble
17, // 16: proto.ProtoMessage.ice:type_name -> proto.ProtoICE
18, // 17: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP
19, // 18: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw
20, // 19: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream
21, // 20: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected
22, // 21: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream
22, // [22:22] is the sub-list for method output_type
22, // [22:22] is the sub-list for method input_type
22, // [22:22] is the sub-list for extension type_name
22, // [22:22] is the sub-list for extension extendee
0, // [0:22] is the sub-list for field type_name
}
func init() { file_messages_proto_init() }
@@ -174,6 +565,28 @@ func file_messages_proto_init() {
}
file_types_proto_init()
file_latency_tracker_proto_init()
file_messages_proto_msgTypes[1].OneofWrappers = []any{
(*ProtoMessage_MouseMove)(nil),
(*ProtoMessage_MouseMoveAbs)(nil),
(*ProtoMessage_MouseWheel)(nil),
(*ProtoMessage_MouseKeyDown)(nil),
(*ProtoMessage_MouseKeyUp)(nil),
(*ProtoMessage_KeyDown)(nil),
(*ProtoMessage_KeyUp)(nil),
(*ProtoMessage_ControllerAttach)(nil),
(*ProtoMessage_ControllerDetach)(nil),
(*ProtoMessage_ControllerButton)(nil),
(*ProtoMessage_ControllerTrigger)(nil),
(*ProtoMessage_ControllerStick)(nil),
(*ProtoMessage_ControllerAxis)(nil),
(*ProtoMessage_ControllerRumble)(nil),
(*ProtoMessage_Ice)(nil),
(*ProtoMessage_Sdp)(nil),
(*ProtoMessage_Raw)(nil),
(*ProtoMessage_ClientRequestRoomStream)(nil),
(*ProtoMessage_ClientDisconnected)(nil),
(*ProtoMessage_ServerPushStream)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{

948
packages/relay/internal/proto/types.pb.go
View File

File diff suppressed because it is too large Load Diff

44
packages/relay/internal/shared/participant.go
View File

@@ -2,43 +2,59 @@ package shared
import (
"fmt"
"log/slog"
"relay/internal/common"
"relay/internal/connections"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/oklog/ulid/v2"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
)
type Participant struct {
ID ulid.ULID
SessionID string // Track session for reconnection
PeerID peer.ID // libp2p peer ID
PeerConnection *webrtc.PeerConnection
DataChannel *connections.NestriDataChannel
// Per-viewer tracks and channels
VideoTrack *webrtc.TrackLocalStaticRTP
AudioTrack *webrtc.TrackLocalStaticRTP
VideoChan chan *rtp.Packet
AudioChan chan *rtp.Packet
}
func NewParticipant() (*Participant, error) {
func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) {
id, err := common.NewULID()
if err != nil {
return nil, fmt.Errorf("failed to create ULID for Participant: %w", err)
}
return &Participant{
ID: id,
SessionID: sessionID,
PeerID: peerID,
VideoChan: make(chan *rtp.Packet, 500),
AudioChan: make(chan *rtp.Packet, 100),
}, nil
}
func (p *Participant) addTrack(trackLocal *webrtc.TrackLocalStaticRTP) error {
rtpSender, err := p.PeerConnection.AddTrack(trackLocal)
// Close cleans up participant resources
func (p *Participant) Close() {
if p.VideoChan != nil {
close(p.VideoChan)
p.VideoChan = nil
}
if p.AudioChan != nil {
close(p.AudioChan)
p.AudioChan = nil
}
if p.PeerConnection != nil {
err := p.PeerConnection.Close()
if err != nil {
return err
slog.Error("Failed to close Participant PeerConnection", err)
}
go func() {
rtcpBuffer := make([]byte, 1400)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuffer); rtcpErr != nil {
break
p.PeerConnection = nil
}
}
}()
return nil
}

111
packages/relay/internal/shared/room.go
View File

@@ -4,9 +4,11 @@ import (
"log/slog"
"relay/internal/common"
"relay/internal/connections"
"time"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/oklog/ulid/v2"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
)
@@ -23,17 +25,31 @@ type Room struct {
VideoTrack *webrtc.TrackLocalStaticRTP
DataChannel *connections.NestriDataChannel
Participants *common.SafeMap[ulid.ULID, *Participant]
// Broadcast queues (unbuffered, fan-out happens async)
videoBroadcastChan chan *rtp.Packet
audioBroadcastChan chan *rtp.Packet
broadcastStop chan struct{}
}
func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room {
return &Room{
r := &Room{
RoomInfo: RoomInfo{
ID: roomID,
Name: name,
OwnerID: ownerID,
},
Participants: common.NewSafeMap[ulid.ULID, *Participant](),
videoBroadcastChan: make(chan *rtp.Packet, 1000), // Large buffer for incoming packets
audioBroadcastChan: make(chan *rtp.Packet, 500),
broadcastStop: make(chan struct{}),
}
// Start async broadcasters
go r.videoBroadcaster()
go r.audioBroadcaster()
return r
}
// AddParticipant adds a Participant to a Room
@@ -42,8 +58,8 @@ func (r *Room) AddParticipant(participant *Participant) {
r.Participants.Set(participant.ID, participant)
}
// Removes a Participant from a Room by participant's ID
func (r *Room) removeParticipantByID(pID ulid.ULID) {
// RemoveParticipantByID removes a Participant from a Room by participant's ID
func (r *Room) RemoveParticipantByID(pID ulid.ULID) {
if _, ok := r.Participants.Get(pID); ok {
r.Participants.Delete(pID)
}
@@ -64,3 +80,92 @@ func (r *Room) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalS
slog.Warn("Unknown track type", "room", r.Name, "trackType", trackType)
}
}
// BroadcastPacket enqueues packet for async broadcast (non-blocking)
func (r *Room) BroadcastPacket(kind webrtc.RTPCodecType, pkt *rtp.Packet) {
start := time.Now()
if kind == webrtc.RTPCodecTypeVideo {
select {
case r.videoBroadcastChan <- pkt:
duration := time.Since(start)
if duration > 10*time.Millisecond {
slog.Warn("Slow video broadcast enqueue", "duration", duration, "room", r.Name)
}
default:
// Broadcast queue full - system overload, drop packet globally
slog.Warn("Video broadcast queue full, dropping packet", "room", r.Name)
}
} else {
select {
case r.audioBroadcastChan <- pkt:
duration := time.Since(start)
if duration > 10*time.Millisecond {
slog.Warn("Slow audio broadcast enqueue", "duration", duration, "room", r.Name)
}
default:
slog.Warn("Audio broadcast queue full, dropping packet", "room", r.Name)
}
}
}
// Close stops the broadcasters
func (r *Room) Close() {
close(r.broadcastStop)
close(r.videoBroadcastChan)
close(r.audioBroadcastChan)
}
// videoBroadcaster runs async fan-out for video packets
func (r *Room) videoBroadcaster() {
for {
select {
case pkt := <-r.videoBroadcastChan:
// Fan out to all participants without blocking
r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool {
if participant.VideoChan != nil {
// Clone packet for each participant to avoid shared pointer issues
clonedPkt := pkt.Clone()
select {
case participant.VideoChan <- clonedPkt:
// Sent
default:
// Participant slow, drop packet
slog.Debug("Dropped video packet for slow participant",
"room", r.Name,
"participant", participant.ID)
}
}
return true
})
case <-r.broadcastStop:
return
}
}
}
// audioBroadcaster runs async fan-out for audio packets
func (r *Room) audioBroadcaster() {
for {
select {
case pkt := <-r.audioBroadcastChan:
r.Participants.Range(func(_ ulid.ULID, participant *Participant) bool {
if participant.AudioChan != nil {
// Clone packet for each participant to avoid shared pointer issues
clonedPkt := pkt.Clone()
select {
case participant.AudioChan <- clonedPkt:
// Sent
default:
// Participant slow, drop packet
slog.Debug("Dropped audio packet for slow participant",
"room", r.Name,
"participant", participant.ID)
}
}
return true
})
case <-r.broadcastStop:
return
}
}
}

1
packages/server/Cargo.lock generated
View File

@@ -3151,6 +3151,7 @@ dependencies = [
"tokio-stream",
"tracing",
"tracing-subscriber",
"unsigned-varint 0.8.0",
"vimputti",
"webrtc",
]

1
packages/server/Cargo.toml
View File

@@ -40,3 +40,4 @@ libp2p-tcp = { version = "0.44", features = ["tokio"] }
libp2p-websocket = "0.45"
dashmap = "6.1"
anyhow = "1.0"
unsigned-varint = "0.8"

155
packages/server/src/input/controller.rs
View File

@@ -1,7 +1,5 @@
use crate::proto::proto::proto_input::InputType::{
ControllerAttach, ControllerAxis, ControllerButton, ControllerDetach, ControllerRumble,
ControllerStick, ControllerTrigger,
};
use crate::proto::proto::ProtoControllerAttach;
use crate::proto::proto::proto_message::Payload;
use anyhow::Result;
use std::collections::HashMap;
use std::sync::Arc;
@@ -48,58 +46,87 @@ impl ControllerInput {
pub struct ControllerManager {
vimputti_client: Arc<vimputti::client::VimputtiClient>,
cmd_tx: mpsc::Sender<crate::proto::proto::ProtoInput>,
cmd_tx: mpsc::Sender<Payload>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, // (slot, strong, weak, duration_ms)
attach_tx: mpsc::Sender<ProtoControllerAttach>,
}
impl ControllerManager {
pub fn new(
vimputti_client: Arc<vimputti::client::VimputtiClient>,
) -> Result<(Self, mpsc::Receiver<(u32, u16, u16, u16)>)> {
let (cmd_tx, cmd_rx) = mpsc::channel(100);
let (rumble_tx, rumble_rx) = mpsc::channel(100);
) -> Result<(
Self,
mpsc::Receiver<(u32, u16, u16, u16)>,
mpsc::Receiver<ProtoControllerAttach>,
)> {
let (cmd_tx, cmd_rx) = mpsc::channel(512);
let (rumble_tx, rumble_rx) = mpsc::channel(256);
let (attach_tx, attach_rx) = mpsc::channel(64);
tokio::spawn(command_loop(
cmd_rx,
vimputti_client.clone(),
rumble_tx.clone(),
attach_tx.clone(),
));
Ok((
Self {
vimputti_client,
cmd_tx,
rumble_tx,
attach_tx,
},
rumble_rx,
attach_rx,
))
}
pub async fn send_command(&self, input: crate::proto::proto::ProtoInput) -> Result<()> {
self.cmd_tx.send(input).await?;
pub async fn send_command(&self, payload: Payload) -> Result<()> {
self.cmd_tx.send(payload).await?;
Ok(())
}
}
struct ControllerSlot {
controller: ControllerInput,
session_id: String,
last_activity: std::time::Instant,
}
// Returns first free controller slot from 0-7
fn get_free_slot(controllers: &HashMap<u32, ControllerSlot>) -> Option<u32> {
for slot in 0..8 {
if !controllers.contains_key(&slot) {
return Some(slot);
}
}
None
}
async fn command_loop(
mut cmd_rx: mpsc::Receiver<crate::proto::proto::ProtoInput>,
mut cmd_rx: mpsc::Receiver<Payload>,
vimputti_client: Arc<vimputti::client::VimputtiClient>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>,
attach_tx: mpsc::Sender<ProtoControllerAttach>,
) {
let mut controllers: HashMap<u32, ControllerInput> = HashMap::new();
while let Some(input) = cmd_rx.recv().await {
if let Some(input_type) = input.input_type {
match input_type {
ControllerAttach(data) => {
// Check if controller already exists in the slot, if so, ignore
if controllers.contains_key(&(data.slot as u32)) {
tracing::warn!(
"Controller slot {} already occupied, ignoring attach",
data.slot
);
} else {
let mut controllers: HashMap<u32, ControllerSlot> = HashMap::new();
while let Some(payload) = cmd_rx.recv().await {
match payload {
Payload::ControllerAttach(data) => {
let session_id = data.session_id.clone();
// Check if this session already has a slot (reconnection)
let existing_slot = controllers
.iter()
.find(|(_, slot)| slot.session_id == session_id && !session_id.is_empty())
.map(|(slot_num, _)| *slot_num);
let slot = existing_slot.or_else(|| get_free_slot(&controllers));
if let Some(slot) = slot {
if let Ok(mut controller) =
ControllerInput::new(data.id.clone(), &vimputti_client).await
{
let slot = data.slot as u32;
let rumble_tx = rumble_tx.clone();
let attach_tx = attach_tx.clone();
controller
.device_mut()
@@ -116,28 +143,58 @@ async fn command_loop(
})
.ok();
controllers.insert(data.slot as u32, controller);
tracing::info!("Controller {} attached to slot {}", data.id, data.slot);
// Return to attach_tx what slot was assigned
let attach_info = ProtoControllerAttach {
id: data.id.clone(),
slot: slot as i32,
session_id: session_id.clone(),
};
match attach_tx.send(attach_info).await {
Ok(_) => {
controllers.insert(
slot,
ControllerSlot {
controller,
session_id: session_id.clone(),
last_activity: std::time::Instant::now(),
},
);
tracing::info!(
"Controller {} attached to slot {} (session: {})",
data.id,
slot,
session_id
);
}
Err(e) => {
tracing::error!(
"Failed to send attach info for slot {}: {}",
slot,
e
);
}
}
} else {
tracing::error!(
"Failed to create controller of type {} for slot {}",
data.id,
data.slot
slot
);
}
}
}
ControllerDetach(data) => {
Payload::ControllerDetach(data) => {
if controllers.remove(&(data.slot as u32)).is_some() {
tracing::info!("Controller detached from slot {}", data.slot);
} else {
tracing::warn!("No controller found in slot {} to detach", data.slot);
}
}
ControllerButton(data) => {
Payload::ControllerButton(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) {
let device = controller.device();
let device = controller.controller.device();
device.button(button, data.pressed);
device.sync();
}
@@ -145,9 +202,9 @@ async fn command_loop(
tracing::warn!("Controller slot {} not found for button event", data.slot);
}
}
ControllerStick(data) => {
Payload::ControllerStick(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
let device = controller.controller.device();
if data.stick == 0 {
// Left stick
device.axis(vimputti::Axis::LeftStickX, data.x);
@@ -164,9 +221,9 @@ async fn command_loop(
tracing::warn!("Controller slot {} not found for stick event", data.slot);
}
}
ControllerTrigger(data) => {
Payload::ControllerTrigger(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
let device = controller.controller.device();
if data.trigger == 0 {
// Left trigger
device.axis(vimputti::Axis::LowerLeftTrigger, data.value);
@@ -179,9 +236,9 @@ async fn command_loop(
tracing::warn!("Controller slot {} not found for trigger event", data.slot);
}
}
ControllerAxis(data) => {
Payload::ControllerAxis(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
let device = controller.controller.device();
if data.axis == 0 {
// dpad x
device.axis(vimputti::Axis::DPadX, data.value);
@@ -192,14 +249,32 @@ async fn command_loop(
device.sync();
}
}
// Rumble will be outgoing event..
ControllerRumble(_) => {
//no-op
Payload::ClientDisconnected(data) => {
tracing::info!(
"Client disconnected, cleaning up controller slots: {:?} (client session: {})",
data.controller_slots,
data.session_id
);
// Remove all controllers for the disconnected slots
for slot in &data.controller_slots {
if controllers.remove(&(*slot as u32)).is_some() {
tracing::info!(
"Removed controller from slot {} (client session: {})",
slot,
data.session_id
);
} else {
tracing::warn!(
"No controller found in slot {} to cleanup (client session: {})",
slot,
data.session_id
);
}
}
}
_ => {
//no-op
}
}
}
}
}

16
packages/server/src/main.rs
View File

@@ -3,7 +3,6 @@ mod enc_helper;
mod gpu;
mod input;
mod latency;
mod messages;
mod nestrisink;
mod p2p;
mod proto;
@@ -257,11 +256,15 @@ async fn main() -> Result<(), Box<dyn Error>> {
None
}
};
let (controller_manager, rumble_rx) = if let Some(vclient) = vimputti_client {
let (controller_manager, rumble_rx) = ControllerManager::new(vclient)?;
(Some(Arc::new(controller_manager)), Some(rumble_rx))
let (controller_manager, rumble_rx, attach_rx) = if let Some(vclient) = vimputti_client {
let (controller_manager, rumble_rx, attach_rx) = ControllerManager::new(vclient)?;
(
Some(Arc::new(controller_manager)),
Some(rumble_rx),
Some(attach_rx),
)
} else {
(None, None)
(None, None, None)
};
/*** PIPELINE CREATION ***/
@@ -416,6 +419,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
video_source.clone(),
controller_manager,
rumble_rx,
attach_rx,
)
.await?;
let webrtcsink = BaseWebRTCSink::with_signaller(Signallable::from(signaller.clone()));
@@ -550,7 +554,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
}
// Make sure QOS is disabled to avoid latency
video_encoder.set_property("qos", false);
video_encoder.set_property("qos", true);
// Optimize latency of pipeline
video_source

50
packages/server/src/messages.rs
View File

@@ -1,50 +0,0 @@
use crate::latency::LatencyTracker;
use serde::{Deserialize, Serialize};
use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageBase {
pub payload_type: String,
pub latency: Option<LatencyTracker>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageRaw {
#[serde(flatten)]
pub base: MessageBase,
pub data: serde_json::Value,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageLog {
#[serde(flatten)]
pub base: MessageBase,
pub level: String,
pub message: String,
pub time: String,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageMetrics {
#[serde(flatten)]
pub base: MessageBase,
pub usage_cpu: f64,
pub usage_memory: f64,
pub uptime: u64,
pub pipeline_latency: f64,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageICE {
#[serde(flatten)]
pub base: MessageBase,
pub candidate: RTCIceCandidateInit,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageSDP {
#[serde(flatten)]
pub base: MessageBase,
pub sdp: RTCSessionDescription,
}

248
packages/server/src/nestrisink/imp.rs
View File

@@ -1,11 +1,11 @@
use crate::input::controller::ControllerManager;
use crate::messages::{MessageBase, MessageICE, MessageRaw, MessageSDP};
use crate::p2p::p2p::NestriConnection;
use crate::p2p::p2p_protocol_stream::NestriStreamProtocol;
use crate::proto::proto::proto_input::InputType::{
KeyDown, KeyUp, MouseKeyDown, MouseKeyUp, MouseMove, MouseMoveAbs, MouseWheel,
use crate::proto::proto::proto_message::Payload;
use crate::proto::proto::{
ProtoControllerAttach, ProtoControllerRumble, ProtoIce, ProtoMessage, ProtoSdp,
ProtoServerPushStream, RtcIceCandidateInit, RtcSessionDescriptionInit,
};
use crate::proto::proto::{ProtoInput, ProtoMessageInput};
use anyhow::Result;
use glib::subclass::prelude::*;
use gstreamer::glib;
@@ -16,8 +16,6 @@ use parking_lot::RwLock as PLRwLock;
use prost::Message;
use std::sync::{Arc, LazyLock};
use tokio::sync::{Mutex, mpsc};
use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
pub struct Signaller {
stream_room: PLRwLock<Option<String>>,
@@ -26,6 +24,7 @@ pub struct Signaller {
data_channel: PLRwLock<Option<Arc<gstreamer_webrtc::WebRTCDataChannel>>>,
controller_manager: PLRwLock<Option<Arc<ControllerManager>>>,
rumble_rx: Mutex<Option<mpsc::Receiver<(u32, u16, u16, u16)>>>,
attach_rx: Mutex<Option<mpsc::Receiver<ProtoControllerAttach>>>,
}
impl Default for Signaller {
fn default() -> Self {
@@ -36,6 +35,7 @@ impl Default for Signaller {
data_channel: PLRwLock::new(None),
controller_manager: PLRwLock::new(None),
rumble_rx: Mutex::new(None),
attach_rx: Mutex::new(None),
}
}
}
@@ -74,11 +74,23 @@ impl Signaller {
*self.rumble_rx.lock().await = Some(rumble_rx);
}
// Change getter to take ownership:
pub async fn take_rumble_rx(&self) -> Option<mpsc::Receiver<(u32, u16, u16, u16)>> {
self.rumble_rx.lock().await.take()
}
pub async fn set_attach_rx(
&self,
attach_rx: mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>,
) {
*self.attach_rx.lock().await = Some(attach_rx);
}
pub async fn take_attach_rx(
&self,
) -> Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>> {
self.attach_rx.lock().await.take()
}
pub fn set_data_channel(&self, data_channel: gstreamer_webrtc::WebRTCDataChannel) {
*self.data_channel.write() = Some(Arc::new(data_channel));
}
@@ -95,59 +107,70 @@ impl Signaller {
};
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("answer", move |data| {
if let Ok(message) = serde_json::from_slice::<MessageSDP>(&data) {
let sdp = gst_sdp::SDPMessage::parse_buffer(message.sdp.sdp.as_bytes())
.map_err(|e| anyhow::anyhow!("Invalid SDP in 'answer': {e:?}"))?;
let answer = WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
Ok(self_obj.emit_by_name::<()>(
stream_protocol.register_callback("answer", move |msg| {
if let Some(payload) = msg.payload {
match payload {
Payload::Sdp(sdp) => {
if let Some(sdp) = sdp.sdp {
let sdp = gst_sdp::SDPMessage::parse_buffer(sdp.sdp.as_bytes())
.map_err(|e| {
anyhow::anyhow!("Invalid SDP in 'answer': {e:?}")
})?;
let answer =
WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
return Ok(self_obj.emit_by_name::<()>(
"session-description",
&[&"unique-session-id", &answer],
))
} else {
anyhow::bail!("Failed to decode SDP message");
));
}
}
_ => {
tracing::warn!("Unexpected payload type for answer");
return Ok(());
}
}
} else {
anyhow::bail!("Failed to decode answer message");
}
Ok(())
});
}
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("ice-candidate", move |data| {
if let Ok(message) = serde_json::from_slice::<MessageICE>(&data) {
let candidate = message.candidate;
let sdp_m_line_index = candidate.sdp_mline_index.unwrap_or(0) as u32;
let sdp_mid = candidate.sdp_mid;
Ok(self_obj.emit_by_name::<()>(
stream_protocol.register_callback("ice-candidate", move |msg| {
if let Some(payload) = msg.payload {
match payload {
Payload::Ice(ice) => {
if let Some(candidate) = ice.candidate {
let sdp_m_line_index = candidate.sdp_m_line_index.unwrap_or(0);
return Ok(self_obj.emit_by_name::<()>(
"handle-ice",
&[
&"unique-session-id",
&sdp_m_line_index,
&sdp_mid,
&candidate.sdp_mid,
&candidate.candidate,
],
))
));
}
}
_ => {
tracing::warn!("Unexpected payload type for ice-candidate");
return Ok(());
}
}
} else {
anyhow::bail!("Failed to decode ICE message");
}
Ok(())
});
}
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("push-stream-ok", move |data| {
if let Ok(answer) = serde_json::from_slice::<MessageRaw>(&data) {
// Decode room name string
if let Some(room_name) = answer.data.as_str() {
gstreamer::info!(
gstreamer::CAT_DEFAULT,
"Received OK answer for room: {}",
room_name
);
} else {
gstreamer::error!(
gstreamer::CAT_DEFAULT,
"Failed to decode room name from answer"
);
}
stream_protocol.register_callback("push-stream-ok", move |msg| {
if let Some(payload) = msg.payload {
return match payload {
Payload::ServerPushStream(_res) => {
// Send our SDP offer
Ok(self_obj.emit_by_name::<()>(
"session-requested",
@@ -157,6 +180,12 @@ impl Signaller {
&None::<WebRTCSessionDescription>,
],
))
}
_ => {
tracing::warn!("Unexpected payload type for push-stream-ok");
Ok(())
}
};
} else {
anyhow::bail!("Failed to decode answer");
}
@@ -200,12 +229,14 @@ impl Signaller {
// Spawn async task to take the receiver and set up
tokio::spawn(async move {
let rumble_rx = signaller.imp().take_rumble_rx().await;
let attach_rx = signaller.imp().take_attach_rx().await;
let controller_manager =
signaller.imp().get_controller_manager();
setup_data_channel(
controller_manager,
rumble_rx,
attach_rx,
data_channel,
&wayland_src,
);
@@ -243,19 +274,18 @@ impl SignallableImpl for Signaller {
return;
};
let push_msg = MessageRaw {
base: MessageBase {
payload_type: "push-stream-room".to_string(),
latency: None,
},
data: serde_json::Value::from(stream_room),
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
let push_msg = crate::proto::create_message(
Payload::ServerPushStream(ProtoServerPushStream {
room_name: stream_room,
}),
"push-stream-room",
None,
);
if let Err(e) = stream_protocol.send_message(&push_msg) {
tracing::error!("Failed to send push stream room message: {:?}", e);
}
@@ -266,20 +296,22 @@ impl SignallableImpl for Signaller {
}
fn send_sdp(&self, _session_id: &str, sdp: &WebRTCSessionDescription) {
let sdp_message = MessageSDP {
base: MessageBase {
payload_type: "offer".to_string(),
latency: None,
},
sdp: RTCSessionDescription::offer(sdp.sdp().as_text().unwrap()).unwrap(),
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
if let Err(e) = stream_protocol.send_message(&sdp_message) {
let sdp_msg = crate::proto::create_message(
Payload::Sdp(ProtoSdp {
sdp: Some(RtcSessionDescriptionInit {
sdp: sdp.sdp().as_text().unwrap(),
r#type: "offer".to_string(),
}),
}),
"offer",
None,
);
if let Err(e) = stream_protocol.send_message(&sdp_msg) {
tracing::error!("Failed to send SDP message: {:?}", e);
}
}
@@ -291,26 +323,25 @@ impl SignallableImpl for Signaller {
sdp_m_line_index: u32,
sdp_mid: Option<String>,
) {
let candidate_init = RTCIceCandidateInit {
candidate: candidate.to_string(),
sdp_mid,
sdp_mline_index: Some(sdp_m_line_index as u16),
..Default::default()
};
let ice_message = MessageICE {
base: MessageBase {
payload_type: "ice-candidate".to_string(),
latency: None,
},
candidate: candidate_init,
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
if let Err(e) = stream_protocol.send_message(&ice_message) {
let candidate_init = RtcIceCandidateInit {
candidate: candidate.to_string(),
sdp_mid,
sdp_m_line_index: Some(sdp_m_line_index),
..Default::default() //username_fragment: Some(session_id.to_string()), TODO: required?
};
let ice_msg = crate::proto::create_message(
Payload::Ice(ProtoIce {
candidate: Some(candidate_init),
}),
"ice-candidate",
None,
);
if let Err(e) = stream_protocol.send_message(&ice_msg) {
tracing::error!("Failed to send ICE candidate message: {:?}", e);
}
}
@@ -352,6 +383,7 @@ impl ObjectImpl for Signaller {
fn setup_data_channel(
controller_manager: Option<Arc<ControllerManager>>,
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>, // (slot, strong, weak, duration_ms)
attach_rx: Option<mpsc::Receiver<ProtoControllerAttach>>,
data_channel: Arc<gstreamer_webrtc::WebRTCDataChannel>,
wayland_src: &gstreamer::Element,
) {
@@ -361,11 +393,11 @@ fn setup_data_channel(
// Spawn async processor
tokio::spawn(async move {
while let Some(data) = rx.recv().await {
match ProtoMessageInput::decode(data.as_slice()) {
Ok(message_input) => {
if let Some(message_base) = message_input.message_base {
match ProtoMessage::decode(data.as_slice()) {
Ok(msg_wrapper) => {
if let Some(message_base) = msg_wrapper.message_base {
if message_base.payload_type == "input" {
if let Some(input_data) = message_input.data {
if let Some(input_data) = msg_wrapper.payload {
if let Some(event) = handle_input_message(input_data) {
// Send the event to wayland source, result bool is ignored
let _ = wayland_src.send_event(event);
@@ -373,7 +405,7 @@ fn setup_data_channel(
}
} else if message_base.payload_type == "controllerInput" {
if let Some(controller_manager) = &controller_manager {
if let Some(input_data) = message_input.data {
if let Some(input_data) = msg_wrapper.payload {
let _ = controller_manager.send_command(input_data).await;
}
}
@@ -392,25 +424,16 @@ fn setup_data_channel(
let data_channel_clone = data_channel.clone();
tokio::spawn(async move {
while let Some((slot, strong, weak, duration_ms)) = rumble_rx.recv().await {
let rumble_msg = ProtoMessageInput {
message_base: Some(crate::proto::proto::ProtoMessageBase {
payload_type: "controllerInput".to_string(),
latency: None,
}),
data: Some(ProtoInput {
input_type: Some(
crate::proto::proto::proto_input::InputType::ControllerRumble(
crate::proto::proto::ProtoControllerRumble {
r#type: "ControllerRumble".to_string(),
let rumble_msg = crate::proto::create_message(
Payload::ControllerRumble(ProtoControllerRumble {
slot: slot as i32,
low_frequency: weak as i32,
high_frequency: strong as i32,
duration: duration_ms as i32,
},
),
),
}),
};
"controllerInput",
None,
);
let data = rumble_msg.encode_to_vec();
let bytes = glib::Bytes::from_owned(data);
@@ -422,6 +445,27 @@ fn setup_data_channel(
});
}
// Spawn attach sender
if let Some(mut attach_rx) = attach_rx {
let data_channel_clone = data_channel.clone();
tokio::spawn(async move {
while let Some(attach_msg) = attach_rx.recv().await {
let proto_msg = crate::proto::create_message(
Payload::ControllerAttach(attach_msg),
"controllerInput",
None,
);
let data = proto_msg.encode_to_vec();
let bytes = glib::Bytes::from_owned(data);
if let Err(e) = data_channel_clone.send_data_full(Some(&bytes)) {
tracing::warn!("Failed to send controller attach data: {}", e);
}
}
});
}
data_channel.connect_on_message_data(move |_data_channel, data| {
if let Some(data) = data {
let _ = tx.send(data.to_vec());
@@ -429,10 +473,9 @@ fn setup_data_channel(
});
}
fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
if let Some(input_type) = input_msg.input_type {
match input_type {
MouseMove(data) => {
fn handle_input_message(payload: Payload) -> Option<gstreamer::Event> {
match payload {
Payload::MouseMove(data) => {
let structure = gstreamer::Structure::builder("MouseMoveRelative")
.field("pointer_x", data.x as f64)
.field("pointer_y", data.y as f64)
@@ -440,7 +483,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseMoveAbs(data) => {
Payload::MouseMoveAbs(data) => {
let structure = gstreamer::Structure::builder("MouseMoveAbsolute")
.field("pointer_x", data.x as f64)
.field("pointer_y", data.y as f64)
@@ -448,7 +491,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
KeyDown(data) => {
Payload::KeyDown(data) => {
let structure = gstreamer::Structure::builder("KeyboardKey")
.field("key", data.key as u32)
.field("pressed", true)
@@ -456,7 +499,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
KeyUp(data) => {
Payload::KeyUp(data) => {
let structure = gstreamer::Structure::builder("KeyboardKey")
.field("key", data.key as u32)
.field("pressed", false)
@@ -464,7 +507,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseWheel(data) => {
Payload::MouseWheel(data) => {
let structure = gstreamer::Structure::builder("MouseAxis")
.field("x", data.x as f64)
.field("y", data.y as f64)
@@ -472,7 +515,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseKeyDown(data) => {
Payload::MouseKeyDown(data) => {
let structure = gstreamer::Structure::builder("MouseButton")
.field("button", data.key as u32)
.field("pressed", true)
@@ -480,7 +523,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseKeyUp(data) => {
Payload::MouseKeyUp(data) => {
let structure = gstreamer::Structure::builder("MouseButton")
.field("button", data.key as u32)
.field("pressed", false)
@@ -490,7 +533,4 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
}
_ => None,
}
} else {
None
}
}

4
packages/server/src/nestrisink/mod.rs
View File

@@ -19,6 +19,7 @@ impl NestriSignaller {
wayland_src: Arc<gstreamer::Element>,
controller_manager: Option<Arc<ControllerManager>>,
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>,
attach_rx: Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>>,
) -> Result<Self, Box<dyn std::error::Error>> {
let obj: Self = glib::Object::new();
obj.imp().set_stream_room(room);
@@ -30,6 +31,9 @@ impl NestriSignaller {
if let Some(rumble_rx) = rumble_rx {
obj.imp().set_rumble_rx(rumble_rx).await;
}
if let Some(attach_rx) = attach_rx {
obj.imp().set_attach_rx(attach_rx).await;
}
Ok(obj)
}
}

29
packages/server/src/p2p/p2p_protocol_stream.rs
View File

@@ -3,21 +3,22 @@ use crate::p2p::p2p_safestream::SafeStream;
use anyhow::Result;
use dashmap::DashMap;
use libp2p::StreamProtocol;
use prost::Message;
use std::sync::Arc;
use tokio::sync::mpsc;
// Cloneable callback type
pub type CallbackInner = dyn Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static;
pub type CallbackInner = dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static;
pub struct Callback(Arc<CallbackInner>);
impl Callback {
pub fn new<F>(f: F) -> Self
where
F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
{
Callback(Arc::new(f))
}
pub fn call(&self, data: Vec<u8>) -> Result<()> {
pub fn call(&self, data: crate::proto::proto::ProtoMessage) -> Result<()> {
self.0(data)
}
}
@@ -104,15 +105,17 @@ impl NestriStreamProtocol {
}
};
match serde_json::from_slice::<crate::messages::MessageBase>(&data) {
Ok(base_message) => {
let response_type = base_message.payload_type;
match crate::proto::proto::ProtoMessage::decode(data.as_slice()) {
Ok(message) => {
if let Some(base_message) = &message.message_base {
let response_type = &base_message.payload_type;
let response_type = response_type.clone();
// With DashMap, we don't need explicit locking
// we just get the callback directly if it exists
if let Some(callback) = callbacks.get(&response_type) {
// Execute the callback
if let Err(e) = callback.call(data.clone()) {
if let Err(e) = callback.call(message) {
tracing::error!(
"Callback for response type '{}' errored: {:?}",
response_type,
@@ -125,6 +128,9 @@ impl NestriStreamProtocol {
response_type
);
}
} else {
tracing::error!("No base message in decoded protobuf message",);
}
}
Err(e) => {
tracing::error!("Failed to decode message: {}", e);
@@ -154,8 +160,9 @@ impl NestriStreamProtocol {
})
}
pub fn send_message<M: serde::Serialize>(&self, message: &M) -> Result<()> {
let json_data = serde_json::to_vec(message)?;
pub fn send_message(&self, message: &crate::proto::proto::ProtoMessage) -> Result<()> {
let mut buf = Vec::new();
message.encode(&mut buf)?;
let Some(tx) = &self.tx else {
return Err(anyhow::Error::msg(
if self.read_handle.is_none() && self.write_handle.is_none() {
@@ -165,13 +172,13 @@ impl NestriStreamProtocol {
},
));
};
tx.try_send(json_data)?;
tx.try_send(buf)?;
Ok(())
}
pub fn register_callback<F>(&self, response_type: &str, callback: F)
where
F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
{
self.callbacks
.insert(response_type.to_string(), Callback::new(callback));

52
packages/server/src/p2p/p2p_safestream.rs
View File

@@ -1,11 +1,9 @@
use anyhow::Result;
use byteorder::{BigEndian, ByteOrder};
use libp2p::futures::io::{ReadHalf, WriteHalf};
use libp2p::futures::{AsyncReadExt, AsyncWriteExt};
use std::sync::Arc;
use tokio::sync::Mutex;
const MAX_SIZE: usize = 1024 * 1024; // 1MB
use unsigned_varint::{decode, encode};
pub struct SafeStream {
stream_read: Arc<Mutex<ReadHalf<libp2p::Stream>>>,
@@ -29,34 +27,52 @@ impl SafeStream {
}
async fn send_with_length_prefix(&self, data: &[u8]) -> Result<()> {
if data.len() > MAX_SIZE {
anyhow::bail!("Data exceeds maximum size");
}
let mut buffer = Vec::with_capacity(4 + data.len());
buffer.extend_from_slice(&(data.len() as u32).to_be_bytes()); // Length prefix
buffer.extend_from_slice(data); // Payload
let mut stream_write = self.stream_write.lock().await;
stream_write.write_all(&buffer).await?; // Single write
// Encode length as varint
let mut length_buf = encode::usize_buffer();
let length_bytes = encode::usize(data.len(), &mut length_buf);
// Write varint length prefix
stream_write.write_all(length_bytes).await?;
// Write payload
stream_write.write_all(data).await?;
stream_write.flush().await?;
Ok(())
}
async fn receive_with_length_prefix(&self) -> Result<Vec<u8>> {
let mut stream_read = self.stream_read.lock().await;
// Read length prefix + data in one syscall
let mut length_prefix = [0u8; 4];
stream_read.read_exact(&mut length_prefix).await?;
let length = BigEndian::read_u32(&length_prefix) as usize;
// Read varint length prefix (up to 10 bytes for u64)
let mut length_buf = Vec::new();
let mut temp_byte = [0u8; 1];
if length > MAX_SIZE {
anyhow::bail!("Received data exceeds maximum size");
loop {
stream_read.read_exact(&mut temp_byte).await?;
length_buf.push(temp_byte[0]);
// Check if this is the last byte (MSB = 0)
if temp_byte[0] & 0x80 == 0 {
break;
}
// Protect against malicious infinite varints
if length_buf.len() > 10 {
anyhow::bail!("Invalid varint encoding");
}
}
// Decode the varint
let (length, _) = decode::usize(&length_buf)
.map_err(|e| anyhow::anyhow!("Failed to decode varint: {}", e))?;
// Read payload
let mut buffer = vec![0u8; length];
stream_read.read_exact(&mut buffer).await?;
Ok(buffer)
}
}

34
packages/server/src/proto.rs
View File

@@ -1 +1,35 @@
pub mod proto;
pub struct CreateMessageOptions {
pub sequence_id: Option<String>,
pub latency: Option<proto::ProtoLatencyTracker>,
}
pub fn create_message(
payload: proto::proto_message::Payload,
payload_type: impl Into<String>,
options: Option<CreateMessageOptions>,
) -> proto::ProtoMessage {
let opts = options.unwrap_or(CreateMessageOptions {
sequence_id: None,
latency: None,
});
let latency = opts.latency.or_else(|| {
opts.sequence_id.map(|seq_id| proto::ProtoLatencyTracker {
sequence_id: seq_id,
timestamps: vec![proto::ProtoTimestampEntry {
stage: "created".to_string(),
time: Some(prost_types::Timestamp::from(std::time::SystemTime::now())),
}],
})
});
proto::ProtoMessage {
message_base: Some(proto::ProtoMessageBase {
payload_type: payload_type.into(),
latency,
}),
payload: Some(payload),
}
}

202
packages/server/src/proto/gen.rs
View File

@@ -1,202 +0,0 @@
// @generated
// This file is @generated by prost-build.
/// EntityState represents the state of an entity in the mesh (e.g., a room).
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EntityState {
/// Type of entity (e.g., "room")
#[prost(string, tag="1")]
pub entity_type: ::prost::alloc::string::String,
/// Unique identifier (e.g., room name)
#[prost(string, tag="2")]
pub entity_id: ::prost::alloc::string::String,
/// Whether the entity is active
#[prost(bool, tag="3")]
pub active: bool,
/// Relay ID that owns this entity
#[prost(string, tag="4")]
pub owner_relay_id: ::prost::alloc::string::String,
}
/// MeshMessage is the top-level message for all relay-to-relay communication.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MeshMessage {
#[prost(oneof="mesh_message::Type", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13")]
pub r#type: ::core::option::Option<mesh_message::Type>,
}
/// Nested message and enum types in `MeshMessage`.
pub mod mesh_message {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Type {
/// Level 0
#[prost(message, tag="1")]
StateUpdate(super::StateUpdate),
#[prost(message, tag="2")]
Ack(super::Ack),
#[prost(message, tag="3")]
RetransmissionRequest(super::RetransmissionRequest),
#[prost(message, tag="4")]
Retransmission(super::Retransmission),
#[prost(message, tag="5")]
Heartbeat(super::Heartbeat),
#[prost(message, tag="6")]
SuspectRelay(super::SuspectRelay),
#[prost(message, tag="7")]
Disconnect(super::Disconnect),
/// Level 1
#[prost(message, tag="8")]
ForwardSdp(super::ForwardSdp),
#[prost(message, tag="9")]
ForwardIce(super::ForwardIce),
#[prost(message, tag="10")]
ForwardIngest(super::ForwardIngest),
#[prost(message, tag="11")]
StreamRequest(super::StreamRequest),
/// Level 2
#[prost(message, tag="12")]
Handshake(super::Handshake),
#[prost(message, tag="13")]
HandshakeResponse(super::HandshakeResponse),
}
}
/// Handshake to inititiate new connection to mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Handshake {
/// UUID of the relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// base64 encoded Diffie-Hellman public key
#[prost(string, tag="2")]
pub dh_public_key: ::prost::alloc::string::String,
}
/// HandshakeResponse to respond to a mesh joiner.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct HandshakeResponse {
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub dh_public_key: ::prost::alloc::string::String,
/// relay id to signature
#[prost(map="string, string", tag="3")]
pub approvals: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
/// Forwarded SDP from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardSdp {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub participant_id: ::prost::alloc::string::String,
#[prost(string, tag="3")]
pub sdp: ::prost::alloc::string::String,
/// "offer" or "answer"
#[prost(string, tag="4")]
pub r#type: ::prost::alloc::string::String,
}
/// Forwarded ICE candidate from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardIce {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub participant_id: ::prost::alloc::string::String,
#[prost(string, tag="3")]
pub candidate: ::prost::alloc::string::String,
}
/// Forwarded ingest room from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardIngest {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
/// Stream request from mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StreamRequest {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
/// StateUpdate propagates entity state changes across the mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StateUpdate {
/// Unique sequence number for this update
#[prost(uint64, tag="1")]
pub sequence_number: u64,
/// Key: entity_id (e.g., room name), Value: EntityState
#[prost(map="string, message", tag="2")]
pub entities: ::std::collections::HashMap<::prost::alloc::string::String, EntityState>,
}
/// Ack acknowledges receipt of a StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Ack {
/// UUID of the acknowledging relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Sequence number being acknowledged
#[prost(uint64, tag="2")]
pub sequence_number: u64,
}
/// RetransmissionRequest requests a missed StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct RetransmissionRequest {
/// UUID of the requesting relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Sequence number of the missed update
#[prost(uint64, tag="2")]
pub sequence_number: u64,
}
/// Retransmission resends a StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Retransmission {
/// UUID of the sending relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// The retransmitted update
#[prost(message, optional, tag="2")]
pub state_update: ::core::option::Option<StateUpdate>,
}
/// Heartbeat signals relay liveness.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Heartbeat {
/// UUID of the sending relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Time of the heartbeat
#[prost(message, optional, tag="2")]
pub timestamp: ::core::option::Option<::prost_types::Timestamp>,
}
/// SuspectRelay marks a relay as potentially unresponsive.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SuspectRelay {
/// UUID of the suspected relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Reason for suspicion (e.g., "no heartbeat")
#[prost(string, tag="2")]
pub reason: ::prost::alloc::string::String,
}
/// Disconnect signals to remove a relay from the mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Disconnect {
/// UUID of the relay to disconnect
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Reason for disconnection (e.g., "unresponsive")
#[prost(string, tag="2")]
pub reason: ::prost::alloc::string::String,
}
// @@protoc_insertion_point(module)

288
packages/server/src/proto/proto.rs
View File

@@ -20,80 +20,59 @@ pub struct ProtoLatencyTracker {
/// MouseMove message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseMove {
/// Fixed value "MouseMove"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseMoveAbs message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseMoveAbs {
/// Fixed value "MouseMoveAbs"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseWheel message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseWheel {
/// Fixed value "MouseWheel"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseKeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseKeyDown {
/// Fixed value "MouseKeyDown"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
/// MouseKeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseKeyUp {
/// Fixed value "MouseKeyUp"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
// Keyboard messages
/// KeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoKeyDown {
/// Fixed value "KeyDown"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
/// KeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoKeyUp {
/// Fixed value "KeyUp"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
// Controller messages
@@ -102,159 +81,167 @@ pub struct ProtoKeyUp {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerAttach {
/// Fixed value "ControllerAttach"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// One of the following enums: "ps", "xbox" or "switch"
#[prost(string, tag="2")]
#[prost(string, tag="1")]
pub id: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub slot: i32,
/// Session ID of the client attaching the controller
#[prost(string, tag="3")]
pub session_id: ::prost::alloc::string::String,
}
/// ControllerDetach message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerDetach {
/// Fixed value "ControllerDetach"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
}
/// ControllerButton message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerButton {
/// Fixed value "ControllerButtons"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
/// Button code (linux input event code)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub button: i32,
/// true if pressed, false if released
#[prost(bool, tag="4")]
#[prost(bool, tag="3")]
pub pressed: bool,
}
/// ControllerTriggers message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerTrigger {
/// Fixed value "ControllerTriggers"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
/// Trigger number (0 for left, 1 for right)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub trigger: i32,
/// trigger value (-32768 to 32767)
#[prost(int32, tag="4")]
#[prost(int32, tag="3")]
pub value: i32,
}
/// ControllerSticks message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerStick {
/// Fixed value "ControllerStick"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
/// Stick number (0 for left, 1 for right)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub stick: i32,
/// X axis value (-32768 to 32767)
#[prost(int32, tag="4")]
#[prost(int32, tag="3")]
pub x: i32,
/// Y axis value (-32768 to 32767)
#[prost(int32, tag="5")]
#[prost(int32, tag="4")]
pub y: i32,
}
/// ControllerAxis message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerAxis {
/// Fixed value "ControllerAxis"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
/// Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub axis: i32,
/// axis value (-1 to 1)
#[prost(int32, tag="4")]
#[prost(int32, tag="3")]
pub value: i32,
}
/// ControllerRumble message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoControllerRumble {
/// Fixed value "ControllerRumble"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub slot: i32,
/// Low frequency rumble (0-65535)
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub low_frequency: i32,
/// High frequency rumble (0-65535)
#[prost(int32, tag="4")]
#[prost(int32, tag="3")]
pub high_frequency: i32,
/// Duration in milliseconds
#[prost(int32, tag="5")]
#[prost(int32, tag="4")]
pub duration: i32,
}
/// Union of all Input types
// WebRTC + signaling
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoInput {
#[prost(oneof="proto_input::InputType", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14")]
pub input_type: ::core::option::Option<proto_input::InputType>,
pub struct RtcIceCandidateInit {
#[prost(string, tag="1")]
pub candidate: ::prost::alloc::string::String,
#[prost(uint32, optional, tag="2")]
pub sdp_m_line_index: ::core::option::Option<u32>,
#[prost(string, optional, tag="3")]
pub sdp_mid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag="4")]
pub username_fragment: ::core::option::Option<::prost::alloc::string::String>,
}
/// Nested message and enum types in `ProtoInput`.
pub mod proto_input {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum InputType {
#[prost(message, tag="1")]
MouseMove(super::ProtoMouseMove),
#[prost(message, tag="2")]
MouseMoveAbs(super::ProtoMouseMoveAbs),
#[prost(message, tag="3")]
MouseWheel(super::ProtoMouseWheel),
#[prost(message, tag="4")]
MouseKeyDown(super::ProtoMouseKeyDown),
#[prost(message, tag="5")]
MouseKeyUp(super::ProtoMouseKeyUp),
#[prost(message, tag="6")]
KeyDown(super::ProtoKeyDown),
#[prost(message, tag="7")]
KeyUp(super::ProtoKeyUp),
#[prost(message, tag="8")]
ControllerAttach(super::ProtoControllerAttach),
#[prost(message, tag="9")]
ControllerDetach(super::ProtoControllerDetach),
#[prost(message, tag="10")]
ControllerButton(super::ProtoControllerButton),
#[prost(message, tag="11")]
ControllerTrigger(super::ProtoControllerTrigger),
#[prost(message, tag="12")]
ControllerStick(super::ProtoControllerStick),
#[prost(message, tag="13")]
ControllerAxis(super::ProtoControllerAxis),
#[prost(message, tag="14")]
ControllerRumble(super::ProtoControllerRumble),
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct RtcSessionDescriptionInit {
#[prost(string, tag="1")]
pub sdp: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub r#type: ::prost::alloc::string::String,
}
/// ProtoICE message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoIce {
#[prost(message, optional, tag="1")]
pub candidate: ::core::option::Option<RtcIceCandidateInit>,
}
/// ProtoSDP message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoSdp {
#[prost(message, optional, tag="1")]
pub sdp: ::core::option::Option<RtcSessionDescriptionInit>,
}
/// ProtoRaw message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoRaw {
#[prost(string, tag="1")]
pub data: ::prost::alloc::string::String,
}
/// ProtoClientRequestRoomStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoClientRequestRoomStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
}
/// ProtoClientDisconnected message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoClientDisconnected {
#[prost(string, tag="1")]
pub session_id: ::prost::alloc::string::String,
#[prost(int32, repeated, tag="2")]
pub controller_slots: ::prost::alloc::vec::Vec<i32>,
}
/// ProtoServerPushStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoServerPushStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -266,10 +253,59 @@ pub struct ProtoMessageBase {
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoMessageInput {
pub struct ProtoMessage {
#[prost(message, optional, tag="1")]
pub message_base: ::core::option::Option<ProtoMessageBase>,
#[prost(message, optional, tag="2")]
pub data: ::core::option::Option<ProtoInput>,
#[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25")]
pub payload: ::core::option::Option<proto_message::Payload>,
}
/// Nested message and enum types in `ProtoMessage`.
pub mod proto_message {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Payload {
/// Input types
#[prost(message, tag="2")]
MouseMove(super::ProtoMouseMove),
#[prost(message, tag="3")]
MouseMoveAbs(super::ProtoMouseMoveAbs),
#[prost(message, tag="4")]
MouseWheel(super::ProtoMouseWheel),
#[prost(message, tag="5")]
MouseKeyDown(super::ProtoMouseKeyDown),
#[prost(message, tag="6")]
MouseKeyUp(super::ProtoMouseKeyUp),
#[prost(message, tag="7")]
KeyDown(super::ProtoKeyDown),
#[prost(message, tag="8")]
KeyUp(super::ProtoKeyUp),
#[prost(message, tag="9")]
ControllerAttach(super::ProtoControllerAttach),
#[prost(message, tag="10")]
ControllerDetach(super::ProtoControllerDetach),
#[prost(message, tag="11")]
ControllerButton(super::ProtoControllerButton),
#[prost(message, tag="12")]
ControllerTrigger(super::ProtoControllerTrigger),
#[prost(message, tag="13")]
ControllerStick(super::ProtoControllerStick),
#[prost(message, tag="14")]
ControllerAxis(super::ProtoControllerAxis),
#[prost(message, tag="15")]
ControllerRumble(super::ProtoControllerRumble),
/// Signaling types
#[prost(message, tag="20")]
Ice(super::ProtoIce),
#[prost(message, tag="21")]
Sdp(super::ProtoSdp),
#[prost(message, tag="22")]
Raw(super::ProtoRaw),
#[prost(message, tag="23")]
ClientRequestRoomStream(super::ProtoClientRequestRoomStream),
#[prost(message, tag="24")]
ClientDisconnected(super::ProtoClientDisconnected),
#[prost(message, tag="25")]
ServerPushStream(super::ProtoServerPushStream),
}
}
// @@protoc_insertion_point(module)

28
protobufs/messages.proto
View File

@@ -12,7 +12,31 @@ message ProtoMessageBase {
ProtoLatencyTracker latency = 2;
}
message ProtoMessageInput {
message ProtoMessage {
ProtoMessageBase message_base = 1;
ProtoInput data = 2;
oneof payload {
// Input types
ProtoMouseMove mouse_move = 2;
ProtoMouseMoveAbs mouse_move_abs = 3;
ProtoMouseWheel mouse_wheel = 4;
ProtoMouseKeyDown mouse_key_down = 5;
ProtoMouseKeyUp mouse_key_up = 6;
ProtoKeyDown key_down = 7;
ProtoKeyUp key_up = 8;
ProtoControllerAttach controller_attach = 9;
ProtoControllerDetach controller_detach = 10;
ProtoControllerButton controller_button = 11;
ProtoControllerTrigger controller_trigger = 12;
ProtoControllerStick controller_stick = 13;
ProtoControllerAxis controller_axis = 14;
ProtoControllerRumble controller_rumble = 15;
// Signaling types
ProtoICE ice = 20;
ProtoSDP sdp = 21;
ProtoRaw raw = 22;
ProtoClientRequestRoomStream client_request_room_stream = 23;
ProtoClientDisconnected client_disconnected = 24;
ProtoServerPushStream server_push_stream = 25;
}
}

137
protobufs/types.proto
View File

@@ -8,124 +8,137 @@ package proto;
// MouseMove message
message ProtoMouseMove {
string type = 1; // Fixed value "MouseMove"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseMoveAbs message
message ProtoMouseMoveAbs {
string type = 1; // Fixed value "MouseMoveAbs"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseWheel message
message ProtoMouseWheel {
string type = 1; // Fixed value "MouseWheel"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseKeyDown message
message ProtoMouseKeyDown {
string type = 1; // Fixed value "MouseKeyDown"
int32 key = 2;
int32 key = 1;
}
// MouseKeyUp message
message ProtoMouseKeyUp {
string type = 1; // Fixed value "MouseKeyUp"
int32 key = 2;
int32 key = 1;
}
/* Keyboard messages */
// KeyDown message
message ProtoKeyDown {
string type = 1; // Fixed value "KeyDown"
int32 key = 2;
int32 key = 1;
}
// KeyUp message
message ProtoKeyUp {
string type = 1; // Fixed value "KeyUp"
int32 key = 2;
int32 key = 1;
}
/* Controller messages */
// ControllerAttach message
message ProtoControllerAttach {
string type = 1; // Fixed value "ControllerAttach"
string id = 2; // One of the following enums: "ps", "xbox" or "switch"
int32 slot = 3; // Slot number (0-3)
string id = 1; // One of the following enums: "ps", "xbox" or "switch"
int32 slot = 2; // Slot number (0-3)
string session_id = 3; // Session ID of the client attaching the controller
}
// ControllerDetach message
message ProtoControllerDetach {
string type = 1; // Fixed value "ControllerDetach"
int32 slot = 2; // Slot number (0-3)
int32 slot = 1; // Slot number (0-3)
}
// ControllerButton message
message ProtoControllerButton {
string type = 1; // Fixed value "ControllerButtons"
int32 slot = 2; // Slot number (0-3)
int32 button = 3; // Button code (linux input event code)
bool pressed = 4; // true if pressed, false if released
int32 slot = 1; // Slot number (0-3)
int32 button = 2; // Button code (linux input event code)
bool pressed = 3; // true if pressed, false if released
}
// ControllerTriggers message
message ProtoControllerTrigger {
string type = 1; // Fixed value "ControllerTriggers"
int32 slot = 2; // Slot number (0-3)
int32 trigger = 3; // Trigger number (0 for left, 1 for right)
int32 value = 4; // trigger value (-32768 to 32767)
int32 slot = 1; // Slot number (0-3)
int32 trigger = 2; // Trigger number (0 for left, 1 for right)
int32 value = 3; // trigger value (-32768 to 32767)
}
// ControllerSticks message
message ProtoControllerStick {
string type = 1; // Fixed value "ControllerStick"
int32 slot = 2; // Slot number (0-3)
int32 stick = 3; // Stick number (0 for left, 1 for right)
int32 x = 4; // X axis value (-32768 to 32767)
int32 y = 5; // Y axis value (-32768 to 32767)
int32 slot = 1; // Slot number (0-3)
int32 stick = 2; // Stick number (0 for left, 1 for right)
int32 x = 3; // X axis value (-32768 to 32767)
int32 y = 4; // Y axis value (-32768 to 32767)
}
// ControllerAxis message
message ProtoControllerAxis {
string type = 1; // Fixed value "ControllerAxis"
int32 slot = 2; // Slot number (0-3)
int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
int32 value = 4; // axis value (-1 to 1)
int32 slot = 1; // Slot number (0-3)
int32 axis = 2; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
int32 value = 3; // axis value (-1 to 1)
}
// ControllerRumble message
message ProtoControllerRumble {
string type = 1; // Fixed value "ControllerRumble"
int32 slot = 2; // Slot number (0-3)
int32 low_frequency = 3; // Low frequency rumble (0-65535)
int32 high_frequency = 4; // High frequency rumble (0-65535)
int32 duration = 5; // Duration in milliseconds
int32 slot = 1; // Slot number (0-3)
int32 low_frequency = 2; // Low frequency rumble (0-65535)
int32 high_frequency = 3; // High frequency rumble (0-65535)
int32 duration = 4; // Duration in milliseconds
}
// Union of all Input types
message ProtoInput {
oneof input_type {
ProtoMouseMove mouse_move = 1;
ProtoMouseMoveAbs mouse_move_abs = 2;
ProtoMouseWheel mouse_wheel = 3;
ProtoMouseKeyDown mouse_key_down = 4;
ProtoMouseKeyUp mouse_key_up = 5;
ProtoKeyDown key_down = 6;
ProtoKeyUp key_up = 7;
ProtoControllerAttach controller_attach = 8;
ProtoControllerDetach controller_detach = 9;
ProtoControllerButton controller_button = 10;
ProtoControllerTrigger controller_trigger = 11;
ProtoControllerStick controller_stick = 12;
ProtoControllerAxis controller_axis = 13;
ProtoControllerRumble controller_rumble = 14;
}
/* WebRTC + signaling */
message RTCIceCandidateInit {
string candidate = 1;
optional uint32 sdpMLineIndex = 2;
optional string sdpMid = 3;
optional string usernameFragment = 4;
}
message RTCSessionDescriptionInit {
string sdp = 1;
string type = 2;
}
// ProtoICE message
message ProtoICE {
RTCIceCandidateInit candidate = 1;
}
// ProtoSDP message
message ProtoSDP {
RTCSessionDescriptionInit sdp = 1;
}
// ProtoRaw message
message ProtoRaw {
string data = 1;
}
// ProtoClientRequestRoomStream message
message ProtoClientRequestRoomStream {
string room_name = 1;
string session_id = 2;
}
// ProtoClientDisconnected message
message ProtoClientDisconnected {
string session_id = 1;
repeated int32 controller_slots = 2;
}
// ProtoServerPushStream message
message ProtoServerPushStream {
string room_name = 1;
}