github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: github-mirrors:dev
Branches Tags
github-mirrors:dev github-mirrors:capacitor-testings github-mirrors:feat/cf github-mirrors:feat/home github-mirrors:feat/images github-mirrors:feat/blog github-mirrors:feat/pricing github-mirrors:feat/usage-billing github-mirrors:feat/play github-mirrors:feat/cloud-init-ubuntu
github-mirrors:v0.2.0 github-mirrors:v0.1.0
...
compare: github-mirrors:b4a1f6d31f46b025fe128a2a844a8311f0308129
Branches Tags
github-mirrors:dev github-mirrors:capacitor-testings github-mirrors:feat/cf github-mirrors:feat/home github-mirrors:feat/images github-mirrors:feat/blog github-mirrors:feat/pricing github-mirrors:feat/usage-billing github-mirrors:feat/play github-mirrors:feat/cloud-init-ubuntu
github-mirrors:v0.2.0 github-mirrors:v0.1.0

3 Commits
dev ... b4a1f6d31f

Author SHA1 Message Date
Kristian Ollikainen
b4a1f6d31f Merge a54cf759fa into 32341574dc 2025-10-25 03:57:46 +03:00
DatCaptainHorse
a54cf759fa Fixed multi-controllers, optimize and improve code in relay and nestri-server 2025-10-25 03:57:26 +03:00
DatCaptainHorse
67f9a7d0a0 Restructure protobufs and use them everywhere 2025-10-21 18:41:45 +03:00
46 changed files with 3843 additions and 3269 deletions
Expand all files Collapse all files

20
containerfiles/runner-builder.Containerfile
View File

@@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \
pacman -Sy --noconfirm lib32-gcc-libs
# Clone repository
RUN git clone --depth 1 --rev "9e8bfd0217eeab011c5afc368d3ea67a4c239e81" https://github.com/DatCaptainHorse/vimputti.git
RUN git clone --depth 1 --rev "f2f21561ddcb814d74455311969d3e8934b052c6" https://github.com/DatCaptainHorse/vimputti.git
#--------------------------------------------------------------------
FROM vimputti-manager-deps AS vimputti-manager-planner
@@ -129,23 +129,8 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \
RUN --mount=type=cache,target=${CARGO_HOME}/registry \
cargo install cargo-c
# Grab cudart from NVIDIA..
RUN wget https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-13.0.96-archive.tar.xz -O cuda_cudart.tar.xz && \
mkdir cuda_cudart && tar -xf cuda_cudart.tar.xz -C cuda_cudart --strip-components=1 && \
cp cuda_cudart/lib/libcudart.so cuda_cudart/lib/libcudart.so.* /usr/lib/ && \
rm -r cuda_cudart && \
rm cuda_cudart.tar.xz
# Grab cuda lib from NVIDIA (it's in driver package of all things..)
RUN wget https://developer.download.nvidia.com/compute/cuda/redist/nvidia_driver/linux-x86_64/nvidia_driver-linux-x86_64-580.95.05-archive.tar.xz -O nvidia_driver.tar.xz && \
mkdir nvidia_driver && tar -xf nvidia_driver.tar.xz -C nvidia_driver --strip-components=1 && \
cp nvidia_driver/lib/libcuda.so.* /usr/lib/libcuda.so && \
ln -s /usr/lib/libcuda.so /usr/lib/libcuda.so.1 && \
rm -r nvidia_driver && \
rm nvidia_driver.tar.xz
# Clone repository
RUN git clone --depth 1 --rev "afa853fa03e8403c83bbb3bc0cf39147ad46c266" https://github.com/games-on-whales/gst-wayland-display.git
RUN git clone --depth 1 --rev "a4abcfe2cffe2d33b564d1308b58504a5e3012b1" https://github.com/games-on-whales/gst-wayland-display.git
#--------------------------------------------------------------------
FROM gst-wayland-deps AS gst-wayland-planner
@@ -214,5 +199,4 @@ COPY --from=gst-wayland-cached-builder /artifacts/include/ /artifacts/include/
COPY --from=vimputti-manager-cached-builder /artifacts/vimputti-manager /artifacts/bin/
COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_64.so /artifacts/lib64/libvimputti_shim.so
COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_32.so /artifacts/lib32/libvimputti_shim.so
COPY --from=gst-wayland-deps /usr/lib/libcuda.so /usr/lib/libcuda.so.* /artifacts/lib/
COPY --from=bubblewrap-builder /artifacts/bin/bwrap /artifacts/bin/

30
packages/input/package.json
View File

@@ -7,24 +7,22 @@
".": "./src/index.ts"
},
"devDependencies": {
"@bufbuild/buf": "^1.57.2",
"@bufbuild/protoc-gen-es": "^2.9.0"
"@bufbuild/buf": "^1.59.0",
"@bufbuild/protoc-gen-es": "^2.10.0"
},
"dependencies": {
"@bufbuild/protobuf": "^2.9.0",
"@chainsafe/libp2p-noise": "^16.1.4",
"@bufbuild/protobuf": "^2.10.0",
"@chainsafe/libp2p-noise": "^17.0.0",
"@chainsafe/libp2p-quic": "^1.1.3",
"@chainsafe/libp2p-yamux": "^7.0.4",
"@libp2p/identify": "^3.0.39",
"@libp2p/interface": "^2.11.0",
"@libp2p/ping": "^2.0.37",
"@libp2p/websockets": "^9.2.19",
"@libp2p/webtransport": "^5.0.51",
"@multiformats/multiaddr": "^12.5.1",
"it-length-prefixed": "^10.0.1",
"it-pipe": "^3.0.1",
"libp2p": "^2.10.0",
"uint8arraylist": "^2.4.8",
"uint8arrays": "^5.1.0"
"@chainsafe/libp2p-yamux": "^8.0.1",
"@libp2p/identify": "^4.0.5",
"@libp2p/interface": "^3.0.2",
"@libp2p/ping": "^3.0.5",
"@libp2p/websockets": "^10.0.6",
"@libp2p/webtransport": "^6.0.7",
"@libp2p/utils": "^7.0.5",
"@multiformats/multiaddr": "^13.0.1",
"libp2p": "^3.0.6",
"uint8arraylist": "^2.4.8"
}
}

454
packages/input/src/controller.ts
View File

@@ -1,12 +1,6 @@
import { controllerButtonToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import {
ProtoMessageBase,
ProtoMessageInput,
ProtoMessageInputSchema,
} from "./proto/messages_pb";
import {
ProtoInputSchema,
ProtoControllerAttachSchema,
ProtoControllerDetachSchema,
ProtoControllerButtonSchema,
@@ -16,6 +10,8 @@ import {
ProtoControllerRumble,
} from "./proto/types_pb";
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -36,7 +32,6 @@ interface GamepadState {
export class Controller {
protected wrtc: WebRTCStream;
protected slot: number;
protected connected: boolean = false;
protected gamepad: Gamepad | null = null;
protected lastState: GamepadState = {
@@ -54,17 +49,21 @@ export class Controller {
protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range)
private updateInterval = 10.0; // 100 updates per second
private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null;
private isIdle: boolean = true;
private lastInputTime: number = Date.now();
private idleUpdateInterval: number = 150.0; // ~6-7 updates per second for keep-alive packets
private inputDetected: boolean = false;
private lastFullStateSend: number = Date.now();
private fullStateSendInterval: number = 500.0; // send full state every 0.5 seconds (helps packet loss)
private forceFullStateSend: boolean = false;
private _dcHandler: ((data: ArrayBuffer) => void) | null = null;
constructor({ webrtc, e }: Props) {
this.wrtc = webrtc;
this.slot = e.gamepad.index;
this.updateInterval = 1000 / webrtc.currentFrameRate;
// Gamepad connected
this.gamepad = e.gamepad;
// Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc")
const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/);
const vendorId = vendorMatch ? vendorMatch[1].toLowerCase() : "unknown";
@@ -72,34 +71,48 @@ export class Controller {
const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/);
const productId = productMatch ? productMatch[1].toLowerCase() : "unknown";
const attachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAttach",
value: create(ProtoControllerAttachSchema, {
type: "ControllerAttach",
id: this.vendor_id_to_controller(vendorId, productId),
slot: this.slot,
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: attachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
// Listen to datachannel events from server
this._dcHandler = (data: ArrayBuffer) => {
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data);
// Listen to feedback rumble events from server
this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer);
this.wrtc.addDataChannelCallback(this._dcRumbleHandler);
if (messageWrapper.payload.case === "controllerRumble") {
this.rumbleCallback(messageWrapper.payload.value);
} else if (messageWrapper.payload.case === "controllerAttach") {
if (this.gamepad) return; // already attached
const attachMsg = messageWrapper.payload.value;
// Gamepad connected succesfully
this.gamepad = e.gamepad;
console.log(
`Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`,
);
}
} catch (err) {
console.error("Error decoding datachannel message:", err);
}
};
this.wrtc.addDataChannelCallback(this._dcHandler);
const attachMsg = createMessage(
create(ProtoControllerAttachSchema, {
id: this.vendor_id_to_controller(vendorId, productId),
sessionSlot: e.gamepad.index,
sessionId: this.wrtc.getSessionID(),
}),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg));
this.run();
}
public getSlot(): number {
return this.gamepad.index;
}
// Maps vendor id and product id to supported controller type
// Currently supported: Sony (ps4, ps5), Microsoft (xbox360, xboxone), Nintendo (switchpro)
// Default fallback to xbox360
@@ -150,18 +163,26 @@ export class Controller {
}
private pollGamepad() {
// Get updated gamepad state
const gamepads = navigator.getGamepads();
if (this.slot < gamepads.length) {
const gamepad = gamepads[this.slot];
if (gamepad) {
// Periodically force send full state to clear stuck inputs
if (Date.now() - this.lastFullStateSend > this.fullStateSendInterval) {
this.forceFullStateSend = true;
this.lastFullStateSend = Date.now();
}
if (this.gamepad) {
if (gamepads[this.gamepad.index]) {
this.gamepad = gamepads[this.gamepad!.index];
/* Button handling */
gamepad.buttons.forEach((button, index) => {
this.gamepad.buttons.forEach((button, index) => {
// Ignore d-pad buttons (12-15) as we handle those as axis
if (index >= 12 && index <= 15) return;
// ignore trigger buttons (6-7) as we handle those as axis
if (index === 6 || index === 7) return;
// If state differs, send
if (button.pressed !== this.lastState.buttonState.get(index)) {
if (button.pressed !== this.lastState.buttonState.get(index) || this.forceFullStateSend) {
const linuxCode = this.controllerButtonToVirtualKeyCode(index);
if (linuxCode === undefined) {
// Skip unmapped button index
@@ -169,29 +190,17 @@ export class Controller {
return;
}
const buttonProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerButton",
value: create(ProtoControllerButtonSchema, {
type: "ControllerButton",
slot: this.slot,
const buttonMessage = createMessage(
create(ProtoControllerButtonSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
button: linuxCode,
pressed: button.pressed,
}),
},
});
const buttonMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: buttonProto,
};
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, buttonMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage));
this.inputDetected = true;
// Store button state
this.lastState.buttonState.set(index, button.pressed);
}
@@ -200,128 +209,108 @@ export class Controller {
/* Trigger handling */
// map trigger value from 0.0 to 1.0 to -32768 to 32767
const leftTrigger = Math.round(
this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767),
this.remapFromTo(
this.gamepad.buttons[6]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
// If state differs, send
if (leftTrigger !== this.lastState.leftTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
if (leftTrigger !== this.lastState.leftTrigger || this.forceFullStateSend) {
const triggerMessage = createMessage(
create(ProtoControllerTriggerSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
trigger: 0, // 0 = left, 1 = right
value: leftTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.leftTrigger = leftTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
this.inputDetected = true;
this.lastState.leftTrigger = leftTrigger;
}
const rightTrigger = Math.round(
this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767),
this.remapFromTo(
this.gamepad.buttons[7]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
// If state differs, send
if (rightTrigger !== this.lastState.rightTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
if (rightTrigger !== this.lastState.rightTrigger || this.forceFullStateSend) {
const triggerMessage = createMessage(
create(ProtoControllerTriggerSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
trigger: 1, // 0 = left, 1 = right
value: rightTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.rightTrigger = rightTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
this.inputDetected = true;
this.lastState.rightTrigger = rightTrigger;
}
/* DPad handling */
// We send dpad buttons as axis values -1 to 1 for left/up, right/down
const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0;
const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0;
const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0;
const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0;
const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0;
if (dpadX !== this.lastState.dpadX) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
if (dpadX !== this.lastState.dpadX || this.forceFullStateSend) {
const dpadMessage = createMessage(
create(ProtoControllerAxisSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
axis: 0, // 0 = dpadX, 1 = dpadY
value: dpadX,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
this.inputDetected = true;
this.lastState.dpadX = dpadX;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
}
const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0;
const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0;
const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0;
const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0;
const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0;
if (dpadY !== this.lastState.dpadY) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
if (dpadY !== this.lastState.dpadY || this.forceFullStateSend) {
const dpadMessage = createMessage(
create(ProtoControllerAxisSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
axis: 1, // 0 = dpadX, 1 = dpadY
value: dpadY,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
this.inputDetected = true;
this.lastState.dpadY = dpadY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
}
/* Stick handling */
// stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767
const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767);
const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767);
const leftX = this.remapFromTo(
this.gamepad.axes[0] ?? 0,
-1,
1,
-32768,
32767,
);
const leftY = this.remapFromTo(
this.gamepad.axes[1] ?? 0,
-1,
1,
-32768,
32767,
);
// Apply deadzone
const sendLeftX =
Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
@@ -331,37 +320,38 @@ export class Controller {
// if moves inside deadzone, zero it if not inside deadzone last time
if (
sendLeftX !== this.lastState.leftX ||
sendLeftY !== this.lastState.leftY
sendLeftY !== this.lastState.leftY || this.forceFullStateSend
) {
// console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY);
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
const stickMessage = createMessage(
create(ProtoControllerStickSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
stick: 0, // 0 = left, 1 = right
x: sendLeftX,
y: sendLeftY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
this.inputDetected = true;
this.lastState.leftX = sendLeftX;
this.lastState.leftY = sendLeftY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767);
const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767);
const rightX = this.remapFromTo(
this.gamepad.axes[2] ?? 0,
-1,
1,
-32768,
32767,
);
const rightY = this.remapFromTo(
this.gamepad.axes[3] ?? 0,
-1,
1,
-32768,
32767,
);
// Apply deadzone
const sendRightX =
Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
@@ -369,49 +359,64 @@ export class Controller {
Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0;
if (
sendRightX !== this.lastState.rightX ||
sendRightY !== this.lastState.rightY
sendRightY !== this.lastState.rightY || this.forceFullStateSend
) {
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
const stickMessage = createMessage(
create(ProtoControllerStickSchema, {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
stick: 1, // 0 = left, 1 = right
x: sendRightX,
y: sendRightY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
this.inputDetected = true;
this.lastState.rightX = sendRightX;
this.lastState.rightY = sendRightY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
}
}
this.forceFullStateSend = false;
}
private loopInterval: any = null;
public run() {
if (this.connected)
this.stop();
if (this.connected) this.stop();
this.connected = true;
// Poll gamepads in setInterval loop
this.isIdle = true;
this.lastInputTime = Date.now();
this.loopInterval = setInterval(() => {
if (this.connected) {
this.inputDetected = false; // Reset before poll
this.pollGamepad();
// Switch polling rate based on input
if (this.inputDetected) {
this.lastInputTime = Date.now();
if (this.isIdle) {
this.isIdle = false;
clearInterval(this.loopInterval);
this.loopInterval = setInterval(() => {
if (this.connected) this.pollGamepad();
}, this.updateInterval);
}
} else if (!this.isIdle && Date.now() - this.lastInputTime > 200) {
// Switch to idle polling after 200ms of no input
this.isIdle = true;
clearInterval(this.loopInterval);
this.loopInterval = setInterval(() => {
if (this.connected) this.pollGamepad();
}, this.idleUpdateInterval);
}
}
}, this.isIdle ? this.idleUpdateInterval : this.updateInterval);
}
public stop() {
if (this.loopInterval) {
@@ -421,69 +426,44 @@ export class Controller {
this.connected = false;
}
public getSlot() {
return this.slot;
}
public dispose() {
this.stop();
// Remove callback
if (this._dcRumbleHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcRumbleHandler);
this._dcRumbleHandler = null;
if (this._dcHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcHandler);
this._dcHandler = null;
}
// Gamepad disconnected
const detachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerDetach",
value: create(ProtoControllerDetachSchema, {
type: "ControllerDetach",
slot: this.slot,
const detachMsg = createMessage(
create(ProtoControllerDetachSchema, {
sessionSlot: this.gamepad.index,
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: detachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg));
}
private controllerButtonToVirtualKeyCode(code: number) {
return controllerButtonToLinuxEventCode[code] || undefined;
}
private rumbleCallback(data: ArrayBuffer) {
private rumbleCallback(rumbleMsg: ProtoControllerRumble) {
// If not connected, ignore
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data);
// Check if it contains controller rumble data
if (messageWrapper.data?.inputType?.case === "controllerRumble") {
const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble;
// Check if aimed at this controller slot
if (rumbleMsg.slot !== this.slot) return;
if (rumbleMsg.sessionId !== this.wrtc.getSessionID() &&
rumbleMsg.sessionSlot !== this.gamepad.index)
return;
// Trigger actual rumble
// Need to remap from 0-65535 to 0.0-1.0 ranges
const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
const rumbleLowFreq = this.remapFromTo(
clampedLowFreq,
const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0);
const clampedHighFreq = Math.max(
0,
65535,
0.0,
1.0,
Math.min(65535, rumbleMsg.highFrequency),
);
const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency));
const rumbleHighFreq = this.remapFromTo(
clampedHighFreq,
0,
@@ -494,16 +474,14 @@ export class Controller {
// Cap to valid range (max 5000)
const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
if (this.gamepad.vibrationActuator) {
this.gamepad.vibrationActuator.playEffect("dual-rumble", {
this.gamepad.vibrationActuator
.playEffect("dual-rumble", {
startDelay: 0,
duration: rumbleDuration,
weakMagnitude: rumbleLowFreq,
strongMagnitude: rumbleHighFreq,
}).catch(console.error);
}
}
} catch (error) {
console.error("Failed to decode rumble message:", error);
})
.catch(console.error);
}
}
}

87
packages/input/src/keyboard.ts
View File

@@ -1,16 +1,9 @@
import {keyCodeToLinuxEventCode} from "./codes"
import { keyCodeToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
import {
ProtoInput,
ProtoInputSchema,
ProtoKeyDownSchema,
ProtoKeyUpSchema,
} from "./proto/types_pb";
import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -26,35 +19,26 @@ export class Keyboard {
constructor({ webrtc }: Props) {
this.wrtc = webrtc;
this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyDown",
value: create(ProtoKeyDownSchema, {
type: "KeyDown",
key: this.keyToVirtualKeyCode(e.code)
this.keydownListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
}
}));
this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyUp",
value: create(ProtoKeyUpSchema, {
type: "KeyUp",
key: this.keyToVirtualKeyCode(e.code)
);
this.keyupListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
}
}));
this.run()
);
this.run();
}
private run() {
if (this.connected)
this.stop()
if (this.connected) this.stop();
this.connected = true
document.addEventListener("keydown", this.keydownListener, {passive: false});
this.connected = true;
document.addEventListener("keydown", this.keydownListener, {
passive: false,
});
document.addEventListener("keyup", this.keyupListener, { passive: false });
}
@@ -65,42 +49,19 @@ export class Keyboard {
}
// Helper function to create and return mouse listeners
private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createKeyboardListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
// Prevent repeated key events from being sent (important for games)
if ((e as any).repeat)
return;
if ((e as any).repeat) return;
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-keyboard");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}

305
packages/input/src/messages.ts
View File

@@ -1,305 +0,0 @@
import { LatencyTracker } from "./latency";
import { Uint8ArrayList } from "uint8arraylist";
import { allocUnsafe } from "uint8arrays/alloc";
import { pipe } from "it-pipe";
import { decode, encode } from "it-length-prefixed";
import { Stream } from "@libp2p/interface";
export interface MessageBase {
payload_type: string;
latency?: LatencyTracker;
}
export interface MessageRaw extends MessageBase {
data: any;
}
export function NewMessageRaw(type: string, data: any): Uint8Array {
const msg = {
payload_type: type,
data: data,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageICE extends MessageBase {
candidate: RTCIceCandidateInit;
}
export function NewMessageICE(
type: string,
candidate: RTCIceCandidateInit,
): Uint8Array {
const msg = {
payload_type: type,
candidate: candidate,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageSDP extends MessageBase {
sdp: RTCSessionDescriptionInit;
}
export function NewMessageSDP(
type: string,
sdp: RTCSessionDescriptionInit,
): Uint8Array {
const msg = {
payload_type: type,
sdp: sdp,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
const MAX_SIZE = 1024 * 1024; // 1MB
const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue
// Custom 4-byte length encoder
export const length4ByteEncoder = (length: number) => {
const buf = allocUnsafe(4);
// Write the length as a 32-bit unsigned integer (4 bytes)
buf[0] = length >>> 24;
buf[1] = (length >>> 16) & 0xff;
buf[2] = (length >>> 8) & 0xff;
buf[3] = length & 0xff;
// Set the bytes property to 4
length4ByteEncoder.bytes = 4;
return buf;
};
length4ByteEncoder.bytes = 4;
// Custom 4-byte length decoder
export const length4ByteDecoder = (data: Uint8ArrayList) => {
if (data.byteLength < 4) {
// Not enough bytes to read the length
return -1;
}
// Read the length from the first 4 bytes
let length = 0;
length =
(data.subarray(0, 1)[0] >>> 0) * 0x1000000 +
(data.subarray(1, 2)[0] >>> 0) * 0x10000 +
(data.subarray(2, 3)[0] >>> 0) * 0x100 +
(data.subarray(3, 4)[0] >>> 0);
// Set bytes read to 4
length4ByteDecoder.bytes = 4;
return length;
};
length4ByteDecoder.bytes = 4;
interface PromiseMessage {
data: Uint8Array;
resolve: () => void;
reject: (error: Error) => void;
}
export class SafeStream {
private stream: Stream;
private callbacks: Map<string, ((data: any) => void)[]> = new Map();
private isReading: boolean = false;
private isWriting: boolean = false;
private closed: boolean = false;
private messageQueue: PromiseMessage[] = [];
private writeLock = false;
private readRetries = 0;
private writeRetries = 0;
private readonly MAX_RETRIES = 5;
constructor(stream: Stream) {
this.stream = stream;
this.startReading();
this.startWriting();
}
private async startReading(): Promise<void> {
if (this.isReading || this.closed) return;
this.isReading = true;
try {
const source = this.stream.source;
const decodedSource = decode(source, {
maxDataLength: MAX_SIZE,
lengthDecoder: length4ByteDecoder,
});
for await (const chunk of decodedSource) {
if (this.closed) break;
this.readRetries = 0;
try {
const data = chunk.slice();
const message = JSON.parse(
new TextDecoder().decode(data),
) as MessageBase;
const msgType = message.payload_type;
if (this.callbacks.has(msgType)) {
const handlers = this.callbacks.get(msgType)!;
for (const handler of handlers) {
try {
handler(message);
} catch (err) {
console.error(`Error in message handler for ${msgType}:`, err);
}
}
}
} catch (err) {
console.error("Error processing message:", err);
}
}
} catch (err) {
console.error("Stream reading error:", err);
} finally {
this.isReading = false;
this.readRetries++;
// If not closed, try to restart reading
if (!this.closed && this.readRetries < this.MAX_RETRIES)
setTimeout(() => this.startReading(), 100);
else if (this.readRetries >= this.MAX_RETRIES)
console.error(
"Max retries reached for reading stream, stopping attempts",
);
}
}
public registerCallback(
msgType: string,
callback: (data: any) => void,
): void {
if (!this.callbacks.has(msgType)) {
this.callbacks.set(msgType, []);
}
this.callbacks.get(msgType)!.push(callback);
}
public removeCallback(msgType: string, callback: (data: any) => void): void {
if (this.callbacks.has(msgType)) {
const callbacks = this.callbacks.get(msgType)!;
const index = callbacks.indexOf(callback);
if (index !== -1) {
callbacks.splice(index, 1);
}
if (callbacks.length === 0) {
this.callbacks.delete(msgType);
}
}
}
private async startWriting(): Promise<void> {
if (this.isWriting || this.closed) return;
this.isWriting = true;
try {
// Create an async generator for real-time message processing
const messageSource = async function* (this: SafeStream) {
while (!this.closed) {
// Check if we have messages to send
if (this.messageQueue.length > 0) {
this.writeLock = true;
try {
const message = this.messageQueue[0];
// Encode the message
const encoded = encode([message.data], {
maxDataLength: MAX_SIZE,
lengthEncoder: length4ByteEncoder,
});
for await (const chunk of encoded) {
yield chunk;
}
// Remove message after successful sending
this.writeRetries = 0;
const sentMessage = this.messageQueue.shift();
if (sentMessage)
sentMessage.resolve();
} catch (err) {
console.error("Error encoding or sending message:", err);
const failedMessage = this.messageQueue.shift();
if (failedMessage)
failedMessage.reject(new Error(`Failed to send message: ${err}`));
} finally {
this.writeLock = false;
}
} else {
// No messages to send, wait for a short period
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}.bind(this);
await pipe(messageSource(), this.stream.sink).catch((err) => {
console.error("Sink error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing to stream sink, stopping attempts");
}
});
} catch (err) {
console.error("Stream writing error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing stream, stopping attempts");
}
}
}
public async writeMessage(message: Uint8Array): Promise<void> {
if (this.closed) {
throw new Error("Cannot write to closed stream");
}
// Validate message size before queuing
if (message.length > MAX_SIZE) {
throw new Error("Message size exceeds maximum size limit");
}
// Check if the message queue is too large
if (this.messageQueue.length >= MAX_QUEUE_SIZE) {
throw new Error("Message queue is full, cannot write message");
}
// Create a promise to resolve when the message is sent
return new Promise((resolve, reject) => {
this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage);
});
}
public close(): void {
this.closed = true;
this.callbacks.clear();
// Reject pending messages
for (const msg of this.messageQueue)
msg.reject(new Error("Stream closed"));
this.messageQueue = [];
this.readRetries = 0;
this.writeRetries = 0;
}
}

150
packages/input/src/mouse.ts
View File

@@ -1,18 +1,14 @@
import { WebRTCStream } from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
import {
ProtoInput, ProtoInputSchema,
ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
ProtoMouseMove,
ProtoMouseKeyDownSchema,
ProtoMouseKeyUpSchema,
ProtoMouseMoveSchema,
ProtoMouseWheel, ProtoMouseWheelSchema
ProtoMouseWheelSchema,
} from "./proto/types_pb";
import { mouseButtonToLinuxEventCode } from "./codes";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import { create, toBinary } from "@bufbuild/protobuf";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,7 +20,7 @@ export class Mouse {
protected canvas: HTMLCanvasElement;
protected connected!: boolean;
private sendInterval = 10 // 100 updates per second
private sendInterval = 10; // 100 updates per second
// Store references to event listeners
private readonly mousemoveListener: (e: MouseEvent) => void;
@@ -48,65 +44,56 @@ export class Mouse {
this.movementY += e.movementY;
};
this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyDown",
value: create(ProtoMouseKeyDownSchema, {
type: "MouseKeyDown",
key: this.keyToVirtualKeyCode(e.button)
this.mousedownListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
}
}));
this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyUp",
value: create(ProtoMouseKeyUpSchema, {
type: "MouseKeyUp",
key: this.keyToVirtualKeyCode(e.button)
);
this.mouseupListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
}
}));
this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseWheel",
value: create(ProtoMouseWheelSchema, {
type: "MouseWheel",
);
this.mousewheelListener = this.createMouseListener((e: any) =>
create(ProtoMouseWheelSchema, {
x: Math.round(e.deltaX),
y: Math.round(e.deltaY),
}),
}
}));
);
this.run()
this.run();
this.startProcessing();
}
private run() {
//calls all the other functions
if (!document.pointerLockElement) {
console.log("no pointerlock")
console.log("no pointerlock");
if (this.connected) {
this.stop()
this.stop();
}
return;
}
if (document.pointerLockElement == this.canvas) {
this.connected = true
this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
this.connected = true;
this.canvas.addEventListener("mousemove", this.mousemoveListener, {
passive: false,
});
this.canvas.addEventListener("mousedown", this.mousedownListener, {
passive: false,
});
this.canvas.addEventListener("mouseup", this.mouseupListener, {
passive: false,
});
this.canvas.addEventListener("wheel", this.mousewheelListener, {
passive: false,
});
} else {
if (this.connected) {
this.stop()
this.stop();
}
}
}
private stop() {
@@ -128,79 +115,26 @@ export class Mouse {
}
private sendAggregatedMouseMove() {
const data = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseMove",
value: create(ProtoMouseMoveSchema, {
type: "MouseMove",
const data = create(ProtoMouseMoveSchema, {
x: Math.round(this.movementX),
y: Math.round(this.movementY),
}),
},
});
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
}
// Helper function to create and return mouse listeners
private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createMouseListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}

2
packages/input/src/proto/latency_tracker_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file latency_tracker.proto (package proto, syntax proto3)
/* eslint-disable */

144
packages/input/src/proto/messages_pb.ts
View File

@@ -1,10 +1,10 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file messages.proto (package proto, syntax proto3)
/* eslint-disable */
import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
import type { ProtoInput } from "./types_pb";
import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerAxis, ProtoControllerButton, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStick, ProtoControllerTrigger, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb";
import { file_types } from "./types_pb";
import type { ProtoLatencyTracker } from "./latency_tracker_pb";
import { file_latency_tracker } from "./latency_tracker_pb";
@@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf";
* Describes the file messages.proto.
*/
export const file_messages: GenFile = /*@__PURE__*/
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiyQgKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9idXR0b24YCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJCdXR0b25IABI7ChJjb250cm9sbGVyX3RyaWdnZXIYDCABKAsyHS5wcm90by5Qcm90b0NvbnRyb2xsZXJUcmlnZ2VySAASNwoQY29udHJvbGxlcl9zdGljaxgNIAEoCzIbLnByb3RvLlByb3RvQ29udHJvbGxlclN0aWNrSAASNQoPY29udHJvbGxlcl9heGlzGA4gASgLMhoucHJvdG8uUHJvdG9Db250cm9sbGVyQXhpc0gAEjkKEWNvbnRyb2xsZXJfcnVtYmxlGA8gASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyUnVtYmxlSAASHgoDaWNlGBQgASgLMg8ucHJvdG8uUHJvdG9JQ0VIABIeCgNzZHAYFSABKAsyDy5wcm90by5Qcm90b1NEUEgAEh4KA3JhdxgWIAEoCzIPLnByb3RvLlByb3RvUmF3SAASSQoaY2xpZW50X3JlcXVlc3Rfcm9vbV9zdHJlYW0YFyABKAsyIy5wcm90by5Qcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtSAASPQoTY2xpZW50X2Rpc2Nvbm5lY3RlZBgYIAEoCzIeLnByb3RvLlByb3RvQ2xpZW50RGlzY29ubmVjdGVkSAASOgoSc2VydmVyX3B1c2hfc3RyZWFtGBkgASgLMhwucHJvdG8uUHJvdG9TZXJ2ZXJQdXNoU3RyZWFtSABCCQoHcGF5bG9hZEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
/**
* @generated from message proto.ProtoMessageBase
@@ -39,24 +39,148 @@ export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*
messageDesc(file_messages, 0);
/**
* @generated from message proto.ProtoMessageInput
* @generated from message proto.ProtoMessage
*/
export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
export type ProtoMessage = Message<"proto.ProtoMessage"> & {
/**
* @generated from field: proto.ProtoMessageBase message_base = 1;
*/
messageBase?: ProtoMessageBase;
/**
* @generated from field: proto.ProtoInput data = 2;
* @generated from oneof proto.ProtoMessage.payload
*/
data?: ProtoInput;
payload: {
/**
* Input types
*
* @generated from field: proto.ProtoMouseMove mouse_move = 2;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 4;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 7;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 8;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* @generated from field: proto.ProtoControllerAttach controller_attach = 9;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 10;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerButton controller_button = 11;
*/
value: ProtoControllerButton;
case: "controllerButton";
} | {
/**
* @generated from field: proto.ProtoControllerTrigger controller_trigger = 12;
*/
value: ProtoControllerTrigger;
case: "controllerTrigger";
} | {
/**
* @generated from field: proto.ProtoControllerStick controller_stick = 13;
*/
value: ProtoControllerStick;
case: "controllerStick";
} | {
/**
* @generated from field: proto.ProtoControllerAxis controller_axis = 14;
*/
value: ProtoControllerAxis;
case: "controllerAxis";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 15;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | {
/**
* Signaling types
*
* @generated from field: proto.ProtoICE ice = 20;
*/
value: ProtoICE;
case: "ice";
} | {
/**
* @generated from field: proto.ProtoSDP sdp = 21;
*/
value: ProtoSDP;
case: "sdp";
} | {
/**
* @generated from field: proto.ProtoRaw raw = 22;
*/
value: ProtoRaw;
case: "raw";
} | {
/**
* @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23;
*/
value: ProtoClientRequestRoomStream;
case: "clientRequestRoomStream";
} | {
/**
* @generated from field: proto.ProtoClientDisconnected client_disconnected = 24;
*/
value: ProtoClientDisconnected;
case: "clientDisconnected";
} | {
/**
* @generated from field: proto.ProtoServerPushStream server_push_stream = 25;
*/
value: ProtoServerPushStream;
case: "serverPushStream";
} | { case: undefined; value?: undefined };
};
/**
* Describes the message proto.ProtoMessageInput.
* Use `create(ProtoMessageInputSchema)` to create a new message.
* Describes the message proto.ProtoMessage.
* Use `create(ProtoMessageSchema)` to create a new message.
*/
export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
export const ProtoMessageSchema: GenMessage<ProtoMessage> = /*@__PURE__*/
messageDesc(file_messages, 1);

416
packages/input/src/proto/types_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file types.proto (package proto, syntax proto3)
/* eslint-disable */
@@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf";
* Describes the file types.proto.
*/
export const file_types: GenFile = /*@__PURE__*/
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSJiChVQcm90b0NvbnRyb2xsZXJCdXR0b24SFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSDgoGYnV0dG9uGAMgASgFEg8KB3ByZXNzZWQYBCABKAgiYgoWUHJvdG9Db250cm9sbGVyVHJpZ2dlchIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFImUKFFByb3RvQ29udHJvbGxlclN0aWNrEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEg0KBXN0aWNrGAMgASgFEgkKAXgYBCABKAUSCQoBeRgFIAEoBSJcChNQcm90b0NvbnRyb2xsZXJBeGlzEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEgwKBGF4aXMYAyABKAUSDQoFdmFsdWUYBCABKAUiggEKFVByb3RvQ29udHJvbGxlclJ1bWJsZRIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIVCg1sb3dfZnJlcXVlbmN5GAMgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAQgASgFEhAKCGR1cmF0aW9uGAUgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
/**
* MouseMove message
@@ -19,19 +19,12 @@ export const file_types: GenFile = /*@__PURE__*/
*/
export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
/**
* Fixed value "MouseMove"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -50,19 +43,12 @@ export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
*/
export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
/**
* Fixed value "MouseMoveAbs"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -81,19 +67,12 @@ export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE_
*/
export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
/**
* Fixed value "MouseWheel"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -112,14 +91,7 @@ export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
*/
export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
/**
* Fixed value "MouseKeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -138,14 +110,7 @@ export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE_
*/
export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
/**
* Fixed value "MouseKeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -164,14 +129,7 @@ export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
*/
export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
/**
* Fixed value "KeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -190,14 +148,7 @@ export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
*/
export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
/**
* Fixed value "KeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -215,26 +166,26 @@ export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
* @generated from message proto.ProtoControllerAttach
*/
export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & {
/**
* Fixed value "ControllerAttach"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* One of the following enums: "ps", "xbox" or "switch"
*
* @generated from field: string id = 2;
* @generated from field: string id = 1;
*/
id: string;
/**
* Slot number (0-3)
* Session specific slot number (0-3)
*
* @generated from field: int32 slot = 3;
* @generated from field: int32 session_slot = 2;
*/
slot: number;
sessionSlot: number;
/**
* Session ID of the client
*
* @generated from field: string session_id = 3;
*/
sessionId: string;
};
/**
@@ -251,18 +202,18 @@ export const ProtoControllerAttachSchema: GenMessage<ProtoControllerAttach> = /*
*/
export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
/**
* Fixed value "ControllerDetach"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
};
/**
@@ -279,18 +230,18 @@ export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*
*/
export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & {
/**
* Fixed value "ControllerButtons"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Button code (linux input event code)
@@ -321,18 +272,18 @@ export const ProtoControllerButtonSchema: GenMessage<ProtoControllerButton> = /*
*/
export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & {
/**
* Fixed value "ControllerTriggers"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Trigger number (0 for left, 1 for right)
@@ -363,18 +314,18 @@ export const ProtoControllerTriggerSchema: GenMessage<ProtoControllerTrigger> =
*/
export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & {
/**
* Fixed value "ControllerStick"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Stick number (0 for left, 1 for right)
@@ -412,18 +363,18 @@ export const ProtoControllerStickSchema: GenMessage<ProtoControllerStick> = /*@_
*/
export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & {
/**
* Fixed value "ControllerAxis"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
@@ -454,18 +405,18 @@ export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__P
*/
export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
/**
* Fixed value "ControllerRumble"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Low frequency rumble (0-65535)
@@ -497,105 +448,180 @@ export const ProtoControllerRumbleSchema: GenMessage<ProtoControllerRumble> = /*
messageDesc(file_types, 13);
/**
* Union of all Input types
*
* @generated from message proto.ProtoInput
* @generated from message proto.RTCIceCandidateInit
*/
export type ProtoInput = Message<"proto.ProtoInput"> & {
export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & {
/**
* @generated from oneof proto.ProtoInput.input_type
* @generated from field: string candidate = 1;
*/
inputType: {
candidate: string;
/**
* @generated from field: proto.ProtoMouseMove mouse_move = 1;
* @generated from field: optional uint32 sdpMLineIndex = 2;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
sdpMLineIndex?: number;
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
* @generated from field: optional string sdpMid = 3;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
sdpMid?: string;
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
* @generated from field: optional string usernameFragment = 4;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 6;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 7;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* @generated from field: proto.ProtoControllerAttach controller_attach = 8;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 9;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerButton controller_button = 10;
*/
value: ProtoControllerButton;
case: "controllerButton";
} | {
/**
* @generated from field: proto.ProtoControllerTrigger controller_trigger = 11;
*/
value: ProtoControllerTrigger;
case: "controllerTrigger";
} | {
/**
* @generated from field: proto.ProtoControllerStick controller_stick = 12;
*/
value: ProtoControllerStick;
case: "controllerStick";
} | {
/**
* @generated from field: proto.ProtoControllerAxis controller_axis = 13;
*/
value: ProtoControllerAxis;
case: "controllerAxis";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 14;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | { case: undefined; value?: undefined };
usernameFragment?: string;
};
/**
* Describes the message proto.ProtoInput.
* Use `create(ProtoInputSchema)` to create a new message.
* Describes the message proto.RTCIceCandidateInit.
* Use `create(RTCIceCandidateInitSchema)` to create a new message.
*/
export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
export const RTCIceCandidateInitSchema: GenMessage<RTCIceCandidateInit> = /*@__PURE__*/
messageDesc(file_types, 14);
/**
* @generated from message proto.RTCSessionDescriptionInit
*/
export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & {
/**
* @generated from field: string sdp = 1;
*/
sdp: string;
/**
* @generated from field: string type = 2;
*/
type: string;
};
/**
* Describes the message proto.RTCSessionDescriptionInit.
* Use `create(RTCSessionDescriptionInitSchema)` to create a new message.
*/
export const RTCSessionDescriptionInitSchema: GenMessage<RTCSessionDescriptionInit> = /*@__PURE__*/
messageDesc(file_types, 15);
/**
* ProtoICE message
*
* @generated from message proto.ProtoICE
*/
export type ProtoICE = Message<"proto.ProtoICE"> & {
/**
* @generated from field: proto.RTCIceCandidateInit candidate = 1;
*/
candidate?: RTCIceCandidateInit;
};
/**
* Describes the message proto.ProtoICE.
* Use `create(ProtoICESchema)` to create a new message.
*/
export const ProtoICESchema: GenMessage<ProtoICE> = /*@__PURE__*/
messageDesc(file_types, 16);
/**
* ProtoSDP message
*
* @generated from message proto.ProtoSDP
*/
export type ProtoSDP = Message<"proto.ProtoSDP"> & {
/**
* @generated from field: proto.RTCSessionDescriptionInit sdp = 1;
*/
sdp?: RTCSessionDescriptionInit;
};
/**
* Describes the message proto.ProtoSDP.
* Use `create(ProtoSDPSchema)` to create a new message.
*/
export const ProtoSDPSchema: GenMessage<ProtoSDP> = /*@__PURE__*/
messageDesc(file_types, 17);
/**
* ProtoRaw message
*
* @generated from message proto.ProtoRaw
*/
export type ProtoRaw = Message<"proto.ProtoRaw"> & {
/**
* @generated from field: string data = 1;
*/
data: string;
};
/**
* Describes the message proto.ProtoRaw.
* Use `create(ProtoRawSchema)` to create a new message.
*/
export const ProtoRawSchema: GenMessage<ProtoRaw> = /*@__PURE__*/
messageDesc(file_types, 18);
/**
* ProtoClientRequestRoomStream message
*
* @generated from message proto.ProtoClientRequestRoomStream
*/
export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
/**
* @generated from field: string session_id = 2;
*/
sessionId: string;
};
/**
* Describes the message proto.ProtoClientRequestRoomStream.
* Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message.
*/
export const ProtoClientRequestRoomStreamSchema: GenMessage<ProtoClientRequestRoomStream> = /*@__PURE__*/
messageDesc(file_types, 19);
/**
* ProtoClientDisconnected message
*
* @generated from message proto.ProtoClientDisconnected
*/
export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & {
/**
* @generated from field: string session_id = 1;
*/
sessionId: string;
/**
* @generated from field: repeated int32 controller_slots = 2;
*/
controllerSlots: number[];
};
/**
* Describes the message proto.ProtoClientDisconnected.
* Use `create(ProtoClientDisconnectedSchema)` to create a new message.
*/
export const ProtoClientDisconnectedSchema: GenMessage<ProtoClientDisconnected> = /*@__PURE__*/
messageDesc(file_types, 20);
/**
* ProtoServerPushStream message
*
* @generated from message proto.ProtoServerPushStream
*/
export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
};
/**
* Describes the message proto.ProtoServerPushStream.
* Use `create(ProtoServerPushStreamSchema)` to create a new message.
*/
export const ProtoServerPushStreamSchema: GenMessage<ProtoServerPushStream> = /*@__PURE__*/
messageDesc(file_types, 21);

81
packages/input/src/streamwrapper.ts Normal file
View File

@@ -0,0 +1,81 @@
import { pbStream, type ProtobufStream } from "@libp2p/utils";
import type { Stream } from "@libp2p/interface";
import { bufbuildAdapter } from "./utils";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBase,
} from "./proto/messages_pb";
type MessageHandler = (
data: any,
base: ProtoMessageBase,
) => void | Promise<void>;
export class P2PMessageStream {
private pb: ProtobufStream;
private handlers = new Map<string, MessageHandler[]>();
private closed = false;
private readLoopRunning = false;
constructor(stream: Stream) {
this.pb = pbStream(stream);
}
public on(payloadType: string, handler: MessageHandler): void {
if (!this.handlers.has(payloadType)) {
this.handlers.set(payloadType, []);
}
this.handlers.get(payloadType)!.push(handler);
if (!this.readLoopRunning) this.startReading().catch(console.error);
}
private async startReading(): Promise<void> {
if (this.readLoopRunning || this.closed) return;
this.readLoopRunning = true;
while (!this.closed) {
try {
const msg: ProtoMessage = await this.pb.read(
bufbuildAdapter(ProtoMessageSchema),
);
const payloadType = msg.messageBase?.payloadType;
if (payloadType && this.handlers.has(payloadType)) {
const handlers = this.handlers.get(payloadType)!;
if (msg.payload.value) {
for (const handler of handlers) {
try {
await handler(msg.payload.value, msg.messageBase);
} catch (err) {
console.error(`Error in handler for ${payloadType}:`, err);
}
}
}
}
} catch (err) {
if (this.closed) break;
console.error("Stream read error:", err);
this.close();
}
}
this.readLoopRunning = false;
}
public async write(
message: ProtoMessage,
options?: { signal?: AbortSignal },
): Promise<void> {
if (this.closed)
throw new Error("Cannot write to closed stream");
await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options);
}
public close(): void {
this.closed = true;
this.handlers.clear();
}
}

95
packages/input/src/utils.ts Normal file
View File

@@ -0,0 +1,95 @@
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import type { Message } from "@bufbuild/protobuf";
import { Uint8ArrayList } from "uint8arraylist";
import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
import { timestampFromDate } from "@bufbuild/protobuf/wkt";
import {
ProtoLatencyTracker,
ProtoLatencyTrackerSchema,
ProtoTimestampEntrySchema,
} from "./proto/latency_tracker_pb";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBaseSchema,
} from "./proto/messages_pb";
export function bufbuildAdapter<T extends Message>(schema: GenMessage<T>) {
return {
encode: (data: T): Uint8Array => {
return toBinary(schema, data);
},
decode: (data: Uint8Array | Uint8ArrayList): T => {
// Convert Uint8ArrayList to Uint8Array if needed
const bytes = data instanceof Uint8ArrayList ? data.subarray() : data;
return fromBinary(schema, bytes);
},
};
}
// Latency tracker helpers
export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker {
return create(ProtoLatencyTrackerSchema, {
sequenceId: sequenceId || crypto.randomUUID(),
timestamps: [],
});
}
export function addLatencyTimestamp(
tracker: ProtoLatencyTracker,
stage: string,
): ProtoLatencyTracker {
const entry = create(ProtoTimestampEntrySchema, {
stage,
time: timestampFromDate(new Date()),
});
return {
...tracker,
timestamps: [...tracker.timestamps, entry],
};
}
interface CreateMessageOptions {
sequenceId?: string;
}
function derivePayloadCase(data: Message): string {
// Extract case from $typeName: "proto.ProtoICE" -> "ice"
// "proto.ProtoControllerAttach" -> "controllerAttach"
const typeName = data.$typeName;
if (!typeName)
throw new Error("Message has no $typeName");
// Remove "proto.Proto" prefix and convert first char to lowercase
const caseName = typeName.replace(/^proto\.Proto/, "");
// Convert PascalCase to camelCase
// If it's all caps (like SDP, ICE), lowercase everything
// Otherwise, just lowercase the first character
if (caseName === caseName.toUpperCase()) {
return caseName.toLowerCase();
}
return caseName.charAt(0).toLowerCase() + caseName.slice(1);
}
export function createMessage(
data: Message,
payloadType: string,
options?: CreateMessageOptions,
): ProtoMessage {
const payloadCase = derivePayloadCase(data);
return create(ProtoMessageSchema, {
messageBase: create(ProtoMessageBaseSchema, {
payloadType,
latency: options?.sequenceId
? createLatencyTracker(options.sequenceId)
: undefined,
}),
payload: {
case: payloadCase,
value: data,
} as any, // Type assertion needed for dynamic case
});
}

119
packages/input/src/webrtc-stream.ts
View File

@@ -1,9 +1,3 @@
import {
NewMessageRaw,
NewMessageSDP,
NewMessageICE,
SafeStream,
} from "./messages";
import { webSockets } from "@libp2p/websockets";
import { webTransport } from "@libp2p/webtransport";
import { createLibp2p, Libp2p } from "libp2p";
@@ -13,19 +7,33 @@ import { identify } from "@libp2p/identify";
import { multiaddr } from "@multiformats/multiaddr";
import { Connection } from "@libp2p/interface";
import { ping } from "@libp2p/ping";
import { createMessage } from "./utils";
import { create } from "@bufbuild/protobuf";
import {
ProtoClientRequestRoomStream,
ProtoClientRequestRoomStreamSchema,
ProtoICE,
ProtoICESchema, ProtoRaw,
ProtoSDP,
ProtoSDPSchema
} from "./proto/types_pb";
import { P2PMessageStream } from "./streamwrapper";
const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
export class WebRTCStream {
private _sessionId: string | null = null;
private _p2p: Libp2p | undefined = undefined;
private _p2pConn: Connection | undefined = undefined;
private _p2pSafeStream: SafeStream | undefined = undefined;
private _msgStream: P2PMessageStream | undefined = undefined;
private _pc: RTCPeerConnection | undefined = undefined;
private _audioTrack: MediaStreamTrack | undefined = undefined;
private _videoTrack: MediaStreamTrack | undefined = undefined;
private _dataChannel: RTCDataChannel | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined =
undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined =
undefined;
private _serverURL: string | undefined = undefined;
private _roomName: string | undefined = undefined;
private _isConnected: boolean = false;
@@ -89,14 +97,20 @@ export class WebRTCStream {
.newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
.catch(console.error);
if (stream) {
this._p2pSafeStream = new SafeStream(stream);
this._msgStream = new P2PMessageStream(stream);
console.log("Stream opened with peer");
let iceHolder: RTCIceCandidateInit[] = [];
this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
this._msgStream.on("ice-candidate", (data: ProtoICE) => {
const cand: RTCIceCandidateInit = {
candidate: data.candidate.candidate,
sdpMLineIndex: data.candidate.sdpMLineIndex,
sdpMid: data.candidate.sdpMid,
usernameFragment: data.candidate.usernameFragment,
};
if (this._pc) {
if (this._pc.remoteDescription) {
this._pc.addIceCandidate(data.candidate).catch((err) => {
this._pc.addIceCandidate(cand).catch((err) => {
console.error("Error adding ICE candidate:", err);
});
// Add held candidates
@@ -107,45 +121,72 @@ export class WebRTCStream {
});
iceHolder = [];
} else {
iceHolder.push(data.candidate);
iceHolder.push(cand);
}
} else {
iceHolder.push(data.candidate);
iceHolder.push(cand);
}
});
this._p2pSafeStream.registerCallback("offer", async (data) => {
this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => {
this._sessionId = data.sessionId;
localStorage.setItem("nestri-session-id", this._sessionId);
console.log("Session ID assigned:", this._sessionId, "for room:", data.roomName);
});
this._msgStream.on("offer", async (data: ProtoSDP) => {
if (!this._pc) {
// Setup peer connection now
this._setupPeerConnection();
}
await this._pc!.setRemoteDescription(data.sdp);
await this._pc!.setRemoteDescription({
sdp: data.sdp.sdp,
type: data.sdp.type as RTCSdpType,
});
// Create our answer
const answer = await this._pc!.createAnswer();
// Force stereo in Chromium browsers
answer.sdp = this.forceOpusStereo(answer.sdp!);
await this._pc!.setLocalDescription(answer);
// Send answer back
const answerMsg = NewMessageSDP("answer", answer);
await this._p2pSafeStream?.writeMessage(answerMsg);
const answerMsg = createMessage(
create(ProtoSDPSchema, {
sdp: answer,
}),
"answer",
);
await this._msgStream?.write(answerMsg);
});
this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
console.warn("Stream is offline for room:", data.roomName);
this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => {
console.warn("Stream is offline for room:", msg.data);
this._onConnected?.(null);
});
const clientId = this.getSessionID();
if (clientId) {
console.debug("Using existing session ID:", clientId);
}
// Send stream request
// marshal room name into json
const request = NewMessageRaw(
const requestMsg = createMessage(
create(ProtoClientRequestRoomStreamSchema, {
roomName: roomName,
sessionId: clientId,
}),
"request-stream-room",
roomName,
);
await this._p2pSafeStream.writeMessage(request);
await this._msgStream.write(requestMsg);
}
}
}
public getSessionID(): string | null {
if (this._sessionId === null)
this._sessionId = localStorage.getItem("nestri-session-id");
return this._sessionId;
}
// Forces opus to stereo in Chromium browsers, because of course
private forceOpusStereo(SDP: string): string {
// Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
@@ -200,11 +241,16 @@ export class WebRTCStream {
this._pc.onicecandidate = (e) => {
if (e.candidate) {
const iceMsg = NewMessageICE("ice-candidate", e.candidate);
if (this._p2pSafeStream) {
this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
console.error("Error sending ICE candidate:", err),
const iceMsg = createMessage(
create(ProtoICESchema, {
candidate: e.candidate,
}),
"ice-candidate",
);
if (this._msgStream) {
this._msgStream
.write(iceMsg)
.catch((err) => console.error("Error sending ICE candidate:", err));
} else {
console.warn("P2P stream not established, cannot send ICE candidate");
}
@@ -218,8 +264,7 @@ export class WebRTCStream {
}
private _checkConnectionState() {
if (!this._pc || !this._p2p || !this._p2pConn)
return;
if (!this._pc || !this._p2p || !this._p2pConn) return;
console.debug("Checking connection state:", {
connectionState: this._pc.connectionState,
@@ -256,7 +301,7 @@ export class WebRTCStream {
// @ts-ignore
receiver.jitterBufferTarget = receiver.jitterBufferDelayHint = receiver.playoutDelayHint = 0;
}
}, 15);
}, 50);
});
}
}
@@ -286,7 +331,9 @@ export class WebRTCStream {
// Attempt to reconnect only if not already connected
if (!this._isConnected && this._serverURL && this._roomName) {
this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
this._setup(this._serverURL, this._roomName).catch((err) =>
console.error("Reconnection failed:", err),
);
}
}
@@ -335,7 +382,9 @@ export class WebRTCStream {
}
public removeDataChannelCallback(callback: (data: any) => void) {
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback);
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(
(cb) => cb !== callback,
);
}
private _setupDataChannelEvents() {
@@ -343,7 +392,7 @@ export class WebRTCStream {
this._dataChannel.onclose = () => console.log("sendChannel has closed");
this._dataChannel.onopen = () => console.log("sendChannel has opened");
this._dataChannel.onmessage = (event => {
this._dataChannel.onmessage = (event) => {
// Parse as ProtoBuf message
const data = event.data;
// Call registered callback if exists
@@ -354,7 +403,7 @@ export class WebRTCStream {
console.error("Error in data channel callback:", err);
}
});
});
};
}
private _gatherFrameRate() {

4
packages/play-standalone/src/pages/[room].astro
View File

@@ -90,11 +90,7 @@ if (envs_map.size > 0) {
let nestriControllers: Controller[] = [];
window.addEventListener("gamepadconnected", (e) => {
// Ignore gamepads with id including "nestri"
console.log("Gamepad connected:", e.gamepad);
if (e.gamepad.id.toLowerCase().includes("nestri"))
return;
const controller = new Controller({
webrtc: stream,
e: e,

2
packages/relay/go.mod
View File

@@ -33,7 +33,7 @@ require (
github.com/ipfs/go-cid v0.5.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/koron/go-ssdp v0.1.0 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect

4
packages/relay/go.sum
View File

@@ -82,8 +82,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/koron/go-ssdp v0.1.0 h1:ckl5x5H6qSNFmi+wCuROvvGUu2FQnMbQrU95IHCcv3Y=

2
packages/relay/internal/common/common.go
View File

@@ -26,7 +26,7 @@ func InitWebRTCAPI() error {
mediaEngine := &webrtc.MediaEngine{}
// Register our extensions
if err := RegisterExtensions(mediaEngine); err != nil {
if err = RegisterExtensions(mediaEngine); err != nil {
return fmt.Errorf("failed to register extensions: %w", err)
}

170
packages/relay/internal/common/safebufio.go
View File

@@ -3,16 +3,28 @@ package common
import (
"bufio"
"encoding/binary"
"encoding/json"
"errors"
"io"
gen "relay/internal/proto"
"sync"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/timestamppb"
)
// MaxSize is the maximum allowed data size (1MB)
const MaxSize = 1024 * 1024
// readUvarint reads an unsigned varint from the reader
func readUvarint(r io.ByteReader) (uint64, error) {
return binary.ReadUvarint(r)
}
// writeUvarint writes an unsigned varint to the writer
func writeUvarint(w io.Writer, x uint64) error {
buf := make([]byte, binary.MaxVarintLen64)
n := binary.PutUvarint(buf, x)
_, err := w.Write(buf[:n])
return err
}
// SafeBufioRW wraps a bufio.ReadWriter for sending and receiving JSON and protobufs safely
type SafeBufioRW struct {
@@ -24,83 +36,6 @@ func NewSafeBufioRW(brw *bufio.ReadWriter) *SafeBufioRW {
return &SafeBufioRW{brw: brw}
}
// SendJSON serializes the given data as JSON and sends it with a 4-byte length prefix
func (bu *SafeBufioRW) SendJSON(data interface{}) error {
bu.mutex.Lock()
defer bu.mutex.Unlock()
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
if len(jsonData) > MaxSize {
return errors.New("JSON data exceeds maximum size")
}
// Write the 4-byte length prefix
if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(jsonData))); err != nil {
return err
}
// Write the JSON data
if _, err = bu.brw.Write(jsonData); err != nil {
return err
}
// Flush the writer to ensure data is sent
return bu.brw.Flush()
}
// ReceiveJSON reads a 4-byte length prefix, then reads and unmarshals the JSON
func (bu *SafeBufioRW) ReceiveJSON(dest interface{}) error {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
return err
}
if length > MaxSize {
return errors.New("received JSON data exceeds maximum size")
}
// Read the JSON data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
return err
}
return json.Unmarshal(data, dest)
}
// Receive reads a 4-byte length prefix, then reads the raw data
func (bu *SafeBufioRW) Receive() ([]byte, error) {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
return nil, err
}
if length > MaxSize {
return nil, errors.New("received data exceeds maximum size")
}
// Read the raw data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
return nil, err
}
return data, nil
}
// SendProto serializes the given protobuf message and sends it with a 4-byte length prefix
func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
bu.mutex.Lock()
defer bu.mutex.Unlock()
@@ -110,12 +45,8 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
return err
}
if len(protoData) > MaxSize {
return errors.New("protobuf data exceeds maximum size")
}
// Write the 4-byte length prefix
if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(protoData))); err != nil {
// Write varint length prefix
if err := writeUvarint(bu.brw, uint64(len(protoData))); err != nil {
return err
}
@@ -124,25 +55,19 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
return err
}
// Flush the writer to ensure data is sent
return bu.brw.Flush()
}
// ReceiveProto reads a 4-byte length prefix, then reads and unmarshals the protobuf
func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
bu.mutex.RLock()
defer bu.mutex.RUnlock()
// Read the 4-byte length prefix
var length uint32
if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
// Read varint length prefix
length, err := readUvarint(bu.brw)
if err != nil {
return err
}
if length > MaxSize {
return errors.New("received Protobuf data exceeds maximum size")
}
// Read the Protobuf data
data := make([]byte, length)
if _, err := io.ReadFull(bu.brw, data); err != nil {
@@ -152,24 +77,51 @@ func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
return proto.Unmarshal(data, msg)
}
// Write writes raw data to the underlying buffer
func (bu *SafeBufioRW) Write(data []byte) (int, error) {
bu.mutex.Lock()
defer bu.mutex.Unlock()
if len(data) > MaxSize {
return 0, errors.New("data exceeds maximum size")
type CreateMessageOptions struct {
SequenceID string
Latency *gen.ProtoLatencyTracker
}
n, err := bu.brw.Write(data)
if err != nil {
return n, err
func CreateMessage(payload proto.Message, payloadType string, opts *CreateMessageOptions) (*gen.ProtoMessage, error) {
msg := &gen.ProtoMessage{
MessageBase: &gen.ProtoMessageBase{
PayloadType: payloadType,
},
}
// Flush the writer to ensure data is sent
if err = bu.brw.Flush(); err != nil {
return n, err
if opts != nil {
if opts.Latency != nil {
msg.MessageBase.Latency = opts.Latency
} else if opts.SequenceID != "" {
msg.MessageBase.Latency = &gen.ProtoLatencyTracker{
SequenceId: opts.SequenceID,
Timestamps: []*gen.ProtoTimestampEntry{
{
Stage: "created",
Time: timestamppb.Now(),
},
},
}
}
}
return n, nil
// Use reflection to set the oneof field automatically
msgReflect := msg.ProtoReflect()
payloadReflect := payload.ProtoReflect()
oneofDesc := msgReflect.Descriptor().Oneofs().ByName("payload")
if oneofDesc == nil {
return nil, errors.New("payload oneof not found")
}
fields := oneofDesc.Fields()
for i := 0; i < fields.Len(); i++ {
field := fields.Get(i)
if field.Message() != nil && field.Message().FullName() == payloadReflect.Descriptor().FullName() {
msgReflect.Set(field, protoreflect.ValueOfMessage(payloadReflect))
return msg, nil
}
}
return nil, errors.New("payload type not found in oneof")
}

10
packages/relay/internal/connections/datachannel.go
View File

@@ -31,16 +31,18 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
}
// Decode message
var base gen.ProtoMessageInput
var base gen.ProtoMessage
if err := proto.Unmarshal(msg.Data, &base); err != nil {
slog.Error("failed to decode binary DataChannel message", "err", err)
return
}
// Handle message type callback
if callback, ok := ndc.callbacks["input"]; ok {
// Route based on PayloadType
if base.MessageBase != nil && len(base.MessageBase.PayloadType) > 0 {
if callback, ok := ndc.callbacks[base.MessageBase.PayloadType]; ok {
go callback(msg.Data)
} // We don't care about unhandled messages
}
}
})
return ndc

94
packages/relay/internal/connections/messages.go
View File

@@ -1,94 +0,0 @@
package connections
import (
"encoding/json"
"relay/internal/common"
"github.com/pion/webrtc/v4"
)
// MessageBase is the base type for any JSON message
type MessageBase struct {
Type string `json:"payload_type"`
Latency *common.LatencyTracker `json:"latency,omitempty"`
}
type MessageRaw struct {
MessageBase
Data json.RawMessage `json:"data"`
}
func NewMessageRaw(t string, data json.RawMessage) *MessageRaw {
return &MessageRaw{
MessageBase: MessageBase{
Type: t,
},
Data: data,
}
}
type MessageLog struct {
MessageBase
Level string `json:"level"`
Message string `json:"message"`
Time string `json:"time"`
}
func NewMessageLog(t string, level, message, time string) *MessageLog {
return &MessageLog{
MessageBase: MessageBase{
Type: t,
},
Level: level,
Message: message,
Time: time,
}
}
type MessageMetrics struct {
MessageBase
UsageCPU float64 `json:"usage_cpu"`
UsageMemory float64 `json:"usage_memory"`
Uptime uint64 `json:"uptime"`
PipelineLatency float64 `json:"pipeline_latency"`
}
func NewMessageMetrics(t string, usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) *MessageMetrics {
return &MessageMetrics{
MessageBase: MessageBase{
Type: t,
},
UsageCPU: usageCPU,
UsageMemory: usageMemory,
Uptime: uptime,
PipelineLatency: pipelineLatency,
}
}
type MessageICE struct {
MessageBase
Candidate webrtc.ICECandidateInit `json:"candidate"`
}
func NewMessageICE(t string, candidate webrtc.ICECandidateInit) *MessageICE {
return &MessageICE{
MessageBase: MessageBase{
Type: t,
},
Candidate: candidate,
}
}
type MessageSDP struct {
MessageBase
SDP webrtc.SessionDescription `json:"sdp"`
}
func NewMessageSDP(t string, sdp webrtc.SessionDescription) *MessageSDP {
return &MessageSDP{
MessageBase: MessageBase{
Type: t,
},
SDP: sdp,
}
}

13
packages/relay/internal/core/core.go
View File

@@ -10,6 +10,7 @@ import (
"os"
"relay/internal/common"
"relay/internal/shared"
"time"
"github.com/libp2p/go-libp2p"
pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -37,6 +38,16 @@ var globalRelay *Relay
// -- Structs --
// ClientSession tracks browser client connections
type ClientSession struct {
PeerID peer.ID
SessionID string
RoomName string
ConnectedAt time.Time
LastActivity time.Time
ControllerSlots []int32 // Track which controller slots this client owns
}
// Relay structure enhanced with metrics and state
type Relay struct {
*PeerInfo
@@ -48,6 +59,7 @@ type Relay struct {
// Local
LocalRooms *common.SafeMap[ulid.ULID, *shared.Room] // room ID -> local Room struct (hosted by this relay)
LocalMeshConnections *common.SafeMap[peer.ID, *webrtc.PeerConnection] // peer ID -> PeerConnection (connected to this relay)
ClientSessions *common.SafeMap[peer.ID, *ClientSession] // peer ID -> ClientSession
// Protocols
ProtocolRegistry
@@ -144,6 +156,7 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay
PingService: pingSvc,
LocalRooms: common.NewSafeMap[ulid.ULID, *shared.Room](),
LocalMeshConnections: common.NewSafeMap[peer.ID, *webrtc.PeerConnection](),
ClientSessions: common.NewSafeMap[peer.ID, *ClientSession](),
}
// Add network notifier after relay is initialized

717
packages/relay/internal/core/protocol_stream.go
View File

@@ -3,14 +3,19 @@ package core
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"math"
"relay/internal/common"
"relay/internal/connections"
"relay/internal/shared"
"time"
gen "relay/internal/proto"
"google.golang.org/protobuf/proto"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
@@ -69,7 +74,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
var currentRoomName string // Track the current room for this stream
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
var msgWrapper gen.ProtoMessage
err := safeBRW.ReceiveProto(&msgWrapper)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Stream request connection closed by peer", "peer", stream.Conn().RemotePeer())
@@ -82,79 +88,153 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message", "err", err)
continue
if msgWrapper.MessageBase == nil {
slog.Error("No MessageBase in stream request")
_ = stream.Reset()
return
}
switch baseMsg.Type {
switch msgWrapper.MessageBase.PayloadType {
case "request-stream-room":
var rawMsg connections.MessageRaw
if err = json.Unmarshal(data, &rawMsg); err != nil {
slog.Error("Failed to unmarshal raw message for room stream request", "err", err)
reqMsg := msgWrapper.GetClientRequestRoomStream()
if reqMsg != nil {
currentRoomName = reqMsg.RoomName
// Generate session ID if not provided (first connection)
sessionID := reqMsg.SessionId
if sessionID == "" {
ulid, err := common.NewULID()
if err != nil {
slog.Error("Failed to generate session ID", "err", err)
continue
}
var roomName string
if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
slog.Error("Failed to unmarshal room name from raw message", "err", err)
continue
sessionID = ulid.String()
}
currentRoomName = roomName // Store the room name
slog.Info("Received stream request for room", "room", roomName)
session := &ClientSession{
PeerID: stream.Conn().RemotePeer(),
SessionID: sessionID,
RoomName: reqMsg.RoomName,
ConnectedAt: time.Now(),
LastActivity: time.Now(),
}
sp.relay.ClientSessions.Set(stream.Conn().RemotePeer(), session)
room := sp.relay.GetRoomByName(roomName)
slog.Info("Client session established", "peer", session.PeerID, "session", sessionID, "room", reqMsg.RoomName)
// Send session ID back to client
sesMsg, err := common.CreateMessage(
&gen.ProtoClientRequestRoomStream{SessionId: sessionID, RoomName: reqMsg.RoomName},
"session-assigned", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(sesMsg); err != nil {
slog.Error("Failed to send session assignment", "err", err)
}
slog.Info("Received stream request for room", "room", reqMsg.RoomName)
room := sp.relay.GetRoomByName(reqMsg.RoomName)
if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID {
// TODO: Allow forward requests to other relays from here?
slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", roomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", reqMsg.RoomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
// Respond with "request-stream-offline" message with room name
// TODO: Store the peer and send "online" message when the room comes online
roomNameData, err := json.Marshal(roomName)
rawMsg, err := common.CreateMessage(
&gen.ProtoRaw{
Data: reqMsg.RoomName,
},
"request-stream-offline", nil,
)
if err != nil {
slog.Error("Failed to marshal room name for request stream offline", "room", roomName, "err", err)
slog.Error("Failed to create proto message", "err", err)
continue
} else {
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"request-stream-offline",
roomNameData,
)); err != nil {
slog.Error("Failed to send request stream offline message", "room", roomName, "err", err)
}
if err = safeBRW.SendProto(rawMsg); err != nil {
slog.Error("Failed to send request stream offline message", "room", reqMsg.RoomName, "err", err)
}
continue
}
pc, err := common.CreatePeerConnection(func() {
slog.Info("PeerConnection closed for requested stream", "room", roomName)
slog.Info("PeerConnection closed for requested stream", "room", reqMsg.RoomName)
// Cleanup the stream connection
if roomMap, ok := sp.servedConns.Get(roomName); ok {
if roomMap, ok := sp.servedConns.Get(reqMsg.RoomName); ok {
roomMap.Delete(stream.Conn().RemotePeer())
// If the room map is empty, delete it
if roomMap.Len() == 0 {
sp.servedConns.Delete(roomName)
sp.servedConns.Delete(reqMsg.RoomName)
}
}
})
if err != nil {
slog.Error("Failed to create PeerConnection for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create PeerConnection for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
// Add tracks
if room.AudioTrack != nil {
if _, err = pc.AddTrack(room.AudioTrack); err != nil {
slog.Error("Failed to add audio track for requested stream", "room", roomName, "err", err)
// Create participant for this viewer
participant, err := shared.NewParticipant(
"",
stream.Conn().RemotePeer(),
)
if err != nil {
slog.Error("Failed to create participant", "room", reqMsg.RoomName, "err", err)
continue
}
// If this is a client session, link it
if session, ok := sp.relay.ClientSessions.Get(stream.Conn().RemotePeer()); ok {
participant.SessionID = session.SessionID
}
if room.VideoTrack != nil {
if _, err = pc.AddTrack(room.VideoTrack); err != nil {
slog.Error("Failed to add video track for requested stream", "room", roomName, "err", err)
continue
// Assign peer connection
participant.PeerConnection = pc
// Add audio/video tracks
{
localTrack, err := webrtc.NewTrackLocalStaticRTP(
room.AudioCodec,
"participant-"+participant.ID.String(),
"participant-"+participant.ID.String()+"-audio",
)
if err != nil {
slog.Error("Failed to create track for stream request", "err", err)
return
}
participant.SetTrack(webrtc.RTPCodecTypeAudio, localTrack)
slog.Debug("Set audio track for requested stream", "room", room.Name)
}
{
localTrack, err := webrtc.NewTrackLocalStaticRTP(
room.VideoCodec,
"participant-"+participant.ID.String(),
"participant-"+participant.ID.String()+"-video",
)
if err != nil {
slog.Error("Failed to create track for stream request", "err", err)
return
}
participant.SetTrack(webrtc.RTPCodecTypeVideo, localTrack)
slog.Debug("Set video track for requested stream", "room", room.Name)
}
// Cleanup on disconnect
cleanupParticipantID := participant.ID
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
if state == webrtc.PeerConnectionStateClosed ||
state == webrtc.PeerConnectionStateFailed ||
state == webrtc.PeerConnectionStateDisconnected {
slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID)
room.RemoveParticipantByID(cleanupParticipantID)
participant.Close()
} else if state == webrtc.PeerConnectionStateConnected {
// Add participant to room when connection is established
room.AddParticipant(participant)
}
})
// DataChannel setup
settingOrdered := true
@@ -164,21 +244,84 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
MaxRetransmits: &settingMaxRetransmits,
})
if err != nil {
slog.Error("Failed to create DataChannel for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create DataChannel for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
ndc := connections.NewNestriDataChannel(dc)
ndc.RegisterOnOpen(func() {
slog.Debug("Relay DataChannel opened for requested stream", "room", roomName)
slog.Debug("Relay DataChannel opened for requested stream", "room", reqMsg.RoomName)
})
ndc.RegisterOnClose(func() {
slog.Debug("Relay DataChannel closed for requested stream", "room", roomName)
slog.Debug("Relay DataChannel closed for requested stream", "room", reqMsg.RoomName)
})
ndc.RegisterMessageCallback("input", func(data []byte) {
if room.DataChannel != nil {
if err = room.DataChannel.SendBinary(data); err != nil {
slog.Error("Failed to forward input message from mesh to upstream room", "room", roomName, "err", err)
slog.Error("Failed to forward input message from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
}
}
})
// Track controller input separately
ndc.RegisterMessageCallback("controllerInput", func(data []byte) {
// Parse the message to track controller slots for client sessions
var msgWrapper gen.ProtoMessage
if err = proto.Unmarshal(data, &msgWrapper); err != nil {
slog.Error("Failed to unmarshal controller input", "err", err)
} else if msgWrapper.Payload != nil {
// Get the peer ID for this connection
peerID := stream.Conn().RemotePeer()
// Check if it's a controller attach with assigned slot
if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.SessionSlot >= 0 {
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
// Check if slot already tracked
hasSlot := false
for _, slot := range session.ControllerSlots {
if slot == attach.SessionSlot {
hasSlot = true
break
}
}
if !hasSlot {
session.ControllerSlots = append(session.ControllerSlots, attach.SessionSlot)
session.LastActivity = time.Now()
slog.Info("Controller slot assigned to client session",
"session", session.SessionID,
"slot", attach.SessionSlot,
"total_slots", len(session.ControllerSlots))
}
}
}
// Check if it's a controller detach
if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.SessionSlot >= 0 {
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
newSlots := make([]int32, 0, len(session.ControllerSlots))
for _, slot := range session.ControllerSlots {
if slot != detach.SessionSlot {
newSlots = append(newSlots, slot)
}
}
session.ControllerSlots = newSlots
session.LastActivity = time.Now()
slog.Info("Controller slot removed from client session",
"session", session.SessionID,
"slot", detach.SessionSlot,
"remaining_slots", len(session.ControllerSlots))
}
}
// Update last activity on any controller input
if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
session.LastActivity = time.Now()
}
}
// Forward to upstream room
if room.DataChannel != nil {
if err = room.DataChannel.SendBinary(data); err != nil {
slog.Error("Failed to forward controller input from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
}
}
})
@@ -189,8 +332,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE("ice-candidate", candidate.ToJSON())); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", roomName, "err", err)
candInit := candidate.ToJSON()
biggified := uint32(*candInit.SDPMLineIndex)
iceMsg, err := common.CreateMessage(
&gen.ProtoICE{
Candidate: &gen.RTCIceCandidateInit{
Candidate: candInit.Candidate,
SdpMLineIndex: &biggified,
SdpMid: candInit.SDPMid,
},
},
"ice-candidate", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
return
}
if err = safeBRW.SendProto(iceMsg); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", reqMsg.RoomName, "err", err)
return
}
})
@@ -198,23 +357,36 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
// Create offer
offer, err := pc.CreateOffer(nil)
if err != nil {
slog.Error("Failed to create offer for requested stream", "room", roomName, "err", err)
slog.Error("Failed to create offer for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
if err = pc.SetLocalDescription(offer); err != nil {
slog.Error("Failed to set local description for requested stream", "room", roomName, "err", err)
slog.Error("Failed to set local description for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageSDP("offer", offer)); err != nil {
slog.Error("Failed to send offer for requested stream", "room", roomName, "err", err)
offerMsg, err := common.CreateMessage(
&gen.ProtoSDP{
Sdp: &gen.RTCSessionDescriptionInit{
Sdp: offer.SDP,
Type: offer.Type.String(),
},
},
"offer", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(offerMsg); err != nil {
slog.Error("Failed to send offer for requested stream", "room", reqMsg.RoomName, "err", err)
continue
}
// Store the connection
roomMap, ok := sp.servedConns.Get(roomName)
roomMap, ok := sp.servedConns.Get(reqMsg.RoomName)
if !ok {
roomMap = common.NewSafeMap[peer.ID, *StreamConnection]()
sp.servedConns.Set(roomName, roomMap)
sp.servedConns.Set(reqMsg.RoomName, roomMap)
}
roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{
pc: pc,
@@ -222,17 +394,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
})
slog.Debug("Sent offer for requested stream")
} else {
slog.Error("Could not get ClientRequestRoomStream for stream request")
}
case "ice-candidate":
var iceMsg connections.MessageICE
if err := json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE message", "err", err)
continue
iceMsg := msgWrapper.GetIce()
if iceMsg != nil {
smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
cand := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
SDPMid: iceMsg.Candidate.SdpMid,
SDPMLineIndex: &smollified,
UsernameFragment: iceMsg.Candidate.UsernameFragment,
}
// Use currentRoomName to get the connection from nested map
if len(currentRoomName) > 0 {
if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil {
if err := conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
if err = conn.pc.AddICECandidate(cand); err != nil {
slog.Error("Failed to add ICE candidate", "err", err)
}
for _, heldIce := range iceHolder {
@@ -244,24 +423,28 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
}
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
} else {
slog.Error("Could not GetIce from ice-candidate")
}
case "answer":
var answerMsg connections.MessageSDP
if err := json.Unmarshal(data, &answerMsg); err != nil {
slog.Error("Failed to unmarshal answer from signaling message", "err", err)
continue
answerMsg := msgWrapper.GetSdp()
if answerMsg != nil {
ansSdp := webrtc.SessionDescription{
SDP: answerMsg.Sdp.Sdp,
Type: webrtc.NewSDPType(answerMsg.Sdp.Type),
}
// Use currentRoomName to get the connection from nested map
if len(currentRoomName) > 0 {
if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok {
if err := conn.pc.SetRemoteDescription(answerMsg.SDP); err != nil {
if err = conn.pc.SetRemoteDescription(ansSdp); err != nil {
slog.Error("Failed to set remote description for answer", "err", err)
continue
}
@@ -273,199 +456,11 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
} else {
slog.Warn("Received answer without active PeerConnection")
}
}
}
}
// requestStream manages the internals of the stream request
func (sp *StreamProtocol) requestStream(stream network.Stream, room *shared.Room) error {
brw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream))
safeBRW := common.NewSafeBufioRW(brw)
slog.Debug("Requesting room stream from peer", "room", room.Name, "peer", stream.Conn().RemotePeer())
// Send room name to the remote peer
roomData, err := json.Marshal(room.Name)
if err != nil {
_ = stream.Close()
return fmt.Errorf("failed to marshal room name: %w", err)
}
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"request-stream-room",
roomData,
)); err != nil {
_ = stream.Close()
return fmt.Errorf("failed to send room request: %w", err)
}
pc, err := common.CreatePeerConnection(func() {
slog.Info("Relay PeerConnection closed for requested stream", "room", room.Name)
_ = stream.Close() // ignore error as may be closed already
// Cleanup the stream connection
if ok := sp.requestedConns.Has(room.Name); ok {
sp.requestedConns.Delete(room.Name)
}
})
if err != nil {
_ = stream.Close()
return fmt.Errorf("failed to create PeerConnection: %w", err)
}
pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
localTrack, _ := webrtc.NewTrackLocalStaticRTP(track.Codec().RTPCodecCapability, track.ID(), "relay-"+room.Name+"-"+track.Kind().String())
slog.Debug("Received track for requested stream", "room", room.Name, "track_kind", track.Kind().String())
room.SetTrack(track.Kind(), localTrack)
go func() {
for {
rtpPacket, _, err := track.ReadRTP()
if err != nil {
if !errors.Is(err, io.EOF) {
slog.Error("Failed to read RTP packet for requested stream room", "room", room.Name, "err", err)
}
break
}
err = localTrack.WriteRTP(rtpPacket)
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("Failed to write RTP to local track for requested stream room", "room", room.Name, "err", err)
break
}
}
}()
})
pc.OnDataChannel(func(dc *webrtc.DataChannel) {
ndc := connections.NewNestriDataChannel(dc)
ndc.RegisterOnOpen(func() {
slog.Debug("Relay DataChannel opened for requested stream", "room", room.Name)
})
ndc.RegisterOnClose(func() {
slog.Debug("Relay DataChannel closed for requested stream", "room", room.Name)
})
// Set the DataChannel in the requestedConns map
if conn, ok := sp.requestedConns.Get(room.Name); ok {
conn.ndc = ndc
} else {
sp.requestedConns.Set(room.Name, &StreamConnection{
pc: pc,
ndc: ndc,
})
}
// We do not handle any messages from upstream here
})
pc.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE(
"ice-candidate",
candidate.ToJSON(),
)); err != nil {
slog.Error("Failed to send ICE candidate message for requested stream", "room", room.Name, "err", err)
return
}
})
// Handle incoming messages (offer and candidates)
go func() {
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Connection for requested stream closed by peer", "room", room.Name)
return
}
slog.Error("Failed to receive data for requested stream", "room", room.Name, "err", err)
_ = stream.Reset()
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message for requested stream", "room", room.Name, "err", err)
return
}
switch baseMsg.Type {
case "ice-candidate":
var iceMsg connections.MessageICE
if err = json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE candidate for requested stream", "room", room.Name, "err", err)
continue
}
if conn, ok := sp.requestedConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
slog.Error("Failed to add ICE candidate for requested stream", "room", room.Name, "err", err)
}
// Add held candidates
for _, heldCandidate := range iceHolder {
if err = conn.pc.AddICECandidate(heldCandidate); err != nil {
slog.Error("Failed to add held ICE candidate for requested stream", "room", room.Name, "err", err)
slog.Warn("Could not GetSdp from answer")
}
}
// Clear the held candidates
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
}
case "offer":
var offerMsg connections.MessageSDP
if err = json.Unmarshal(data, &offerMsg); err != nil {
slog.Error("Failed to unmarshal offer for requested stream", "room", room.Name, "err", err)
continue
}
if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
slog.Error("Failed to set remote description for requested stream", "room", room.Name, "err", err)
continue
}
answer, err := pc.CreateAnswer(nil)
if err != nil {
slog.Error("Failed to create answer for requested stream", "room", room.Name, "err", err)
if err = stream.Reset(); err != nil {
slog.Error("Failed to reset stream for requested stream", "err", err)
}
return
}
if err = pc.SetLocalDescription(answer); err != nil {
slog.Error("Failed to set local description for requested stream", "room", room.Name, "err", err)
if err = stream.Reset(); err != nil {
slog.Error("Failed to reset stream for requested stream", "err", err)
}
return
}
if err = safeBRW.SendJSON(connections.NewMessageSDP(
"answer",
answer,
)); err != nil {
slog.Error("Failed to send answer for requested stream", "room", room.Name, "err", err)
continue
}
// Store the connection
sp.requestedConns.Set(room.Name, &StreamConnection{
pc: pc,
ndc: nil,
})
slog.Debug("Sent answer for requested stream", "room", room.Name)
default:
slog.Warn("Unknown signaling message type", "room", room.Name, "type", baseMsg.Type)
}
}
}()
return nil
}
// handleStreamPush manages a stream push from a node (nestri-server)
@@ -476,42 +471,39 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
var room *shared.Room
iceHolder := make([]webrtc.ICECandidateInit, 0)
for {
data, err := safeBRW.Receive()
var msgWrapper gen.ProtoMessage
err := safeBRW.ReceiveProto(&msgWrapper)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err)
if room != nil {
room.Close()
sp.incomingConns.Set(room.Name, nil)
}
return
}
slog.Error("Failed to receive data for stream push", "err", err)
_ = stream.Reset()
if room != nil {
room.Close()
sp.incomingConns.Set(room.Name, nil)
}
return
}
var baseMsg connections.MessageBase
if err = json.Unmarshal(data, &baseMsg); err != nil {
slog.Error("Failed to unmarshal base message from base message", "err", err)
if msgWrapper.MessageBase == nil {
slog.Error("No MessageBase in stream push")
continue
}
switch baseMsg.Type {
switch msgWrapper.MessageBase.PayloadType {
case "push-stream-room":
var rawMsg connections.MessageRaw
if err = json.Unmarshal(data, &rawMsg); err != nil {
slog.Error("Failed to unmarshal room name from data", "err", err)
continue
}
pushMsg := msgWrapper.GetServerPushStream()
if pushMsg != nil {
slog.Info("Received stream push request for room", "room", pushMsg.RoomName)
var roomName string
if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
slog.Error("Failed to unmarshal room name from raw message", "err", err)
continue
}
slog.Info("Received stream push request for room", "room", roomName)
room = sp.relay.GetRoomByName(roomName)
room = sp.relay.GetRoomByName(pushMsg.RoomName)
if room != nil {
if room.OwnerID != sp.relay.ID {
slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID)
@@ -523,34 +515,43 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
}
} else {
// Create a new room if it doesn't exist
room = sp.relay.CreateRoom(roomName)
room = sp.relay.CreateRoom(pushMsg.RoomName)
}
// Respond with an OK with the room name
roomData, err := json.Marshal(room.Name)
resMsg, err := common.CreateMessage(
&gen.ProtoServerPushStream{
RoomName: pushMsg.RoomName,
},
"push-stream-ok", nil,
)
if err != nil {
slog.Error("Failed to marshal room name for push stream response", "err", err)
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageRaw(
"push-stream-ok",
roomData,
)); err != nil {
if err = safeBRW.SendProto(resMsg); err != nil {
slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err)
continue
}
} else {
slog.Error("Failed to GetServerPushStream in push-stream-room")
}
case "ice-candidate":
var iceMsg connections.MessageICE
if err = json.Unmarshal(data, &iceMsg); err != nil {
slog.Error("Failed to unmarshal ICE candidate from data", "err", err)
continue
iceMsg := msgWrapper.GetIce()
if iceMsg != nil {
smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
cand := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
SDPMid: iceMsg.Candidate.SdpMid,
SDPMLineIndex: &smollified,
UsernameFragment: iceMsg.Candidate.UsernameFragment,
}
if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
if err = conn.pc.AddICECandidate(cand); err != nil {
slog.Error("Failed to add ICE candidate for pushed stream", "err", err)
}
for _, heldIce := range iceHolder {
if err := conn.pc.AddICECandidate(heldIce); err != nil {
if err = conn.pc.AddICECandidate(heldIce); err != nil {
slog.Error("Failed to add held ICE candidate for pushed stream", "err", err)
}
}
@@ -558,7 +559,10 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
iceHolder = make([]webrtc.ICECandidateInit, 0)
} else {
// Hold the candidate until remote description is set
iceHolder = append(iceHolder, iceMsg.Candidate)
iceHolder = append(iceHolder, cand)
}
} else {
slog.Error("Failed to GetIce in pushed stream ice-candidate")
}
case "offer":
// Make sure we have room set to push to (set by "push-stream-room")
@@ -567,12 +571,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
continue
}
var offerMsg connections.MessageSDP
if err = json.Unmarshal(data, &offerMsg); err != nil {
slog.Error("Failed to unmarshal offer from data", "err", err)
continue
offerMsg := msgWrapper.GetSdp()
if offerMsg != nil {
offSdp := webrtc.SessionDescription{
SDP: offerMsg.Sdp.Sdp,
Type: webrtc.NewSDPType(offerMsg.Sdp.Type),
}
// Create PeerConnection for the incoming stream
pc, err := common.CreatePeerConnection(func() {
slog.Info("PeerConnection closed for pushed stream", "room", room.Name)
@@ -586,6 +590,9 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
continue
}
// Assign room peer connection
room.PeerConnection = pc
pc.OnDataChannel(func(dc *webrtc.DataChannel) {
// TODO: Is this the best way to handle DataChannel? Should we just use the map directly?
room.DataChannel = connections.NewNestriDataChannel(dc)
@@ -595,20 +602,19 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
room.DataChannel.RegisterOnClose(func() {
slog.Debug("DataChannel closed for pushed stream", "room", room.Name)
})
room.DataChannel.RegisterMessageCallback("input", func(data []byte) {
if room.DataChannel != nil {
// Pass to servedConns DataChannels for this specific room
// Handle controller feedback reverse-flow (like rumble events coming from game to client)
room.DataChannel.RegisterMessageCallback("controllerInput", func(data []byte) {
// Forward controller input to all viewers
if roomMap, ok := sp.servedConns.Get(room.Name); ok {
roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool {
if conn.ndc != nil {
if err = conn.ndc.SendBinary(data); err != nil {
slog.Error("Failed to forward input message from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
}
}
return true // Continue iteration
return true
})
}
}
})
// Set the DataChannel in the incomingConns map
@@ -627,27 +633,29 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
return
}
if err = safeBRW.SendJSON(connections.NewMessageICE(
"ice-candidate",
candidate.ToJSON(),
)); err != nil {
candInit := candidate.ToJSON()
biggified := uint32(*candInit.SDPMLineIndex)
iceMsg, err := common.CreateMessage(
&gen.ProtoICE{
Candidate: &gen.RTCIceCandidateInit{
Candidate: candInit.Candidate,
SdpMLineIndex: &biggified,
SdpMid: candInit.SDPMid,
},
},
"ice-candidate", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
return
}
if err = safeBRW.SendProto(iceMsg); err != nil {
slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err)
return
}
})
pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String()))
if err != nil {
slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err)
return
}
slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String())
// Set track for Room
room.SetTrack(remoteTrack.Kind(), localTrack)
// Prepare PlayoutDelayExtension so we don't need to recreate it for each packet
playoutExt := &rtp.PlayoutDelayExtension{
MinDelay: 0,
@@ -659,6 +667,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
return
}
if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio {
room.AudioCodec = remoteTrack.Codec().RTPCodecCapability
} else if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
room.VideoCodec = remoteTrack.Codec().RTPCodecCapability
}
for {
rtpPacket, _, err := remoteTrack.ReadRTP()
if err != nil {
@@ -670,27 +684,65 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
// Use PlayoutDelayExtension for low latency, if set for this track kind
if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok {
if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil {
if err = rtpPacket.SetExtension(extID, playoutPayload); err != nil {
slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err)
continue
}
}
err = localTrack.WriteRTP(rtpPacket)
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("Failed to write RTP to local track for room", "room", room.Name, "err", err)
break
// Calculate differences
var timeDiff int64
var sequenceDiff int
if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastVideoTimestamp)
if !room.VideoTimestampSet {
timeDiff = 0
room.VideoTimestampSet = true
} else if timeDiff < -(math.MaxUint32 / 10) {
timeDiff += math.MaxUint32 + 1
}
sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastVideoSequenceNumber)
if !room.VideoSequenceSet {
sequenceDiff = 0
room.VideoSequenceSet = true
} else if sequenceDiff < -(math.MaxUint16 / 10) {
sequenceDiff += math.MaxUint16 + 1
}
room.LastVideoTimestamp = rtpPacket.Timestamp
room.LastVideoSequenceNumber = rtpPacket.SequenceNumber
} else { // Audio
timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastAudioTimestamp)
if !room.AudioTimestampSet {
timeDiff = 0
room.AudioTimestampSet = true
} else if timeDiff < -(math.MaxUint32 / 10) {
timeDiff += math.MaxUint32 + 1
}
sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastAudioSequenceNumber)
if !room.AudioSequenceSet {
sequenceDiff = 0
room.AudioSequenceSet = true
} else if sequenceDiff < -(math.MaxUint16 / 10) {
sequenceDiff += math.MaxUint16 + 1
}
room.LastAudioTimestamp = rtpPacket.Timestamp
room.LastAudioSequenceNumber = rtpPacket.SequenceNumber
}
// Broadcast with differences
room.BroadcastPacketRetimed(remoteTrack.Kind(), rtpPacket, timeDiff, sequenceDiff)
}
slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())
// Cleanup the track from the room
room.SetTrack(remoteTrack.Kind(), nil)
})
// Set the remote description
if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
if err = pc.SetRemoteDescription(offSdp); err != nil {
slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err)
continue
}
@@ -706,10 +758,20 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err)
continue
}
if err = safeBRW.SendJSON(connections.NewMessageSDP(
"answer",
answer,
)); err != nil {
answerMsg, err := common.CreateMessage(
&gen.ProtoSDP{
Sdp: &gen.RTCSessionDescriptionInit{
Sdp: answer.SDP,
Type: answer.Type.String(),
},
},
"answer", nil,
)
if err != nil {
slog.Error("Failed to create proto message", "err", err)
continue
}
if err = safeBRW.SendProto(answerMsg); err != nil {
slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err)
}
@@ -722,15 +784,16 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
}
}
}
}
// --- Public Usable Methods ---
// RequestStream sends a request to get room stream from another relay
func (sp *StreamProtocol) RequestStream(ctx context.Context, room *shared.Room, peerID peer.ID) error {
stream, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
_, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
if err != nil {
return fmt.Errorf("failed to create stream: %w", err)
}
return sp.requestStream(stream, room)
return nil /* TODO: This? */
}

2
packages/relay/internal/core/room.go
View File

@@ -45,7 +45,7 @@ func (r *Relay) DeleteRoomIfEmpty(room *shared.Room) {
if room == nil {
return
}
if room.Participants.Len() == 0 && r.LocalRooms.Has(room.ID) {
if len(room.Participants) <= 0 && r.LocalRooms.Has(room.ID) {
slog.Debug("Deleting empty room without participants", "room", room.Name)
r.LocalRooms.Delete(room.ID)
err := room.PeerConnection.Close()

54
packages/relay/internal/core/state.go
View File

@@ -5,9 +5,14 @@ import (
"encoding/json"
"errors"
"log/slog"
"relay/internal/common"
"relay/internal/shared"
"time"
gen "relay/internal/proto"
"google.golang.org/protobuf/proto"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
@@ -129,12 +134,51 @@ func (r *Relay) onPeerConnected(peerID peer.ID) {
// onPeerDisconnected marks a peer as disconnected in our status view and removes latency info
func (r *Relay) onPeerDisconnected(peerID peer.ID) {
// Check if this was a client session disconnect
if session, ok := r.ClientSessions.Get(peerID); ok {
slog.Info("Client session disconnected",
"peer", peerID,
"session", session.SessionID,
"room", session.RoomName,
"controller_slots", session.ControllerSlots)
// Send cleanup message to nestri-server if client had active controllers
if len(session.ControllerSlots) > 0 {
room := r.GetRoomByName(session.RoomName)
if room != nil && room.DataChannel != nil {
// Create disconnect notification
disconnectMsg, err := common.CreateMessage(&gen.ProtoClientDisconnected{
SessionId: session.SessionID,
ControllerSlots: session.ControllerSlots,
}, "client-disconnected", nil)
if err != nil {
slog.Error("Failed to create client disconnect message", "err", err)
}
disMarshal, err := proto.Marshal(disconnectMsg)
if err != nil {
slog.Error("Failed to marshal client disconnect message", "err", err)
} else {
if err = room.DataChannel.SendBinary(disMarshal); err != nil {
slog.Error("Failed to send client disconnect notification", "err", err)
} else {
slog.Info("Sent controller cleanup notification to nestri-server",
"session", session.SessionID,
"slots", session.ControllerSlots)
}
}
}
}
r.ClientSessions.Delete(peerID)
return
}
// Relay peer disconnect handling
slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID)
// Remove peer from local mesh peers
if r.Peers.Has(peerID) {
r.Peers.Delete(peerID)
}
// Remove any rooms associated with this peer
if r.Rooms.Has(peerID.String()) {
r.Rooms.Delete(peerID.String())
}
@@ -151,18 +195,18 @@ func (r *Relay) updateMeshRoomStates(peerID peer.ID, states []shared.RoomInfo) {
}
// If previously did not exist, but does now, request a connection if participants exist for our room
existed := r.Rooms.Has(state.ID.String())
/*existed := r.Rooms.Has(state.ID.String())
if !existed {
// Request connection to this peer if we have participants in our local room
if room, ok := r.LocalRooms.Get(state.ID); ok {
if room.Participants.Len() > 0 {
if len(room.Participants) > 0 {
slog.Debug("Got new remote room state, we locally have participants for, requesting stream", "room_name", room.Name, "peer", peerID)
if err := r.StreamProtocol.RequestStream(context.Background(), room, peerID); err != nil {
slog.Error("Failed to request stream for new remote room state", "room_name", room.Name, "peer", peerID, "err", err)
}
}
}
}
}*/
r.Rooms.Set(state.ID.String(), state)
}

463
packages/relay/internal/proto/messages.pb.go
View File

@@ -73,28 +73,50 @@ func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker {
return nil
}
type ProtoMessageInput struct {
type ProtoMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
MessageBase *ProtoMessageBase `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"`
Data *ProtoInput `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
// Types that are valid to be assigned to Payload:
//
// *ProtoMessage_MouseMove
// *ProtoMessage_MouseMoveAbs
// *ProtoMessage_MouseWheel
// *ProtoMessage_MouseKeyDown
// *ProtoMessage_MouseKeyUp
// *ProtoMessage_KeyDown
// *ProtoMessage_KeyUp
// *ProtoMessage_ControllerAttach
// *ProtoMessage_ControllerDetach
// *ProtoMessage_ControllerButton
// *ProtoMessage_ControllerTrigger
// *ProtoMessage_ControllerStick
// *ProtoMessage_ControllerAxis
// *ProtoMessage_ControllerRumble
// *ProtoMessage_Ice
// *ProtoMessage_Sdp
// *ProtoMessage_Raw
// *ProtoMessage_ClientRequestRoomStream
// *ProtoMessage_ClientDisconnected
// *ProtoMessage_ServerPushStream
Payload isProtoMessage_Payload `protobuf_oneof:"payload"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ProtoMessageInput) Reset() {
*x = ProtoMessageInput{}
func (x *ProtoMessage) Reset() {
*x = ProtoMessage{}
mi := &file_messages_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ProtoMessageInput) String() string {
func (x *ProtoMessage) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ProtoMessageInput) ProtoMessage() {}
func (*ProtoMessage) ProtoMessage() {}
func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
func (x *ProtoMessage) ProtoReflect() protoreflect.Message {
mi := &file_messages_proto_msgTypes[1]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -106,25 +128,331 @@ func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead.
func (*ProtoMessageInput) Descriptor() ([]byte, []int) {
// Deprecated: Use ProtoMessage.ProtoReflect.Descriptor instead.
func (*ProtoMessage) Descriptor() ([]byte, []int) {
return file_messages_proto_rawDescGZIP(), []int{1}
}
func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase {
func (x *ProtoMessage) GetMessageBase() *ProtoMessageBase {
if x != nil {
return x.MessageBase
}
return nil
}
func (x *ProtoMessageInput) GetData() *ProtoInput {
func (x *ProtoMessage) GetPayload() isProtoMessage_Payload {
if x != nil {
return x.Data
return x.Payload
}
return nil
}
func (x *ProtoMessage) GetMouseMove() *ProtoMouseMove {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseMove); ok {
return x.MouseMove
}
}
return nil
}
func (x *ProtoMessage) GetMouseMoveAbs() *ProtoMouseMoveAbs {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseMoveAbs); ok {
return x.MouseMoveAbs
}
}
return nil
}
func (x *ProtoMessage) GetMouseWheel() *ProtoMouseWheel {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseWheel); ok {
return x.MouseWheel
}
}
return nil
}
func (x *ProtoMessage) GetMouseKeyDown() *ProtoMouseKeyDown {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseKeyDown); ok {
return x.MouseKeyDown
}
}
return nil
}
func (x *ProtoMessage) GetMouseKeyUp() *ProtoMouseKeyUp {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_MouseKeyUp); ok {
return x.MouseKeyUp
}
}
return nil
}
func (x *ProtoMessage) GetKeyDown() *ProtoKeyDown {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_KeyDown); ok {
return x.KeyDown
}
}
return nil
}
func (x *ProtoMessage) GetKeyUp() *ProtoKeyUp {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_KeyUp); ok {
return x.KeyUp
}
}
return nil
}
func (x *ProtoMessage) GetControllerAttach() *ProtoControllerAttach {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerAttach); ok {
return x.ControllerAttach
}
}
return nil
}
func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerDetach); ok {
return x.ControllerDetach
}
}
return nil
}
func (x *ProtoMessage) GetControllerButton() *ProtoControllerButton {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerButton); ok {
return x.ControllerButton
}
}
return nil
}
func (x *ProtoMessage) GetControllerTrigger() *ProtoControllerTrigger {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerTrigger); ok {
return x.ControllerTrigger
}
}
return nil
}
func (x *ProtoMessage) GetControllerStick() *ProtoControllerStick {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerStick); ok {
return x.ControllerStick
}
}
return nil
}
func (x *ProtoMessage) GetControllerAxis() *ProtoControllerAxis {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerAxis); ok {
return x.ControllerAxis
}
}
return nil
}
func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok {
return x.ControllerRumble
}
}
return nil
}
func (x *ProtoMessage) GetIce() *ProtoICE {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Ice); ok {
return x.Ice
}
}
return nil
}
func (x *ProtoMessage) GetSdp() *ProtoSDP {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Sdp); ok {
return x.Sdp
}
}
return nil
}
func (x *ProtoMessage) GetRaw() *ProtoRaw {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_Raw); ok {
return x.Raw
}
}
return nil
}
func (x *ProtoMessage) GetClientRequestRoomStream() *ProtoClientRequestRoomStream {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ClientRequestRoomStream); ok {
return x.ClientRequestRoomStream
}
}
return nil
}
func (x *ProtoMessage) GetClientDisconnected() *ProtoClientDisconnected {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ClientDisconnected); ok {
return x.ClientDisconnected
}
}
return nil
}
func (x *ProtoMessage) GetServerPushStream() *ProtoServerPushStream {
if x != nil {
if x, ok := x.Payload.(*ProtoMessage_ServerPushStream); ok {
return x.ServerPushStream
}
}
return nil
}
type isProtoMessage_Payload interface {
isProtoMessage_Payload()
}
type ProtoMessage_MouseMove struct {
// Input types
MouseMove *ProtoMouseMove `protobuf:"bytes,2,opt,name=mouse_move,json=mouseMove,proto3,oneof"`
}
type ProtoMessage_MouseMoveAbs struct {
MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,3,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"`
}
type ProtoMessage_MouseWheel struct {
MouseWheel *ProtoMouseWheel `protobuf:"bytes,4,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"`
}
type ProtoMessage_MouseKeyDown struct {
MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,5,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"`
}
type ProtoMessage_MouseKeyUp struct {
MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,6,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"`
}
type ProtoMessage_KeyDown struct {
KeyDown *ProtoKeyDown `protobuf:"bytes,7,opt,name=key_down,json=keyDown,proto3,oneof"`
}
type ProtoMessage_KeyUp struct {
KeyUp *ProtoKeyUp `protobuf:"bytes,8,opt,name=key_up,json=keyUp,proto3,oneof"`
}
type ProtoMessage_ControllerAttach struct {
ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"`
}
type ProtoMessage_ControllerDetach struct {
ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"`
}
type ProtoMessage_ControllerButton struct {
ControllerButton *ProtoControllerButton `protobuf:"bytes,11,opt,name=controller_button,json=controllerButton,proto3,oneof"`
}
type ProtoMessage_ControllerTrigger struct {
ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,12,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"`
}
type ProtoMessage_ControllerStick struct {
ControllerStick *ProtoControllerStick `protobuf:"bytes,13,opt,name=controller_stick,json=controllerStick,proto3,oneof"`
}
type ProtoMessage_ControllerAxis struct {
ControllerAxis *ProtoControllerAxis `protobuf:"bytes,14,opt,name=controller_axis,json=controllerAxis,proto3,oneof"`
}
type ProtoMessage_ControllerRumble struct {
ControllerRumble *ProtoControllerRumble `protobuf:"bytes,15,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"`
}
type ProtoMessage_Ice struct {
// Signaling types
Ice *ProtoICE `protobuf:"bytes,20,opt,name=ice,proto3,oneof"`
}
type ProtoMessage_Sdp struct {
Sdp *ProtoSDP `protobuf:"bytes,21,opt,name=sdp,proto3,oneof"`
}
type ProtoMessage_Raw struct {
Raw *ProtoRaw `protobuf:"bytes,22,opt,name=raw,proto3,oneof"`
}
type ProtoMessage_ClientRequestRoomStream struct {
ClientRequestRoomStream *ProtoClientRequestRoomStream `protobuf:"bytes,23,opt,name=client_request_room_stream,json=clientRequestRoomStream,proto3,oneof"`
}
type ProtoMessage_ClientDisconnected struct {
ClientDisconnected *ProtoClientDisconnected `protobuf:"bytes,24,opt,name=client_disconnected,json=clientDisconnected,proto3,oneof"`
}
type ProtoMessage_ServerPushStream struct {
ServerPushStream *ProtoServerPushStream `protobuf:"bytes,25,opt,name=server_push_stream,json=serverPushStream,proto3,oneof"`
}
func (*ProtoMessage_MouseMove) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseMoveAbs) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseWheel) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseKeyDown) isProtoMessage_Payload() {}
func (*ProtoMessage_MouseKeyUp) isProtoMessage_Payload() {}
func (*ProtoMessage_KeyDown) isProtoMessage_Payload() {}
func (*ProtoMessage_KeyUp) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerButton) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerTrigger) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerStick) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerAxis) isProtoMessage_Payload() {}
func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {}
func (*ProtoMessage_Ice) isProtoMessage_Payload() {}
func (*ProtoMessage_Sdp) isProtoMessage_Payload() {}
func (*ProtoMessage_Raw) isProtoMessage_Payload() {}
func (*ProtoMessage_ClientRequestRoomStream) isProtoMessage_Payload() {}
func (*ProtoMessage_ClientDisconnected) isProtoMessage_Payload() {}
func (*ProtoMessage_ServerPushStream) isProtoMessage_Payload() {}
var File_messages_proto protoreflect.FileDescriptor
const file_messages_proto_rawDesc = "" +
@@ -132,10 +460,35 @@ const file_messages_proto_rawDesc = "" +
"\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" +
"\x10ProtoMessageBase\x12!\n" +
"\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" +
"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"v\n" +
"\x11ProtoMessageInput\x12:\n" +
"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x12%\n" +
"\x04data\x18\x02 \x01(\v2\x11.proto.ProtoInputR\x04dataB\x16Z\x14relay/internal/protob\x06proto3"
"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\xef\n" +
"\n" +
"\fProtoMessage\x12:\n" +
"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" +
"\n" +
"mouse_move\x18\x02 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" +
"\x0emouse_move_abs\x18\x03 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" +
"\vmouse_wheel\x18\x04 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" +
"mouseWheel\x12@\n" +
"\x0emouse_key_down\x18\x05 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" +
"\fmouse_key_up\x18\x06 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" +
"mouseKeyUp\x120\n" +
"\bkey_down\x18\a \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" +
"\x06key_up\x18\b \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" +
"\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" +
"\x11controller_detach\x18\n" +
" \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" +
"\x11controller_button\x18\v \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" +
"\x12controller_trigger\x18\f \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" +
"\x10controller_stick\x18\r \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" +
"\x0fcontroller_axis\x18\x0e \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" +
"\x11controller_rumble\x18\x0f \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12#\n" +
"\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" +
"\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" +
"\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" +
"\x1aclient_request_room_stream\x18\x17 \x01(\v2#.proto.ProtoClientRequestRoomStreamH\x00R\x17clientRequestRoomStream\x12Q\n" +
"\x13client_disconnected\x18\x18 \x01(\v2\x1e.proto.ProtoClientDisconnectedH\x00R\x12clientDisconnected\x12L\n" +
"\x12server_push_stream\x18\x19 \x01(\v2\x1c.proto.ProtoServerPushStreamH\x00R\x10serverPushStreamB\t\n" +
"\apayloadB\x16Z\x14relay/internal/protob\x06proto3"
var (
file_messages_proto_rawDescOnce sync.Once
@@ -152,19 +505,57 @@ func file_messages_proto_rawDescGZIP() []byte {
var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_messages_proto_goTypes = []any{
(*ProtoMessageBase)(nil), // 0: proto.ProtoMessageBase
(*ProtoMessageInput)(nil), // 1: proto.ProtoMessageInput
(*ProtoMessage)(nil), // 1: proto.ProtoMessage
(*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker
(*ProtoInput)(nil), // 3: proto.ProtoInput
(*ProtoMouseMove)(nil), // 3: proto.ProtoMouseMove
(*ProtoMouseMoveAbs)(nil), // 4: proto.ProtoMouseMoveAbs
(*ProtoMouseWheel)(nil), // 5: proto.ProtoMouseWheel
(*ProtoMouseKeyDown)(nil), // 6: proto.ProtoMouseKeyDown
(*ProtoMouseKeyUp)(nil), // 7: proto.ProtoMouseKeyUp
(*ProtoKeyDown)(nil), // 8: proto.ProtoKeyDown
(*ProtoKeyUp)(nil), // 9: proto.ProtoKeyUp
(*ProtoControllerAttach)(nil), // 10: proto.ProtoControllerAttach
(*ProtoControllerDetach)(nil), // 11: proto.ProtoControllerDetach
(*ProtoControllerButton)(nil), // 12: proto.ProtoControllerButton
(*ProtoControllerTrigger)(nil), // 13: proto.ProtoControllerTrigger
(*ProtoControllerStick)(nil), // 14: proto.ProtoControllerStick
(*ProtoControllerAxis)(nil), // 15: proto.ProtoControllerAxis
(*ProtoControllerRumble)(nil), // 16: proto.ProtoControllerRumble
(*ProtoICE)(nil), // 17: proto.ProtoICE
(*ProtoSDP)(nil), // 18: proto.ProtoSDP
(*ProtoRaw)(nil), // 19: proto.ProtoRaw
(*ProtoClientRequestRoomStream)(nil), // 20: proto.ProtoClientRequestRoomStream
(*ProtoClientDisconnected)(nil), // 21: proto.ProtoClientDisconnected
(*ProtoServerPushStream)(nil), // 22: proto.ProtoServerPushStream
}
var file_messages_proto_depIdxs = []int32{
2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker
0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase
3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
0, // 1: proto.ProtoMessage.message_base:type_name -> proto.ProtoMessageBase
3, // 2: proto.ProtoMessage.mouse_move:type_name -> proto.ProtoMouseMove
4, // 3: proto.ProtoMessage.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs
5, // 4: proto.ProtoMessage.mouse_wheel:type_name -> proto.ProtoMouseWheel
6, // 5: proto.ProtoMessage.mouse_key_down:type_name -> proto.ProtoMouseKeyDown
7, // 6: proto.ProtoMessage.mouse_key_up:type_name -> proto.ProtoMouseKeyUp
8, // 7: proto.ProtoMessage.key_down:type_name -> proto.ProtoKeyDown
9, // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp
10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach
11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach
12, // 11: proto.ProtoMessage.controller_button:type_name -> proto.ProtoControllerButton
13, // 12: proto.ProtoMessage.controller_trigger:type_name -> proto.ProtoControllerTrigger
14, // 13: proto.ProtoMessage.controller_stick:type_name -> proto.ProtoControllerStick
15, // 14: proto.ProtoMessage.controller_axis:type_name -> proto.ProtoControllerAxis
16, // 15: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble
17, // 16: proto.ProtoMessage.ice:type_name -> proto.ProtoICE
18, // 17: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP
19, // 18: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw
20, // 19: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream
21, // 20: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected
22, // 21: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream
22, // [22:22] is the sub-list for method output_type
22, // [22:22] is the sub-list for method input_type
22, // [22:22] is the sub-list for extension type_name
22, // [22:22] is the sub-list for extension extendee
0, // [0:22] is the sub-list for field type_name
}
func init() { file_messages_proto_init() }
@@ -174,6 +565,28 @@ func file_messages_proto_init() {
}
file_types_proto_init()
file_latency_tracker_proto_init()
file_messages_proto_msgTypes[1].OneofWrappers = []any{
(*ProtoMessage_MouseMove)(nil),
(*ProtoMessage_MouseMoveAbs)(nil),
(*ProtoMessage_MouseWheel)(nil),
(*ProtoMessage_MouseKeyDown)(nil),
(*ProtoMessage_MouseKeyUp)(nil),
(*ProtoMessage_KeyDown)(nil),
(*ProtoMessage_KeyUp)(nil),
(*ProtoMessage_ControllerAttach)(nil),
(*ProtoMessage_ControllerDetach)(nil),
(*ProtoMessage_ControllerButton)(nil),
(*ProtoMessage_ControllerTrigger)(nil),
(*ProtoMessage_ControllerStick)(nil),
(*ProtoMessage_ControllerAxis)(nil),
(*ProtoMessage_ControllerRumble)(nil),
(*ProtoMessage_Ice)(nil),
(*ProtoMessage_Sdp)(nil),
(*ProtoMessage_Raw)(nil),
(*ProtoMessage_ClientRequestRoomStream)(nil),
(*ProtoMessage_ClientDisconnected)(nil),
(*ProtoMessage_ServerPushStream)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{

1014
packages/relay/internal/proto/types.pb.go
View File

File diff suppressed because it is too large Load Diff

118
packages/relay/internal/shared/participant.go
View File

@@ -1,44 +1,136 @@
package shared
import (
"errors"
"fmt"
"io"
"log/slog"
"relay/internal/common"
"relay/internal/connections"
"sync"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/oklog/ulid/v2"
"github.com/pion/webrtc/v4"
)
type Participant struct {
ID ulid.ULID
SessionID string // Track session for reconnection
PeerID peer.ID // libp2p peer ID
PeerConnection *webrtc.PeerConnection
DataChannel *connections.NestriDataChannel
// Per-viewer tracks and channels
VideoTrack *webrtc.TrackLocalStaticRTP
AudioTrack *webrtc.TrackLocalStaticRTP
// Per-viewer RTP state for retiming
VideoSequenceNumber uint16
VideoTimestamp uint32
AudioSequenceNumber uint16
AudioTimestamp uint32
packetQueue chan *participantPacket
closeOnce sync.Once
}
func NewParticipant() (*Participant, error) {
func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) {
id, err := common.NewULID()
if err != nil {
return nil, fmt.Errorf("failed to create ULID for Participant: %w", err)
}
return &Participant{
p := &Participant{
ID: id,
}, nil
SessionID: sessionID,
PeerID: peerID,
VideoSequenceNumber: 0,
VideoTimestamp: 0,
AudioSequenceNumber: 0,
AudioTimestamp: 0,
packetQueue: make(chan *participantPacket, 1000),
}
func (p *Participant) addTrack(trackLocal *webrtc.TrackLocalStaticRTP) error {
rtpSender, err := p.PeerConnection.AddTrack(trackLocal)
go p.packetWriter()
return p, nil
}
// SetTrack sets audio/video track for Participant
func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) {
switch trackType {
case webrtc.RTPCodecTypeAudio:
p.AudioTrack = track
_, err := p.PeerConnection.AddTrack(track)
if err != nil {
return err
slog.Error("Failed to add Participant audio track", err)
}
case webrtc.RTPCodecTypeVideo:
p.VideoTrack = track
_, err := p.PeerConnection.AddTrack(track)
if err != nil {
slog.Error("Failed to add Participant video track", err)
}
default:
slog.Warn("Unknown track type", "participant", p.ID, "trackType", trackType)
}
}
go func() {
rtcpBuffer := make([]byte, 1400)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuffer); rtcpErr != nil {
break
// Close cleans up participant resources
func (p *Participant) Close() {
if p.DataChannel != nil {
err := p.DataChannel.Close()
if err != nil {
slog.Error("Failed to close Participant DataChannel", err)
}
p.DataChannel = nil
}
if p.PeerConnection != nil {
err := p.PeerConnection.Close()
if err != nil {
slog.Error("Failed to close Participant PeerConnection", err)
}
p.PeerConnection = nil
}
if p.VideoTrack != nil {
p.VideoTrack = nil
}
if p.AudioTrack != nil {
p.AudioTrack = nil
}
}
}()
return nil
func (p *Participant) packetWriter() {
for pkt := range p.packetQueue {
var track *webrtc.TrackLocalStaticRTP
var sequenceNumber uint16
var timestamp uint32
// No mutex needed - only this goroutine modifies these
if pkt.kind == webrtc.RTPCodecTypeAudio {
track = p.AudioTrack
p.AudioSequenceNumber = uint16(int(p.AudioSequenceNumber) + pkt.sequenceDiff)
p.AudioTimestamp = uint32(int64(p.AudioTimestamp) + pkt.timeDiff)
sequenceNumber = p.AudioSequenceNumber
timestamp = p.AudioTimestamp
} else {
track = p.VideoTrack
p.VideoSequenceNumber = uint16(int(p.VideoSequenceNumber) + pkt.sequenceDiff)
p.VideoTimestamp = uint32(int64(p.VideoTimestamp) + pkt.timeDiff)
sequenceNumber = p.VideoSequenceNumber
timestamp = p.VideoTimestamp
}
if track != nil {
pkt.packet.SequenceNumber = sequenceNumber
pkt.packet.Timestamp = timestamp
if err := track.WriteRTP(pkt.packet); err != nil && !errors.Is(err, io.ErrClosedPipe) {
slog.Error("WriteRTP failed", "participant", p.ID, "kind", pkt.kind, "err", err)
}
}
// Return packet struct to pool
participantPacketPool.Put(pkt)
}
}

147
packages/relay/internal/shared/room.go
View File

@@ -2,14 +2,29 @@ package shared
import (
"log/slog"
"relay/internal/common"
"relay/internal/connections"
"sync"
"sync/atomic"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/oklog/ulid/v2"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
)
var participantPacketPool = sync.Pool{
New: func() interface{} {
return &participantPacket{}
},
}
type participantPacket struct {
kind webrtc.RTPCodecType
packet *rtp.Packet
timeDiff int64
sequenceDiff int
}
type RoomInfo struct {
ID ulid.ULID `json:"id"`
Name string `json:"name"`
@@ -18,49 +33,139 @@ type RoomInfo struct {
type Room struct {
RoomInfo
AudioCodec webrtc.RTPCodecCapability
VideoCodec webrtc.RTPCodecCapability
PeerConnection *webrtc.PeerConnection
AudioTrack *webrtc.TrackLocalStaticRTP
VideoTrack *webrtc.TrackLocalStaticRTP
DataChannel *connections.NestriDataChannel
Participants *common.SafeMap[ulid.ULID, *Participant]
// Atomic pointer to slice of participant channels
participantChannels atomic.Pointer[[]chan<- *participantPacket]
participantsMtx sync.Mutex // Use only for add/remove
Participants map[ulid.ULID]*Participant // Keep general track of Participant(s)
// Track last seen values to calculate diffs
LastVideoTimestamp uint32
LastVideoSequenceNumber uint16
LastAudioTimestamp uint32
LastAudioSequenceNumber uint16
VideoTimestampSet bool
VideoSequenceSet bool
AudioTimestampSet bool
AudioSequenceSet bool
}
func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room {
return &Room{
r := &Room{
RoomInfo: RoomInfo{
ID: roomID,
Name: name,
OwnerID: ownerID,
},
Participants: common.NewSafeMap[ulid.ULID, *Participant](),
PeerConnection: nil,
DataChannel: nil,
Participants: make(map[ulid.ULID]*Participant),
}
emptyChannels := make([]chan<- *participantPacket, 0)
r.participantChannels.Store(&emptyChannels)
return r
}
// Close closes up Room (stream ended)
func (r *Room) Close() {
if r.DataChannel != nil {
err := r.DataChannel.Close()
if err != nil {
slog.Error("Failed to close Room DataChannel", err)
}
r.DataChannel = nil
}
if r.PeerConnection != nil {
err := r.PeerConnection.Close()
if err != nil {
slog.Error("Failed to close Room PeerConnection", err)
}
r.PeerConnection = nil
}
}
// AddParticipant adds a Participant to a Room
func (r *Room) AddParticipant(participant *Participant) {
slog.Debug("Adding participant to room", "participant", participant.ID, "room", r.Name)
r.Participants.Set(participant.ID, participant)
r.participantsMtx.Lock()
defer r.participantsMtx.Unlock()
r.Participants[participant.ID] = participant
// Update channel slice atomically
current := r.participantChannels.Load()
newChannels := make([]chan<- *participantPacket, len(*current)+1)
copy(newChannels, *current)
newChannels[len(*current)] = participant.packetQueue
r.participantChannels.Store(&newChannels)
slog.Debug("Added participant", "participant", participant.ID, "room", r.Name)
}
// Removes a Participant from a Room by participant's ID
func (r *Room) removeParticipantByID(pID ulid.ULID) {
if _, ok := r.Participants.Get(pID); ok {
r.Participants.Delete(pID)
// RemoveParticipantByID removes a Participant from a Room by participant's ID
func (r *Room) RemoveParticipantByID(pID ulid.ULID) {
r.participantsMtx.Lock()
defer r.participantsMtx.Unlock()
participant, ok := r.Participants[pID]
if !ok {
return
}
delete(r.Participants, pID)
// Update channel slice
current := r.participantChannels.Load()
newChannels := make([]chan<- *participantPacket, 0, len(*current)-1)
for _, ch := range *current {
if ch != participant.packetQueue {
newChannels = append(newChannels, ch)
}
}
// IsOnline checks if the room is online (has both audio and video tracks)
r.participantChannels.Store(&newChannels)
slog.Debug("Removed participant", "participant", pID, "room", r.Name)
}
// IsOnline checks if the room is online
func (r *Room) IsOnline() bool {
return r.AudioTrack != nil && r.VideoTrack != nil
return r.PeerConnection != nil
}
func (r *Room) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) {
switch trackType {
case webrtc.RTPCodecTypeAudio:
r.AudioTrack = track
case webrtc.RTPCodecTypeVideo:
r.VideoTrack = track
func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet, timeDiff int64, sequenceDiff int) {
// Lock-free load of channel slice
channels := r.participantChannels.Load()
// no participants..
if len(*channels) == 0 {
return
}
// Send to each participant channel (non-blocking)
for i, ch := range *channels {
// Get packet struct from pool
pp := participantPacketPool.Get().(*participantPacket)
pp.kind = kind
pp.packet = pkt.Clone()
pp.timeDiff = timeDiff
pp.sequenceDiff = sequenceDiff
select {
case ch <- pp:
// Sent successfully
default:
slog.Warn("Unknown track type", "room", r.Name, "trackType", trackType)
// Channel full, drop packet, log?
slog.Warn("Channel full, dropping packet", "channel_index", i)
participantPacketPool.Put(pp)
}
}
}

28
packages/scripts/entrypoint.sh
View File

@@ -15,13 +15,13 @@ NVIDIA_INSTALLER_DIR="/tmp"
TIMEOUT_SECONDS=10
ENTCMD_PREFIX=""
# Ensures user directory ownership
chown_user_directory() {
# Ensures user ownership across directories
handle_user_permissions() {
if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}" 2>/dev/null; then
echo "Error: Failed to change ownership of ${NESTRI_HOME} to ${NESTRI_USER}:${NESTRI_USER}" >&2
return 1
fi
# Also apply to .cache separately
# Also apply to .cache
if [[ -d "${NESTRI_HOME}/.cache" ]]; then
if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}/.cache" 2>/dev/null; then
echo "Error: Failed to change ownership of ${NESTRI_HOME}/.cache to ${NESTRI_USER}:${NESTRI_USER}" >&2
@@ -324,9 +324,23 @@ main() {
log "Skipping CAP_SYS_NICE for gamescope, capability not available"
fi
# Handle user directory permissions
log "Ensuring user directory permissions..."
chown_user_directory || exit 1
# Make sure /tmp/.X11-unix exists..
if [[ ! -d "/tmp/.X11-unix" ]]; then
log "Creating /tmp/.X11-unix directory.."
$ENTCMD_PREFIX mkdir -p /tmp/.X11-unix || {
log "Error: Failed to create /tmp/.X11-unix directory"
exit 1
}
# Set required perms..
$ENTCMD_PREFIX chmod 1777 /tmp/.X11-unix || {
log "Error: Failed to chmod /tmp/.X11-unix to 1777"
exit 1
}
fi
# Handle user permissions
log "Ensuring user permissions..."
handle_user_permissions || exit 1
# Setup namespaceless env if needed for container runtime
if [[ "$container_runtime" != "podman" ]]; then
@@ -336,7 +350,7 @@ main() {
# Make sure /run/udev/ directory exists with /run/udev/control, needed for virtual controller support
if [[ ! -d "/run/udev" || ! -e "/run/udev/control" ]]; then
log "Creating /run/udev directory and control file..."
log "Creating /run/udev directory and control file.."
$ENTCMD_PREFIX mkdir -p /run/udev || {
log "Error: Failed to create /run/udev directory"
exit 1

2
packages/scripts/entrypoint_nestri.sh
View File

@@ -187,7 +187,7 @@ start_compositor() {
if [[ -n "${NESTRI_LAUNCH_CMD}" ]]; then
log "Starting application: $NESTRI_LAUNCH_CMD"
WAYLAND_DISPLAY=wayland-0 /bin/bash -c "$NESTRI_LAUNCH_CMD" &
WAYLAND_DISPLAY="$COMPOSITOR_SOCKET" /bin/bash -c "$NESTRI_LAUNCH_CMD" &
APP_PID=$!
fi
else

129
packages/server/Cargo.lock generated
View File

@@ -181,7 +181,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"synstructure",
]
@@ -193,7 +193,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"synstructure",
]
@@ -205,7 +205,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -246,7 +246,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -257,7 +257,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -395,7 +395,7 @@ version = "0.72.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"cexpr",
"clang-sys",
"itertools 0.13.0",
@@ -406,7 +406,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -417,9 +417,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.9.4"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
[[package]]
name = "blake2"
@@ -603,9 +603,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.49"
version = "4.5.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f"
checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623"
dependencies = [
"clap_builder",
"clap_derive",
@@ -613,9 +613,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.49"
version = "4.5.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730"
checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0"
dependencies = [
"anstream",
"anstyle",
@@ -632,7 +632,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -839,7 +839,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -879,7 +879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
dependencies = [
"data-encoding",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -956,7 +956,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -1090,7 +1090,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -1262,7 +1262,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -1396,7 +1396,7 @@ version = "0.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1f2cbc4577536c849335878552f42086bfd25a8dcd6f54a18655cf818b20c8f"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"futures-channel",
"futures-core",
"futures-executor",
@@ -1421,7 +1421,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -2299,9 +2299,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.11.4"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
dependencies = [
"equivalent",
"hashbrown 0.16.0",
@@ -2368,9 +2368,9 @@ dependencies = [
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itertools"
@@ -2828,7 +2828,7 @@ checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c"
dependencies = [
"heck",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -3151,6 +3151,7 @@ dependencies = [
"tokio-stream",
"tracing",
"tracing-subscriber",
"unsigned-varint 0.8.0",
"vimputti",
"webrtc",
]
@@ -3238,7 +3239,7 @@ version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"cfg-if",
"cfg_aliases",
"libc",
@@ -3354,9 +3355,9 @@ dependencies = [
[[package]]
name = "once_cell_polyfill"
version = "1.70.1"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "opaque-debug"
@@ -3498,7 +3499,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -3603,7 +3604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -3626,9 +3627,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.101"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7"
dependencies = [
"unicode-ident",
]
@@ -3653,7 +3654,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -3676,7 +3677,7 @@ dependencies = [
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -3860,7 +3861,7 @@ version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
]
[[package]]
@@ -4037,7 +4038,7 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"errno",
"libc",
"linux-raw-sys",
@@ -4046,9 +4047,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.33"
version = "0.23.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c"
checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7"
dependencies = [
"aws-lc-rs",
"log",
@@ -4179,7 +4180,7 @@ version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"core-foundation 0.10.1",
"core-foundation-sys",
"libc",
@@ -4239,7 +4240,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4480,9 +4481,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.106"
version = "2.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
dependencies = [
"proc-macro2",
"quote",
@@ -4506,7 +4507,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4515,7 +4516,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"core-foundation 0.9.4",
"system-configuration-sys",
]
@@ -4573,7 +4574,7 @@ checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4602,7 +4603,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4613,7 +4614,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4706,7 +4707,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4816,7 +4817,7 @@ version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
"bitflags 2.9.4",
"bitflags 2.10.0",
"bytes",
"futures-util",
"http",
@@ -4860,7 +4861,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -4985,9 +4986,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
[[package]]
name = "unicode-ident"
version = "1.0.19"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
[[package]]
name = "universal-hash"
@@ -5078,9 +5079,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "vimputti"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5839a89185ccec572f746ccc02e37702cc6c0b62a6aa0d9bcd6e5921edba12"
checksum = "ffb370ee43e3ee4ca5329886e64dc5b27c83dc8cced5a63c2418777dac9a41a8"
dependencies = [
"anyhow",
"libc",
@@ -5176,7 +5177,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"wasm-bindgen-shared",
]
@@ -5211,7 +5212,7 @@ checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -5501,7 +5502,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -5512,7 +5513,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -5958,7 +5959,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"synstructure",
]
@@ -5979,7 +5980,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -5999,7 +6000,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
"synstructure",
]
@@ -6020,7 +6021,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]
[[package]]
@@ -6053,5 +6054,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.106",
"syn 2.0.108",
]

3
packages/server/Cargo.toml
View File

@@ -22,7 +22,7 @@ rand = "0.9"
rustls = { version = "0.23", features = ["ring"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
vimputti = "0.1.3"
vimputti = "0.1.4"
chrono = "0.4"
prost = "0.14"
prost-types = "0.14"
@@ -40,3 +40,4 @@ libp2p-tcp = { version = "0.44", features = ["tokio"] }
libp2p-websocket = "0.45"
dashmap = "6.1"
anyhow = "1.0"
unsigned-varint = "0.8"

8
packages/server/src/args.rs
View File

@@ -211,6 +211,14 @@ impl Args {
.value_parser(value_parser!(u32).range(1..))
.default_value("192"),
)
.arg(
Arg::new("software-render")
.long("software-render")
.env("SOFTWARE_RENDER")
.help("Use software rendering for wayland")
.value_parser(BoolishValueParser::new())
.default_value("false"),
)
.arg(
Arg::new("zero-copy")
.long("zero-copy")

8
packages/server/src/args/app_args.rs
View File

@@ -15,6 +15,9 @@ pub struct AppArgs {
/// vimputti socket path
pub vimputti_path: Option<String>,
/// Use software rendering for wayland display
pub software_render: bool,
/// Experimental zero-copy pipeline support
/// TODO: Move to video encoding flags
pub zero_copy: bool,
@@ -51,6 +54,10 @@ impl AppArgs {
vimputti_path: matches
.get_one::<String>("vimputti-path")
.map(|s| s.clone()),
software_render: matches
.get_one::<bool>("software-render")
.unwrap_or(&false)
.clone(),
zero_copy: matches
.get_one::<bool>("zero-copy")
.unwrap_or(&false)
@@ -73,6 +80,7 @@ impl AppArgs {
"> vimputti_path: '{}'",
self.vimputti_path.as_ref().map_or("None", |s| s.as_str())
);
tracing::info!("> software_render: {}", self.software_render);
tracing::info!("> zero_copy: {}", self.zero_copy);
}
}

56
packages/server/src/enc_helper.rs
View File

@@ -585,7 +585,6 @@ pub fn get_best_working_encoder(
encoders: &Vec<VideoEncoderInfo>,
codec: &Codec,
encoder_type: &EncoderType,
zero_copy: bool,
) -> Result<VideoEncoderInfo, Box<dyn Error>> {
let mut candidates = get_encoders_by_videocodec(
encoders,
@@ -601,7 +600,7 @@ pub fn get_best_working_encoder(
while !candidates.is_empty() {
let best = get_best_compatible_encoder(&candidates, codec, encoder_type)?;
tracing::info!("Testing encoder: {}", best.name,);
if test_encoder(&best, zero_copy).is_ok() {
if test_encoder(&best).is_ok() {
return Ok(best);
} else {
// Remove this encoder and try next best
@@ -613,25 +612,10 @@ pub fn get_best_working_encoder(
}
/// Test if a pipeline with the given encoder can be created and set to Playing
pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), Box<dyn Error>> {
let src = gstreamer::ElementFactory::make("waylanddisplaysrc").build()?;
if let Some(gpu_info) = &encoder.gpu_info {
src.set_property_from_str("render-node", gpu_info.render_path());
}
pub fn test_encoder(encoder: &VideoEncoderInfo) -> Result<(), Box<dyn Error>> {
let src = gstreamer::ElementFactory::make("videotestsrc").build()?;
let caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?;
let caps = gstreamer::Caps::from_str(&format!(
"{},width=1280,height=720,framerate=30/1{}",
if zero_copy {
if encoder.encoder_api == EncoderAPI::NVENC {
"video/x-raw(memory:CUDAMemory)"
} else {
"video/x-raw(memory:DMABuf)"
}
} else {
"video/x-raw"
},
if zero_copy { "" } else { ",format=RGBx" }
))?;
let caps = gstreamer::Caps::from_str("video/x-raw,width=1280,height=720,framerate=30/1")?;
caps_filter.set_property("caps", &caps);
let enc = gstreamer::ElementFactory::make(&encoder.name).build()?;
@@ -642,41 +626,9 @@ pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), B
// Create pipeline and link elements
let pipeline = gstreamer::Pipeline::new();
if zero_copy {
if encoder.encoder_api == EncoderAPI::NVENC {
// NVENC zero-copy path
pipeline.add_many(&[&src, &caps_filter, &enc, &sink])?;
gstreamer::Element::link_many(&[&src, &caps_filter, &enc, &sink])?;
} else {
// VA-API/QSV zero-copy path
let vapostproc = gstreamer::ElementFactory::make("vapostproc").build()?;
let va_caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?;
let va_caps = gstreamer::Caps::from_str("video/x-raw(memory:VAMemory),format=NV12")?;
va_caps_filter.set_property("caps", &va_caps);
pipeline.add_many(&[
&src,
&caps_filter,
&vapostproc,
&va_caps_filter,
&enc,
&sink,
])?;
gstreamer::Element::link_many(&[
&src,
&caps_filter,
&vapostproc,
&va_caps_filter,
&enc,
&sink,
])?;
}
} else {
// Non-zero-copy path for all encoders - needs videoconvert
let videoconvert = gstreamer::ElementFactory::make("videoconvert").build()?;
pipeline.add_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?;
gstreamer::Element::link_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?;
}
let bus = pipeline.bus().ok_or("Pipeline has no bus")?;
pipeline.set_state(gstreamer::State::Playing)?;

183
packages/server/src/input/controller.rs
View File

@@ -1,7 +1,5 @@
use crate::proto::proto::proto_input::InputType::{
ControllerAttach, ControllerAxis, ControllerButton, ControllerDetach, ControllerRumble,
ControllerStick, ControllerTrigger,
};
use crate::proto::proto::ProtoControllerAttach;
use crate::proto::proto::proto_message::Payload;
use anyhow::Result;
use std::collections::HashMap;
use std::sync::Arc;
@@ -48,63 +46,94 @@ impl ControllerInput {
pub struct ControllerManager {
vimputti_client: Arc<vimputti::client::VimputtiClient>,
cmd_tx: mpsc::Sender<crate::proto::proto::ProtoInput>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, // (slot, strong, weak, duration_ms)
cmd_tx: mpsc::Sender<Payload>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>, // (slot, strong, weak, duration_ms, session_id)
attach_tx: mpsc::Sender<ProtoControllerAttach>,
}
impl ControllerManager {
pub fn new(
vimputti_client: Arc<vimputti::client::VimputtiClient>,
) -> Result<(Self, mpsc::Receiver<(u32, u16, u16, u16)>)> {
let (cmd_tx, cmd_rx) = mpsc::channel(100);
let (rumble_tx, rumble_rx) = mpsc::channel(100);
) -> Result<(
Self,
mpsc::Receiver<(u32, u16, u16, u16, String)>,
mpsc::Receiver<ProtoControllerAttach>,
)> {
let (cmd_tx, cmd_rx) = mpsc::channel(512);
let (rumble_tx, rumble_rx) = mpsc::channel(256);
let (attach_tx, attach_rx) = mpsc::channel(64);
tokio::spawn(command_loop(
cmd_rx,
vimputti_client.clone(),
rumble_tx.clone(),
attach_tx.clone(),
));
Ok((
Self {
vimputti_client,
cmd_tx,
rumble_tx,
attach_tx,
},
rumble_rx,
attach_rx,
))
}
pub async fn send_command(&self, input: crate::proto::proto::ProtoInput) -> Result<()> {
self.cmd_tx.send(input).await?;
pub async fn send_command(&self, payload: Payload) -> Result<()> {
self.cmd_tx.send(payload).await?;
Ok(())
}
}
struct ControllerSlot {
controller: ControllerInput,
session_id: String,
session_slot: u32,
}
// Returns first free controller slot from 0-16
fn get_free_slot(controllers: &HashMap<u32, ControllerSlot>) -> Option<u32> {
for slot in 0..17 {
if !controllers.contains_key(&slot) {
return Some(slot);
}
}
None
}
async fn command_loop(
mut cmd_rx: mpsc::Receiver<crate::proto::proto::ProtoInput>,
mut cmd_rx: mpsc::Receiver<Payload>,
vimputti_client: Arc<vimputti::client::VimputtiClient>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>,
rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>,
attach_tx: mpsc::Sender<ProtoControllerAttach>,
) {
let mut controllers: HashMap<u32, ControllerInput> = HashMap::new();
while let Some(input) = cmd_rx.recv().await {
if let Some(input_type) = input.input_type {
match input_type {
ControllerAttach(data) => {
// Check if controller already exists in the slot, if so, ignore
if controllers.contains_key(&(data.slot as u32)) {
tracing::warn!(
"Controller slot {} already occupied, ignoring attach",
data.slot
);
} else {
let mut controllers: HashMap<u32, ControllerSlot> = HashMap::new();
while let Some(payload) = cmd_rx.recv().await {
match payload {
Payload::ControllerAttach(data) => {
let session_id = data.session_id.clone();
let session_slot = data.session_slot.clone();
// Check if this session already has a slot (reconnection)
let existing_slot = controllers
.iter()
.find(|(_, slot)| {
slot.session_id == session_id && slot.session_slot == session_slot as u32
})
.map(|(slot_num, _)| *slot_num);
let slot = existing_slot.or_else(|| get_free_slot(&controllers));
if let Some(slot) = slot {
if let Ok(mut controller) =
ControllerInput::new(data.id.clone(), &vimputti_client).await
{
let slot = data.slot as u32;
let rumble_tx = rumble_tx.clone();
let attach_tx = attach_tx.clone();
controller
.device_mut()
.on_rumble(move |strong, weak, duration_ms| {
let _ = rumble_tx.try_send((slot, strong, weak, duration_ms));
let _ = rumble_tx.try_send((slot, strong, weak, duration_ms, data.session_id.clone()));
})
.await
.map_err(|e| {
@@ -116,38 +145,68 @@ async fn command_loop(
})
.ok();
controllers.insert(data.slot as u32, controller);
tracing::info!("Controller {} attached to slot {}", data.id, data.slot);
// Return to attach_tx what slot was assigned
let attach_info = ProtoControllerAttach {
id: data.id.clone(),
session_slot: slot as i32,
session_id: session_id.clone(),
};
match attach_tx.send(attach_info).await {
Ok(_) => {
controllers.insert(
slot,
ControllerSlot {
controller,
session_id: session_id.clone(),
session_slot: session_slot.clone() as u32,
},
);
tracing::info!(
"Controller {} attached to slot {} (session: {})",
data.id,
slot,
session_id
);
}
Err(e) => {
tracing::error!(
"Failed to send attach info for slot {}: {}",
slot,
e
);
}
}
} else {
tracing::error!(
"Failed to create controller of type {} for slot {}",
data.id,
data.slot
slot
);
}
}
}
ControllerDetach(data) => {
if controllers.remove(&(data.slot as u32)).is_some() {
tracing::info!("Controller detached from slot {}", data.slot);
Payload::ControllerDetach(data) => {
if controllers.remove(&(data.session_slot as u32)).is_some() {
tracing::info!("Controller detached from slot {}", data.session_slot);
} else {
tracing::warn!("No controller found in slot {} to detach", data.slot);
tracing::warn!("No controller found in slot {} to detach", data.session_slot);
}
}
ControllerButton(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
Payload::ControllerButton(data) => {
if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) {
let device = controller.device();
let device = controller.controller.device();
device.button(button, data.pressed);
device.sync();
}
} else {
tracing::warn!("Controller slot {} not found for button event", data.slot);
tracing::warn!("Controller slot {} not found for button event", data.session_slot);
}
}
ControllerStick(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
Payload::ControllerStick(data) => {
if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
let device = controller.controller.device();
if data.stick == 0 {
// Left stick
device.axis(vimputti::Axis::LeftStickX, data.x);
@@ -161,12 +220,12 @@ async fn command_loop(
}
device.sync();
} else {
tracing::warn!("Controller slot {} not found for stick event", data.slot);
tracing::warn!("Controller slot {} not found for stick event", data.session_slot);
}
}
ControllerTrigger(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
Payload::ControllerTrigger(data) => {
if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
let device = controller.controller.device();
if data.trigger == 0 {
// Left trigger
device.axis(vimputti::Axis::LowerLeftTrigger, data.value);
@@ -176,12 +235,12 @@ async fn command_loop(
}
device.sync();
} else {
tracing::warn!("Controller slot {} not found for trigger event", data.slot);
tracing::warn!("Controller slot {} not found for trigger event", data.session_slot);
}
}
ControllerAxis(data) => {
if let Some(controller) = controllers.get(&(data.slot as u32)) {
let device = controller.device();
Payload::ControllerAxis(data) => {
if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
let device = controller.controller.device();
if data.axis == 0 {
// dpad x
device.axis(vimputti::Axis::DPadX, data.value);
@@ -192,9 +251,28 @@ async fn command_loop(
device.sync();
}
}
// Rumble will be outgoing event..
ControllerRumble(_) => {
//no-op
Payload::ClientDisconnected(data) => {
tracing::info!(
"Client disconnected, cleaning up controller slots: {:?} (client session: {})",
data.controller_slots,
data.session_id
);
// Remove all controllers for the disconnected slots
for slot in &data.controller_slots {
if controllers.remove(&(*slot as u32)).is_some() {
tracing::info!(
"Removed controller from slot {} (client session: {})",
slot,
data.session_id
);
} else {
tracing::warn!(
"No controller found in slot {} to cleanup (client session: {})",
slot,
data.session_id
);
}
}
}
_ => {
//no-op
@@ -202,4 +280,3 @@ async fn command_loop(
}
}
}
}

45
packages/server/src/main.rs
View File

@@ -3,7 +3,6 @@ mod enc_helper;
mod gpu;
mod input;
mod latency;
mod messages;
mod nestrisink;
mod p2p;
mod proto;
@@ -25,7 +24,7 @@ use tracing_subscriber::EnvFilter;
use tracing_subscriber::filter::LevelFilter;
// Handles gathering GPU information and selecting the most suitable GPU
fn handle_gpus(args: &args::Args) -> Result<Vec<gpu::GPUInfo>, Box<dyn Error>> {
fn handle_gpus(args: &args::Args) -> Result<Vec<GPUInfo>, Box<dyn Error>> {
tracing::info!("Gathering GPU information..");
let mut gpus = gpu::get_gpus()?;
if gpus.is_empty() {
@@ -120,7 +119,6 @@ fn handle_encoder_video(
&video_encoders,
&args.encoding.video.codec,
&args.encoding.video.encoder_type,
args.app.zero_copy,
)?;
}
tracing::info!("Selected video encoder: '{}'", video_encoder.name);
@@ -257,11 +255,15 @@ async fn main() -> Result<(), Box<dyn Error>> {
None
}
};
let (controller_manager, rumble_rx) = if let Some(vclient) = vimputti_client {
let (controller_manager, rumble_rx) = ControllerManager::new(vclient)?;
(Some(Arc::new(controller_manager)), Some(rumble_rx))
let (controller_manager, rumble_rx, attach_rx) = if let Some(vclient) = vimputti_client {
let (controller_manager, rumble_rx, attach_rx) = ControllerManager::new(vclient)?;
(
Some(Arc::new(controller_manager)),
Some(rumble_rx),
Some(attach_rx),
)
} else {
(None, None)
(None, None, None)
};
/*** PIPELINE CREATION ***/
@@ -320,7 +322,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
/* Video */
// Video Source Element
let video_source = Arc::new(gstreamer::ElementFactory::make("waylanddisplaysrc").build()?);
if let Some(gpu_info) = &video_encoder_info.gpu_info {
if args.app.software_render {
video_source.set_property_from_str("render-node", "software");
} else if let Some(gpu_info) = &video_encoder_info.gpu_info {
video_source.set_property_from_str("render-node", gpu_info.render_path());
}
@@ -416,6 +420,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
video_source.clone(),
controller_manager,
rumble_rx,
attach_rx,
)
.await?;
let webrtcsink = BaseWebRTCSink::with_signaller(Signallable::from(signaller.clone()));
@@ -424,20 +429,16 @@ async fn main() -> Result<(), Box<dyn Error>> {
webrtcsink.set_property("do-retransmission", false);
/* Queues */
let video_source_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 5u32)
.build()?;
let audio_source_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 5u32)
.build()?;
let video_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 5u32)
.property("max-size-buffers", 2u32)
.property("max-size-time", 0u64)
.property("max-size-bytes", 0u32)
.build()?;
let audio_queue = gstreamer::ElementFactory::make("queue")
.property("max-size-buffers", 5u32)
.property("max-size-buffers", 2u32)
.property("max-size-time", 0u64)
.property("max-size-bytes", 0u32)
.build()?;
/* Clock Sync */
@@ -456,7 +457,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
&caps_filter,
&video_queue,
&video_clocksync,
&video_source_queue,
&video_source,
&audio_encoder,
&audio_capsfilter,
@@ -464,7 +464,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
&audio_clocksync,
&audio_rate,
&audio_converter,
&audio_source_queue,
&audio_source,
])?;
@@ -491,7 +490,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
// Link main audio branch
gstreamer::Element::link_many(&[
&audio_source,
&audio_source_queue,
&audio_converter,
&audio_rate,
&audio_capsfilter,
@@ -513,7 +511,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
if let (Some(vapostproc), Some(va_caps_filter)) = (&vapostproc, &va_caps_filter) {
gstreamer::Element::link_many(&[
&video_source,
&video_source_queue,
&caps_filter,
&video_queue,
&video_clocksync,
@@ -525,7 +522,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
// NVENC pipeline
gstreamer::Element::link_many(&[
&video_source,
&video_source_queue,
&caps_filter,
&video_encoder,
])?;
@@ -533,7 +529,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
} else {
gstreamer::Element::link_many(&[
&video_source,
&video_source_queue,
&caps_filter,
&video_queue,
&video_clocksync,
@@ -550,7 +545,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
}
// Make sure QOS is disabled to avoid latency
video_encoder.set_property("qos", false);
video_encoder.set_property("qos", true);
// Optimize latency of pipeline
video_source

50
packages/server/src/messages.rs
View File

@@ -1,50 +0,0 @@
use crate::latency::LatencyTracker;
use serde::{Deserialize, Serialize};
use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageBase {
pub payload_type: String,
pub latency: Option<LatencyTracker>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageRaw {
#[serde(flatten)]
pub base: MessageBase,
pub data: serde_json::Value,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageLog {
#[serde(flatten)]
pub base: MessageBase,
pub level: String,
pub message: String,
pub time: String,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageMetrics {
#[serde(flatten)]
pub base: MessageBase,
pub usage_cpu: f64,
pub usage_memory: f64,
pub uptime: u64,
pub pipeline_latency: f64,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageICE {
#[serde(flatten)]
pub base: MessageBase,
pub candidate: RTCIceCandidateInit,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MessageSDP {
#[serde(flatten)]
pub base: MessageBase,
pub sdp: RTCSessionDescription,
}

261
packages/server/src/nestrisink/imp.rs
View File

@@ -1,11 +1,11 @@
use crate::input::controller::ControllerManager;
use crate::messages::{MessageBase, MessageICE, MessageRaw, MessageSDP};
use crate::p2p::p2p::NestriConnection;
use crate::p2p::p2p_protocol_stream::NestriStreamProtocol;
use crate::proto::proto::proto_input::InputType::{
KeyDown, KeyUp, MouseKeyDown, MouseKeyUp, MouseMove, MouseMoveAbs, MouseWheel,
use crate::proto::proto::proto_message::Payload;
use crate::proto::proto::{
ProtoControllerAttach, ProtoControllerRumble, ProtoIce, ProtoMessage, ProtoSdp,
ProtoServerPushStream, RtcIceCandidateInit, RtcSessionDescriptionInit,
};
use crate::proto::proto::{ProtoInput, ProtoMessageInput};
use anyhow::Result;
use glib::subclass::prelude::*;
use gstreamer::glib;
@@ -16,8 +16,6 @@ use parking_lot::RwLock as PLRwLock;
use prost::Message;
use std::sync::{Arc, LazyLock};
use tokio::sync::{Mutex, mpsc};
use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
pub struct Signaller {
stream_room: PLRwLock<Option<String>>,
@@ -25,7 +23,8 @@ pub struct Signaller {
wayland_src: PLRwLock<Option<Arc<gstreamer::Element>>>,
data_channel: PLRwLock<Option<Arc<gstreamer_webrtc::WebRTCDataChannel>>>,
controller_manager: PLRwLock<Option<Arc<ControllerManager>>>,
rumble_rx: Mutex<Option<mpsc::Receiver<(u32, u16, u16, u16)>>>,
rumble_rx: Mutex<Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>>,
attach_rx: Mutex<Option<mpsc::Receiver<ProtoControllerAttach>>>,
}
impl Default for Signaller {
fn default() -> Self {
@@ -36,6 +35,7 @@ impl Default for Signaller {
data_channel: PLRwLock::new(None),
controller_manager: PLRwLock::new(None),
rumble_rx: Mutex::new(None),
attach_rx: Mutex::new(None),
}
}
}
@@ -70,15 +70,27 @@ impl Signaller {
self.controller_manager.read().clone()
}
pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16)>) {
pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16, String)>) {
*self.rumble_rx.lock().await = Some(rumble_rx);
}
// Change getter to take ownership:
pub async fn take_rumble_rx(&self) -> Option<mpsc::Receiver<(u32, u16, u16, u16)>> {
pub async fn take_rumble_rx(&self) -> Option<mpsc::Receiver<(u32, u16, u16, u16, String)>> {
self.rumble_rx.lock().await.take()
}
pub async fn set_attach_rx(
&self,
attach_rx: mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>,
) {
*self.attach_rx.lock().await = Some(attach_rx);
}
pub async fn take_attach_rx(
&self,
) -> Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>> {
self.attach_rx.lock().await.take()
}
pub fn set_data_channel(&self, data_channel: gstreamer_webrtc::WebRTCDataChannel) {
*self.data_channel.write() = Some(Arc::new(data_channel));
}
@@ -95,59 +107,70 @@ impl Signaller {
};
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("answer", move |data| {
if let Ok(message) = serde_json::from_slice::<MessageSDP>(&data) {
let sdp = gst_sdp::SDPMessage::parse_buffer(message.sdp.sdp.as_bytes())
.map_err(|e| anyhow::anyhow!("Invalid SDP in 'answer': {e:?}"))?;
let answer = WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
Ok(self_obj.emit_by_name::<()>(
stream_protocol.register_callback("answer", move |msg| {
if let Some(payload) = msg.payload {
match payload {
Payload::Sdp(sdp) => {
if let Some(sdp) = sdp.sdp {
let sdp = gst_sdp::SDPMessage::parse_buffer(sdp.sdp.as_bytes())
.map_err(|e| {
anyhow::anyhow!("Invalid SDP in 'answer': {e:?}")
})?;
let answer =
WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
return Ok(self_obj.emit_by_name::<()>(
"session-description",
&[&"unique-session-id", &answer],
))
} else {
anyhow::bail!("Failed to decode SDP message");
));
}
}
_ => {
tracing::warn!("Unexpected payload type for answer");
return Ok(());
}
}
} else {
anyhow::bail!("Failed to decode answer message");
}
Ok(())
});
}
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("ice-candidate", move |data| {
if let Ok(message) = serde_json::from_slice::<MessageICE>(&data) {
let candidate = message.candidate;
let sdp_m_line_index = candidate.sdp_mline_index.unwrap_or(0) as u32;
let sdp_mid = candidate.sdp_mid;
Ok(self_obj.emit_by_name::<()>(
stream_protocol.register_callback("ice-candidate", move |msg| {
if let Some(payload) = msg.payload {
match payload {
Payload::Ice(ice) => {
if let Some(candidate) = ice.candidate {
let sdp_m_line_index = candidate.sdp_m_line_index.unwrap_or(0);
return Ok(self_obj.emit_by_name::<()>(
"handle-ice",
&[
&"unique-session-id",
&sdp_m_line_index,
&sdp_mid,
&candidate.sdp_mid,
&candidate.candidate,
],
))
));
}
}
_ => {
tracing::warn!("Unexpected payload type for ice-candidate");
return Ok(());
}
}
} else {
anyhow::bail!("Failed to decode ICE message");
}
Ok(())
});
}
{
let self_obj = self.obj().clone();
stream_protocol.register_callback("push-stream-ok", move |data| {
if let Ok(answer) = serde_json::from_slice::<MessageRaw>(&data) {
// Decode room name string
if let Some(room_name) = answer.data.as_str() {
gstreamer::info!(
gstreamer::CAT_DEFAULT,
"Received OK answer for room: {}",
room_name
);
} else {
gstreamer::error!(
gstreamer::CAT_DEFAULT,
"Failed to decode room name from answer"
);
}
stream_protocol.register_callback("push-stream-ok", move |msg| {
if let Some(payload) = msg.payload {
return match payload {
Payload::ServerPushStream(_res) => {
// Send our SDP offer
Ok(self_obj.emit_by_name::<()>(
"session-requested",
@@ -157,6 +180,12 @@ impl Signaller {
&None::<WebRTCSessionDescription>,
],
))
}
_ => {
tracing::warn!("Unexpected payload type for push-stream-ok");
Ok(())
}
};
} else {
anyhow::bail!("Failed to decode answer");
}
@@ -200,12 +229,14 @@ impl Signaller {
// Spawn async task to take the receiver and set up
tokio::spawn(async move {
let rumble_rx = signaller.imp().take_rumble_rx().await;
let attach_rx = signaller.imp().take_attach_rx().await;
let controller_manager =
signaller.imp().get_controller_manager();
setup_data_channel(
controller_manager,
rumble_rx,
attach_rx,
data_channel,
&wayland_src,
);
@@ -243,19 +274,18 @@ impl SignallableImpl for Signaller {
return;
};
let push_msg = MessageRaw {
base: MessageBase {
payload_type: "push-stream-room".to_string(),
latency: None,
},
data: serde_json::Value::from(stream_room),
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
let push_msg = crate::proto::create_message(
Payload::ServerPushStream(ProtoServerPushStream {
room_name: stream_room,
}),
"push-stream-room",
None,
);
if let Err(e) = stream_protocol.send_message(&push_msg) {
tracing::error!("Failed to send push stream room message: {:?}", e);
}
@@ -266,20 +296,22 @@ impl SignallableImpl for Signaller {
}
fn send_sdp(&self, _session_id: &str, sdp: &WebRTCSessionDescription) {
let sdp_message = MessageSDP {
base: MessageBase {
payload_type: "offer".to_string(),
latency: None,
},
sdp: RTCSessionDescription::offer(sdp.sdp().as_text().unwrap()).unwrap(),
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
if let Err(e) = stream_protocol.send_message(&sdp_message) {
let sdp_msg = crate::proto::create_message(
Payload::Sdp(ProtoSdp {
sdp: Some(RtcSessionDescriptionInit {
sdp: sdp.sdp().as_text().unwrap(),
r#type: "offer".to_string(),
}),
}),
"offer",
None,
);
if let Err(e) = stream_protocol.send_message(&sdp_msg) {
tracing::error!("Failed to send SDP message: {:?}", e);
}
}
@@ -291,26 +323,25 @@ impl SignallableImpl for Signaller {
sdp_m_line_index: u32,
sdp_mid: Option<String>,
) {
let candidate_init = RTCIceCandidateInit {
candidate: candidate.to_string(),
sdp_mid,
sdp_mline_index: Some(sdp_m_line_index as u16),
..Default::default()
};
let ice_message = MessageICE {
base: MessageBase {
payload_type: "ice-candidate".to_string(),
latency: None,
},
candidate: candidate_init,
};
let Some(stream_protocol) = self.get_stream_protocol() else {
gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
return;
};
if let Err(e) = stream_protocol.send_message(&ice_message) {
let candidate_init = RtcIceCandidateInit {
candidate: candidate.to_string(),
sdp_mid,
sdp_m_line_index: Some(sdp_m_line_index),
..Default::default() //username_fragment: Some(session_id.to_string()), TODO: required?
};
let ice_msg = crate::proto::create_message(
Payload::Ice(ProtoIce {
candidate: Some(candidate_init),
}),
"ice-candidate",
None,
);
if let Err(e) = stream_protocol.send_message(&ice_msg) {
tracing::error!("Failed to send ICE candidate message: {:?}", e);
}
}
@@ -351,7 +382,8 @@ impl ObjectImpl for Signaller {
fn setup_data_channel(
controller_manager: Option<Arc<ControllerManager>>,
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>, // (slot, strong, weak, duration_ms)
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>, // (slot, strong, weak, duration_ms, session_id)
attach_rx: Option<mpsc::Receiver<ProtoControllerAttach>>,
data_channel: Arc<gstreamer_webrtc::WebRTCDataChannel>,
wayland_src: &gstreamer::Element,
) {
@@ -361,11 +393,11 @@ fn setup_data_channel(
// Spawn async processor
tokio::spawn(async move {
while let Some(data) = rx.recv().await {
match ProtoMessageInput::decode(data.as_slice()) {
Ok(message_input) => {
if let Some(message_base) = message_input.message_base {
match ProtoMessage::decode(data.as_slice()) {
Ok(msg_wrapper) => {
if let Some(message_base) = msg_wrapper.message_base {
if message_base.payload_type == "input" {
if let Some(input_data) = message_input.data {
if let Some(input_data) = msg_wrapper.payload {
if let Some(event) = handle_input_message(input_data) {
// Send the event to wayland source, result bool is ignored
let _ = wayland_src.send_event(event);
@@ -373,7 +405,7 @@ fn setup_data_channel(
}
} else if message_base.payload_type == "controllerInput" {
if let Some(controller_manager) = &controller_manager {
if let Some(input_data) = message_input.data {
if let Some(input_data) = msg_wrapper.payload {
let _ = controller_manager.send_command(input_data).await;
}
}
@@ -391,26 +423,18 @@ fn setup_data_channel(
if let Some(mut rumble_rx) = rumble_rx {
let data_channel_clone = data_channel.clone();
tokio::spawn(async move {
while let Some((slot, strong, weak, duration_ms)) = rumble_rx.recv().await {
let rumble_msg = ProtoMessageInput {
message_base: Some(crate::proto::proto::ProtoMessageBase {
payload_type: "controllerInput".to_string(),
latency: None,
}),
data: Some(ProtoInput {
input_type: Some(
crate::proto::proto::proto_input::InputType::ControllerRumble(
crate::proto::proto::ProtoControllerRumble {
r#type: "ControllerRumble".to_string(),
slot: slot as i32,
while let Some((slot, strong, weak, duration_ms, session_id)) = rumble_rx.recv().await {
let rumble_msg = crate::proto::create_message(
Payload::ControllerRumble(ProtoControllerRumble {
session_slot: slot as i32,
session_id: session_id,
low_frequency: weak as i32,
high_frequency: strong as i32,
duration: duration_ms as i32,
},
),
),
}),
};
"controllerInput",
None,
);
let data = rumble_msg.encode_to_vec();
let bytes = glib::Bytes::from_owned(data);
@@ -422,6 +446,27 @@ fn setup_data_channel(
});
}
// Spawn attach sender
if let Some(mut attach_rx) = attach_rx {
let data_channel_clone = data_channel.clone();
tokio::spawn(async move {
while let Some(attach_msg) = attach_rx.recv().await {
let proto_msg = crate::proto::create_message(
Payload::ControllerAttach(attach_msg),
"controllerInput",
None,
);
let data = proto_msg.encode_to_vec();
let bytes = glib::Bytes::from_owned(data);
if let Err(e) = data_channel_clone.send_data_full(Some(&bytes)) {
tracing::warn!("Failed to send controller attach data: {}", e);
}
}
});
}
data_channel.connect_on_message_data(move |_data_channel, data| {
if let Some(data) = data {
let _ = tx.send(data.to_vec());
@@ -429,10 +474,9 @@ fn setup_data_channel(
});
}
fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
if let Some(input_type) = input_msg.input_type {
match input_type {
MouseMove(data) => {
fn handle_input_message(payload: Payload) -> Option<gstreamer::Event> {
match payload {
Payload::MouseMove(data) => {
let structure = gstreamer::Structure::builder("MouseMoveRelative")
.field("pointer_x", data.x as f64)
.field("pointer_y", data.y as f64)
@@ -440,7 +484,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseMoveAbs(data) => {
Payload::MouseMoveAbs(data) => {
let structure = gstreamer::Structure::builder("MouseMoveAbsolute")
.field("pointer_x", data.x as f64)
.field("pointer_y", data.y as f64)
@@ -448,7 +492,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
KeyDown(data) => {
Payload::KeyDown(data) => {
let structure = gstreamer::Structure::builder("KeyboardKey")
.field("key", data.key as u32)
.field("pressed", true)
@@ -456,7 +500,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
KeyUp(data) => {
Payload::KeyUp(data) => {
let structure = gstreamer::Structure::builder("KeyboardKey")
.field("key", data.key as u32)
.field("pressed", false)
@@ -464,7 +508,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseWheel(data) => {
Payload::MouseWheel(data) => {
let structure = gstreamer::Structure::builder("MouseAxis")
.field("x", data.x as f64)
.field("y", data.y as f64)
@@ -472,7 +516,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseKeyDown(data) => {
Payload::MouseKeyDown(data) => {
let structure = gstreamer::Structure::builder("MouseButton")
.field("button", data.key as u32)
.field("pressed", true)
@@ -480,7 +524,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
Some(gstreamer::event::CustomUpstream::new(structure))
}
MouseKeyUp(data) => {
Payload::MouseKeyUp(data) => {
let structure = gstreamer::Structure::builder("MouseButton")
.field("button", data.key as u32)
.field("pressed", false)
@@ -490,7 +534,4 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
}
_ => None,
}
} else {
None
}
}

6
packages/server/src/nestrisink/mod.rs
View File

@@ -18,7 +18,8 @@ impl NestriSignaller {
nestri_conn: NestriConnection,
wayland_src: Arc<gstreamer::Element>,
controller_manager: Option<Arc<ControllerManager>>,
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>,
rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>,
attach_rx: Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>>,
) -> Result<Self, Box<dyn std::error::Error>> {
let obj: Self = glib::Object::new();
obj.imp().set_stream_room(room);
@@ -30,6 +31,9 @@ impl NestriSignaller {
if let Some(rumble_rx) = rumble_rx {
obj.imp().set_rumble_rx(rumble_rx).await;
}
if let Some(attach_rx) = attach_rx {
obj.imp().set_attach_rx(attach_rx).await;
}
Ok(obj)
}
}

29
packages/server/src/p2p/p2p_protocol_stream.rs
View File

@@ -3,21 +3,22 @@ use crate::p2p::p2p_safestream::SafeStream;
use anyhow::Result;
use dashmap::DashMap;
use libp2p::StreamProtocol;
use prost::Message;
use std::sync::Arc;
use tokio::sync::mpsc;
// Cloneable callback type
pub type CallbackInner = dyn Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static;
pub type CallbackInner = dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static;
pub struct Callback(Arc<CallbackInner>);
impl Callback {
pub fn new<F>(f: F) -> Self
where
F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
{
Callback(Arc::new(f))
}
pub fn call(&self, data: Vec<u8>) -> Result<()> {
pub fn call(&self, data: crate::proto::proto::ProtoMessage) -> Result<()> {
self.0(data)
}
}
@@ -104,15 +105,17 @@ impl NestriStreamProtocol {
}
};
match serde_json::from_slice::<crate::messages::MessageBase>(&data) {
Ok(base_message) => {
let response_type = base_message.payload_type;
match crate::proto::proto::ProtoMessage::decode(data.as_slice()) {
Ok(message) => {
if let Some(base_message) = &message.message_base {
let response_type = &base_message.payload_type;
let response_type = response_type.clone();
// With DashMap, we don't need explicit locking
// we just get the callback directly if it exists
if let Some(callback) = callbacks.get(&response_type) {
// Execute the callback
if let Err(e) = callback.call(data.clone()) {
if let Err(e) = callback.call(message) {
tracing::error!(
"Callback for response type '{}' errored: {:?}",
response_type,
@@ -125,6 +128,9 @@ impl NestriStreamProtocol {
response_type
);
}
} else {
tracing::error!("No base message in decoded protobuf message",);
}
}
Err(e) => {
tracing::error!("Failed to decode message: {}", e);
@@ -154,8 +160,9 @@ impl NestriStreamProtocol {
})
}
pub fn send_message<M: serde::Serialize>(&self, message: &M) -> Result<()> {
let json_data = serde_json::to_vec(message)?;
pub fn send_message(&self, message: &crate::proto::proto::ProtoMessage) -> Result<()> {
let mut buf = Vec::new();
message.encode(&mut buf)?;
let Some(tx) = &self.tx else {
return Err(anyhow::Error::msg(
if self.read_handle.is_none() && self.write_handle.is_none() {
@@ -165,13 +172,13 @@ impl NestriStreamProtocol {
},
));
};
tx.try_send(json_data)?;
tx.try_send(buf)?;
Ok(())
}
pub fn register_callback<F>(&self, response_type: &str, callback: F)
where
F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
{
self.callbacks
.insert(response_type.to_string(), Callback::new(callback));

52
packages/server/src/p2p/p2p_safestream.rs
View File

@@ -1,11 +1,9 @@
use anyhow::Result;
use byteorder::{BigEndian, ByteOrder};
use libp2p::futures::io::{ReadHalf, WriteHalf};
use libp2p::futures::{AsyncReadExt, AsyncWriteExt};
use std::sync::Arc;
use tokio::sync::Mutex;
const MAX_SIZE: usize = 1024 * 1024; // 1MB
use unsigned_varint::{decode, encode};
pub struct SafeStream {
stream_read: Arc<Mutex<ReadHalf<libp2p::Stream>>>,
@@ -29,34 +27,52 @@ impl SafeStream {
}
async fn send_with_length_prefix(&self, data: &[u8]) -> Result<()> {
if data.len() > MAX_SIZE {
anyhow::bail!("Data exceeds maximum size");
}
let mut buffer = Vec::with_capacity(4 + data.len());
buffer.extend_from_slice(&(data.len() as u32).to_be_bytes()); // Length prefix
buffer.extend_from_slice(data); // Payload
let mut stream_write = self.stream_write.lock().await;
stream_write.write_all(&buffer).await?; // Single write
// Encode length as varint
let mut length_buf = encode::usize_buffer();
let length_bytes = encode::usize(data.len(), &mut length_buf);
// Write varint length prefix
stream_write.write_all(length_bytes).await?;
// Write payload
stream_write.write_all(data).await?;
stream_write.flush().await?;
Ok(())
}
async fn receive_with_length_prefix(&self) -> Result<Vec<u8>> {
let mut stream_read = self.stream_read.lock().await;
// Read length prefix + data in one syscall
let mut length_prefix = [0u8; 4];
stream_read.read_exact(&mut length_prefix).await?;
let length = BigEndian::read_u32(&length_prefix) as usize;
// Read varint length prefix (up to 10 bytes for u64)
let mut length_buf = Vec::new();
let mut temp_byte = [0u8; 1];
if length > MAX_SIZE {
anyhow::bail!("Received data exceeds maximum size");
loop {
stream_read.read_exact(&mut temp_byte).await?;
length_buf.push(temp_byte[0]);
// Check if this is the last byte (MSB = 0)
if temp_byte[0] & 0x80 == 0 {
break;
}
// Protect against malicious infinite varints
if length_buf.len() > 10 {
anyhow::bail!("Invalid varint encoding");
}
}
// Decode the varint
let (length, _) = decode::usize(&length_buf)
.map_err(|e| anyhow::anyhow!("Failed to decode varint: {}", e))?;
// Read payload
let mut buffer = vec![0u8; length];
stream_read.read_exact(&mut buffer).await?;
Ok(buffer)
}
}

34
packages/server/src/proto.rs
View File

@@ -1 +1,35 @@
pub mod proto;
pub struct CreateMessageOptions {
pub sequence_id: Option<String>,
pub latency: Option<proto::ProtoLatencyTracker>,
}
pub fn create_message(
payload: proto::proto_message::Payload,
payload_type: impl Into<String>,
options: Option<CreateMessageOptions>,
) -> proto::ProtoMessage {
let opts = options.unwrap_or(CreateMessageOptions {
sequence_id: None,
latency: None,
});
let latency = opts.latency.or_else(|| {
opts.sequence_id.map(|seq_id| proto::ProtoLatencyTracker {
sequence_id: seq_id,
timestamps: vec![proto::ProtoTimestampEntry {
stage: "created".to_string(),
time: Some(prost_types::Timestamp::from(std::time::SystemTime::now())),
}],
})
});
proto::ProtoMessage {
message_base: Some(proto::ProtoMessageBase {
payload_type: payload_type.into(),
latency,
}),
payload: Some(payload),
}
}

202
packages/server/src/proto/gen.rs
View File

@@ -1,202 +0,0 @@
// @generated
// This file is @generated by prost-build.
/// EntityState represents the state of an entity in the mesh (e.g., a room).
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EntityState {
/// Type of entity (e.g., "room")
#[prost(string, tag="1")]
pub entity_type: ::prost::alloc::string::String,
/// Unique identifier (e.g., room name)
#[prost(string, tag="2")]
pub entity_id: ::prost::alloc::string::String,
/// Whether the entity is active
#[prost(bool, tag="3")]
pub active: bool,
/// Relay ID that owns this entity
#[prost(string, tag="4")]
pub owner_relay_id: ::prost::alloc::string::String,
}
/// MeshMessage is the top-level message for all relay-to-relay communication.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MeshMessage {
#[prost(oneof="mesh_message::Type", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13")]
pub r#type: ::core::option::Option<mesh_message::Type>,
}
/// Nested message and enum types in `MeshMessage`.
pub mod mesh_message {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Type {
/// Level 0
#[prost(message, tag="1")]
StateUpdate(super::StateUpdate),
#[prost(message, tag="2")]
Ack(super::Ack),
#[prost(message, tag="3")]
RetransmissionRequest(super::RetransmissionRequest),
#[prost(message, tag="4")]
Retransmission(super::Retransmission),
#[prost(message, tag="5")]
Heartbeat(super::Heartbeat),
#[prost(message, tag="6")]
SuspectRelay(super::SuspectRelay),
#[prost(message, tag="7")]
Disconnect(super::Disconnect),
/// Level 1
#[prost(message, tag="8")]
ForwardSdp(super::ForwardSdp),
#[prost(message, tag="9")]
ForwardIce(super::ForwardIce),
#[prost(message, tag="10")]
ForwardIngest(super::ForwardIngest),
#[prost(message, tag="11")]
StreamRequest(super::StreamRequest),
/// Level 2
#[prost(message, tag="12")]
Handshake(super::Handshake),
#[prost(message, tag="13")]
HandshakeResponse(super::HandshakeResponse),
}
}
/// Handshake to inititiate new connection to mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Handshake {
/// UUID of the relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// base64 encoded Diffie-Hellman public key
#[prost(string, tag="2")]
pub dh_public_key: ::prost::alloc::string::String,
}
/// HandshakeResponse to respond to a mesh joiner.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct HandshakeResponse {
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub dh_public_key: ::prost::alloc::string::String,
/// relay id to signature
#[prost(map="string, string", tag="3")]
pub approvals: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
/// Forwarded SDP from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardSdp {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub participant_id: ::prost::alloc::string::String,
#[prost(string, tag="3")]
pub sdp: ::prost::alloc::string::String,
/// "offer" or "answer"
#[prost(string, tag="4")]
pub r#type: ::prost::alloc::string::String,
}
/// Forwarded ICE candidate from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardIce {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub participant_id: ::prost::alloc::string::String,
#[prost(string, tag="3")]
pub candidate: ::prost::alloc::string::String,
}
/// Forwarded ingest room from another relay.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ForwardIngest {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
/// Stream request from mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StreamRequest {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
/// StateUpdate propagates entity state changes across the mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StateUpdate {
/// Unique sequence number for this update
#[prost(uint64, tag="1")]
pub sequence_number: u64,
/// Key: entity_id (e.g., room name), Value: EntityState
#[prost(map="string, message", tag="2")]
pub entities: ::std::collections::HashMap<::prost::alloc::string::String, EntityState>,
}
/// Ack acknowledges receipt of a StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Ack {
/// UUID of the acknowledging relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Sequence number being acknowledged
#[prost(uint64, tag="2")]
pub sequence_number: u64,
}
/// RetransmissionRequest requests a missed StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct RetransmissionRequest {
/// UUID of the requesting relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Sequence number of the missed update
#[prost(uint64, tag="2")]
pub sequence_number: u64,
}
/// Retransmission resends a StateUpdate.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Retransmission {
/// UUID of the sending relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// The retransmitted update
#[prost(message, optional, tag="2")]
pub state_update: ::core::option::Option<StateUpdate>,
}
/// Heartbeat signals relay liveness.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Heartbeat {
/// UUID of the sending relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Time of the heartbeat
#[prost(message, optional, tag="2")]
pub timestamp: ::core::option::Option<::prost_types::Timestamp>,
}
/// SuspectRelay marks a relay as potentially unresponsive.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SuspectRelay {
/// UUID of the suspected relay
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Reason for suspicion (e.g., "no heartbeat")
#[prost(string, tag="2")]
pub reason: ::prost::alloc::string::String,
}
/// Disconnect signals to remove a relay from the mesh.
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Disconnect {
/// UUID of the relay to disconnect
#[prost(string, tag="1")]
pub relay_id: ::prost::alloc::string::String,
/// Reason for disconnection (e.g., "unresponsive")
#[prost(string, tag="2")]
pub reason: ::prost::alloc::string::String,
}
// @@protoc_insertion_point(module)

294
packages/server/src/proto/proto.rs
View File

@@ -20,80 +20,59 @@ pub struct ProtoLatencyTracker {
/// MouseMove message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseMove {
/// Fixed value "MouseMove"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseMoveAbs message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseMoveAbs {
/// Fixed value "MouseMoveAbs"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseWheel message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseWheel {
/// Fixed value "MouseWheel"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub x: i32,
#[prost(int32, tag="3")]
#[prost(int32, tag="2")]
pub y: i32,
}
/// MouseKeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseKeyDown {
/// Fixed value "MouseKeyDown"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
/// MouseKeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoMouseKeyUp {
/// Fixed value "MouseKeyUp"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
// Keyboard messages
/// KeyDown message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoKeyDown {
/// Fixed value "KeyDown"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
/// KeyUp message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
#[derive(Clone, Copy, PartialEq, ::prost::Message)]
pub struct ProtoKeyUp {
/// Fixed value "KeyUp"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
#[prost(int32, tag="2")]
#[prost(int32, tag="1")]
pub key: i32,
}
// Controller messages
@@ -102,37 +81,37 @@ pub struct ProtoKeyUp {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerAttach {
/// Fixed value "ControllerAttach"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// One of the following enums: "ps", "xbox" or "switch"
#[prost(string, tag="2")]
#[prost(string, tag="1")]
pub id: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="3")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="2")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="3")]
pub session_id: ::prost::alloc::string::String,
}
/// ControllerDetach message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerDetach {
/// Fixed value "ControllerDetach"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
}
/// ControllerButton message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerButton {
/// Fixed value "ControllerButtons"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
/// Button code (linux input event code)
#[prost(int32, tag="3")]
pub button: i32,
@@ -144,12 +123,12 @@ pub struct ProtoControllerButton {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerTrigger {
/// Fixed value "ControllerTriggers"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
/// Trigger number (0 for left, 1 for right)
#[prost(int32, tag="3")]
pub trigger: i32,
@@ -161,12 +140,12 @@ pub struct ProtoControllerTrigger {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerStick {
/// Fixed value "ControllerStick"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
/// Stick number (0 for left, 1 for right)
#[prost(int32, tag="3")]
pub stick: i32,
@@ -181,12 +160,12 @@ pub struct ProtoControllerStick {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerAxis {
/// Fixed value "ControllerAxis"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
/// Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
#[prost(int32, tag="3")]
pub axis: i32,
@@ -198,12 +177,12 @@ pub struct ProtoControllerAxis {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoControllerRumble {
/// Fixed value "ControllerRumble"
#[prost(string, tag="1")]
pub r#type: ::prost::alloc::string::String,
/// Slot number (0-3)
#[prost(int32, tag="2")]
pub slot: i32,
/// Session specific slot number (0-3)
#[prost(int32, tag="1")]
pub session_slot: i32,
/// Session ID of the client
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
/// Low frequency rumble (0-65535)
#[prost(int32, tag="3")]
pub low_frequency: i32,
@@ -214,47 +193,73 @@ pub struct ProtoControllerRumble {
#[prost(int32, tag="5")]
pub duration: i32,
}
/// Union of all Input types
// WebRTC + signaling
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoInput {
#[prost(oneof="proto_input::InputType", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14")]
pub input_type: ::core::option::Option<proto_input::InputType>,
pub struct RtcIceCandidateInit {
#[prost(string, tag="1")]
pub candidate: ::prost::alloc::string::String,
#[prost(uint32, optional, tag="2")]
pub sdp_m_line_index: ::core::option::Option<u32>,
#[prost(string, optional, tag="3")]
pub sdp_mid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag="4")]
pub username_fragment: ::core::option::Option<::prost::alloc::string::String>,
}
/// Nested message and enum types in `ProtoInput`.
pub mod proto_input {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum InputType {
#[prost(message, tag="1")]
MouseMove(super::ProtoMouseMove),
#[prost(message, tag="2")]
MouseMoveAbs(super::ProtoMouseMoveAbs),
#[prost(message, tag="3")]
MouseWheel(super::ProtoMouseWheel),
#[prost(message, tag="4")]
MouseKeyDown(super::ProtoMouseKeyDown),
#[prost(message, tag="5")]
MouseKeyUp(super::ProtoMouseKeyUp),
#[prost(message, tag="6")]
KeyDown(super::ProtoKeyDown),
#[prost(message, tag="7")]
KeyUp(super::ProtoKeyUp),
#[prost(message, tag="8")]
ControllerAttach(super::ProtoControllerAttach),
#[prost(message, tag="9")]
ControllerDetach(super::ProtoControllerDetach),
#[prost(message, tag="10")]
ControllerButton(super::ProtoControllerButton),
#[prost(message, tag="11")]
ControllerTrigger(super::ProtoControllerTrigger),
#[prost(message, tag="12")]
ControllerStick(super::ProtoControllerStick),
#[prost(message, tag="13")]
ControllerAxis(super::ProtoControllerAxis),
#[prost(message, tag="14")]
ControllerRumble(super::ProtoControllerRumble),
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct RtcSessionDescriptionInit {
#[prost(string, tag="1")]
pub sdp: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub r#type: ::prost::alloc::string::String,
}
/// ProtoICE message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoIce {
#[prost(message, optional, tag="1")]
pub candidate: ::core::option::Option<RtcIceCandidateInit>,
}
/// ProtoSDP message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoSdp {
#[prost(message, optional, tag="1")]
pub sdp: ::core::option::Option<RtcSessionDescriptionInit>,
}
/// ProtoRaw message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoRaw {
#[prost(string, tag="1")]
pub data: ::prost::alloc::string::String,
}
/// ProtoClientRequestRoomStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoClientRequestRoomStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
#[prost(string, tag="2")]
pub session_id: ::prost::alloc::string::String,
}
/// ProtoClientDisconnected message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoClientDisconnected {
#[prost(string, tag="1")]
pub session_id: ::prost::alloc::string::String,
#[prost(int32, repeated, tag="2")]
pub controller_slots: ::prost::alloc::vec::Vec<i32>,
}
/// ProtoServerPushStream message
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoServerPushStream {
#[prost(string, tag="1")]
pub room_name: ::prost::alloc::string::String,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -266,10 +271,59 @@ pub struct ProtoMessageBase {
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ProtoMessageInput {
pub struct ProtoMessage {
#[prost(message, optional, tag="1")]
pub message_base: ::core::option::Option<ProtoMessageBase>,
#[prost(message, optional, tag="2")]
pub data: ::core::option::Option<ProtoInput>,
#[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25")]
pub payload: ::core::option::Option<proto_message::Payload>,
}
/// Nested message and enum types in `ProtoMessage`.
pub mod proto_message {
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Payload {
/// Input types
#[prost(message, tag="2")]
MouseMove(super::ProtoMouseMove),
#[prost(message, tag="3")]
MouseMoveAbs(super::ProtoMouseMoveAbs),
#[prost(message, tag="4")]
MouseWheel(super::ProtoMouseWheel),
#[prost(message, tag="5")]
MouseKeyDown(super::ProtoMouseKeyDown),
#[prost(message, tag="6")]
MouseKeyUp(super::ProtoMouseKeyUp),
#[prost(message, tag="7")]
KeyDown(super::ProtoKeyDown),
#[prost(message, tag="8")]
KeyUp(super::ProtoKeyUp),
#[prost(message, tag="9")]
ControllerAttach(super::ProtoControllerAttach),
#[prost(message, tag="10")]
ControllerDetach(super::ProtoControllerDetach),
#[prost(message, tag="11")]
ControllerButton(super::ProtoControllerButton),
#[prost(message, tag="12")]
ControllerTrigger(super::ProtoControllerTrigger),
#[prost(message, tag="13")]
ControllerStick(super::ProtoControllerStick),
#[prost(message, tag="14")]
ControllerAxis(super::ProtoControllerAxis),
#[prost(message, tag="15")]
ControllerRumble(super::ProtoControllerRumble),
/// Signaling types
#[prost(message, tag="20")]
Ice(super::ProtoIce),
#[prost(message, tag="21")]
Sdp(super::ProtoSdp),
#[prost(message, tag="22")]
Raw(super::ProtoRaw),
#[prost(message, tag="23")]
ClientRequestRoomStream(super::ProtoClientRequestRoomStream),
#[prost(message, tag="24")]
ClientDisconnected(super::ProtoClientDisconnected),
#[prost(message, tag="25")]
ServerPushStream(super::ProtoServerPushStream),
}
}
// @@protoc_insertion_point(module)

28
protobufs/messages.proto
View File

@@ -12,7 +12,31 @@ message ProtoMessageBase {
ProtoLatencyTracker latency = 2;
}
message ProtoMessageInput {
message ProtoMessage {
ProtoMessageBase message_base = 1;
ProtoInput data = 2;
oneof payload {
// Input types
ProtoMouseMove mouse_move = 2;
ProtoMouseMoveAbs mouse_move_abs = 3;
ProtoMouseWheel mouse_wheel = 4;
ProtoMouseKeyDown mouse_key_down = 5;
ProtoMouseKeyUp mouse_key_up = 6;
ProtoKeyDown key_down = 7;
ProtoKeyUp key_up = 8;
ProtoControllerAttach controller_attach = 9;
ProtoControllerDetach controller_detach = 10;
ProtoControllerButton controller_button = 11;
ProtoControllerTrigger controller_trigger = 12;
ProtoControllerStick controller_stick = 13;
ProtoControllerAxis controller_axis = 14;
ProtoControllerRumble controller_rumble = 15;
// Signaling types
ProtoICE ice = 20;
ProtoSDP sdp = 21;
ProtoRaw raw = 22;
ProtoClientRequestRoomStream client_request_room_stream = 23;
ProtoClientDisconnected client_disconnected = 24;
ProtoServerPushStream server_push_stream = 25;
}
}

117
protobufs/types.proto
View File

@@ -8,86 +8,79 @@ package proto;
// MouseMove message
message ProtoMouseMove {
string type = 1; // Fixed value "MouseMove"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseMoveAbs message
message ProtoMouseMoveAbs {
string type = 1; // Fixed value "MouseMoveAbs"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseWheel message
message ProtoMouseWheel {
string type = 1; // Fixed value "MouseWheel"
int32 x = 2;
int32 y = 3;
int32 x = 1;
int32 y = 2;
}
// MouseKeyDown message
message ProtoMouseKeyDown {
string type = 1; // Fixed value "MouseKeyDown"
int32 key = 2;
int32 key = 1;
}
// MouseKeyUp message
message ProtoMouseKeyUp {
string type = 1; // Fixed value "MouseKeyUp"
int32 key = 2;
int32 key = 1;
}
/* Keyboard messages */
// KeyDown message
message ProtoKeyDown {
string type = 1; // Fixed value "KeyDown"
int32 key = 2;
int32 key = 1;
}
// KeyUp message
message ProtoKeyUp {
string type = 1; // Fixed value "KeyUp"
int32 key = 2;
int32 key = 1;
}
/* Controller messages */
// ControllerAttach message
message ProtoControllerAttach {
string type = 1; // Fixed value "ControllerAttach"
string id = 2; // One of the following enums: "ps", "xbox" or "switch"
int32 slot = 3; // Slot number (0-3)
string id = 1; // One of the following enums: "ps", "xbox" or "switch"
int32 session_slot = 2; // Session specific slot number (0-3)
string session_id = 3; // Session ID of the client
}
// ControllerDetach message
message ProtoControllerDetach {
string type = 1; // Fixed value "ControllerDetach"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
}
// ControllerButton message
message ProtoControllerButton {
string type = 1; // Fixed value "ControllerButtons"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
int32 button = 3; // Button code (linux input event code)
bool pressed = 4; // true if pressed, false if released
}
// ControllerTriggers message
message ProtoControllerTrigger {
string type = 1; // Fixed value "ControllerTriggers"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
int32 trigger = 3; // Trigger number (0 for left, 1 for right)
int32 value = 4; // trigger value (-32768 to 32767)
}
// ControllerSticks message
message ProtoControllerStick {
string type = 1; // Fixed value "ControllerStick"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
int32 stick = 3; // Stick number (0 for left, 1 for right)
int32 x = 4; // X axis value (-32768 to 32767)
int32 y = 5; // Y axis value (-32768 to 32767)
@@ -95,37 +88,63 @@ message ProtoControllerStick {
// ControllerAxis message
message ProtoControllerAxis {
string type = 1; // Fixed value "ControllerAxis"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
int32 value = 4; // axis value (-1 to 1)
}
// ControllerRumble message
message ProtoControllerRumble {
string type = 1; // Fixed value "ControllerRumble"
int32 slot = 2; // Slot number (0-3)
int32 session_slot = 1; // Session specific slot number (0-3)
string session_id = 2; // Session ID of the client
int32 low_frequency = 3; // Low frequency rumble (0-65535)
int32 high_frequency = 4; // High frequency rumble (0-65535)
int32 duration = 5; // Duration in milliseconds
}
// Union of all Input types
message ProtoInput {
oneof input_type {
ProtoMouseMove mouse_move = 1;
ProtoMouseMoveAbs mouse_move_abs = 2;
ProtoMouseWheel mouse_wheel = 3;
ProtoMouseKeyDown mouse_key_down = 4;
ProtoMouseKeyUp mouse_key_up = 5;
ProtoKeyDown key_down = 6;
ProtoKeyUp key_up = 7;
ProtoControllerAttach controller_attach = 8;
ProtoControllerDetach controller_detach = 9;
ProtoControllerButton controller_button = 10;
ProtoControllerTrigger controller_trigger = 11;
ProtoControllerStick controller_stick = 12;
ProtoControllerAxis controller_axis = 13;
ProtoControllerRumble controller_rumble = 14;
/* WebRTC + signaling */
message RTCIceCandidateInit {
string candidate = 1;
optional uint32 sdpMLineIndex = 2;
optional string sdpMid = 3;
optional string usernameFragment = 4;
}
message RTCSessionDescriptionInit {
string sdp = 1;
string type = 2;
}
// ProtoICE message
message ProtoICE {
RTCIceCandidateInit candidate = 1;
}
// ProtoSDP message
message ProtoSDP {
RTCSessionDescriptionInit sdp = 1;
}
// ProtoRaw message
message ProtoRaw {
string data = 1;
}
// ProtoClientRequestRoomStream message
message ProtoClientRequestRoomStream {
string room_name = 1;
string session_id = 2;
}
// ProtoClientDisconnected message
message ProtoClientDisconnected {
string session_id = 1;
repeated int32 controller_slots = 2;
}
// ProtoServerPushStream message
message ProtoServerPushStream {
string room_name = 1;
}