github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Fully use protobuf, fix controller issues and cleanup (#305)

Browse Source 
## Description
### First commit
Restructured protobuf schemas to make them easier to use across
languages, switched to using them in-place of JSON for signaling as
well, so there's no 2 different message formats flying about. Few new
message types to deal with clients and nestri-servers better (not final
format, may see changes still).

General cleanup of dead/unused code along some bug squashing and package
updates.

TODO for future commits:
- [x] Fix additional controllers not doing inputs (possibly needs
vimputti changes)
- [x] ~~Restructure relay protocols code a bit, to reduce bloatiness of
the currently single file for them, more code re-use.~~
- Gonna keep this PR somewhat manageable without poking more at relay..
- [x] ~~Try to fix issue where with multiple clients, static stream
content causes video to freeze until there's some movement.~~
- Was caused by server tuned profile being `throughput-performance`,
causing CPU latency to be too high.
- [x] Ponder the orb


### Second + third commit
Redid the controller polling handling and fixed multi-controller
handling in vimputti and nestri code sides. Remove some dead relay code
as well to clean up the protocol source file, we'll revisit the meshing
functionality later.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added software rendering option and MangoHud runtime config;
controller sessions now support reconnection and batched state updates
with persistent session IDs.

* **Bug Fixes**
* Restored previously-filtered NES-like gamepads so they connect
correctly.

* **Chores**
* Modernized dependencies and protobuf tooling, migrated to
protobuf-based messaging and streaming, and removed obsolete CUDA build
steps.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
This commit is contained in:
Kristian Ollikainen
2025-11-08 13:10:28 +02:00
committed by GitHub
parent 32341574dc
commit d87a0b35dd
50 changed files with 4413 additions and 3883 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/input/package.json
View File

@@ -7,24 +7,22 @@
".": "./src/index.ts"
},
"devDependencies": {
"@bufbuild/buf": "^1.57.2",
"@bufbuild/protoc-gen-es": "^2.9.0"
"@bufbuild/buf": "^1.59.0",
"@bufbuild/protoc-gen-es": "^2.10.0"
},
"dependencies": {
"@bufbuild/protobuf": "^2.9.0",
"@chainsafe/libp2p-noise": "^16.1.4",
"@bufbuild/protobuf": "^2.10.0",
"@chainsafe/libp2p-noise": "^17.0.0",
"@chainsafe/libp2p-quic": "^1.1.3",
"@chainsafe/libp2p-yamux": "^7.0.4",
"@libp2p/identify": "^3.0.39",
"@libp2p/interface": "^2.11.0",
"@libp2p/ping": "^2.0.37",
"@libp2p/websockets": "^9.2.19",
"@libp2p/webtransport": "^5.0.51",
"@multiformats/multiaddr": "^12.5.1",
"it-length-prefixed": "^10.0.1",
"it-pipe": "^3.0.1",
"libp2p": "^2.10.0",
"uint8arraylist": "^2.4.8",
"uint8arrays": "^5.1.0"
"@chainsafe/libp2p-yamux": "^8.0.1",
"@libp2p/identify": "^4.0.5",
"@libp2p/interface": "^3.0.2",
"@libp2p/ping": "^3.0.5",
"@libp2p/websockets": "^10.0.6",
"@libp2p/webtransport": "^6.0.7",
"@libp2p/utils": "^7.0.5",
"@multiformats/multiaddr": "^13.0.1",
"libp2p": "^3.0.6",
"uint8arraylist": "^2.4.8"
}
}

766
packages/input/src/controller.ts
View File

@@ -1,21 +1,15 @@
import { controllerButtonToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import {
ProtoMessageBase,
ProtoMessageInput,
ProtoMessageInputSchema,
} from "./proto/messages_pb";
import {
ProtoInputSchema,
ProtoControllerAttachSchema,
ProtoControllerDetachSchema,
ProtoControllerButtonSchema,
ProtoControllerTriggerSchema,
ProtoControllerAxisSchema,
ProtoControllerStickSchema,
ProtoControllerStateBatchSchema,
ProtoControllerStateBatch,
ProtoControllerRumble,
} from "./proto/types_pb";
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -23,6 +17,7 @@ interface Props {
}
interface GamepadState {
previousButtonState: Map<number, boolean>;
buttonState: Map<number, boolean>;
leftTrigger: number;
rightTrigger: number;
@@ -34,12 +29,17 @@ interface GamepadState {
dpadY: number;
}
enum PollState {
IDLE,
RUNNING,
}
export class Controller {
protected wrtc: WebRTCStream;
protected slot: number;
protected connected: boolean = false;
protected gamepad: Gamepad | null = null;
protected lastState: GamepadState = {
protected state: GamepadState = {
previousButtonState: new Map<number, boolean>(),
buttonState: new Map<number, boolean>(),
leftTrigger: 0,
rightTrigger: 0,
@@ -53,17 +53,33 @@ export class Controller {
// TODO: As user configurable, set quite low now for decent controllers (not Nintendo ones :P)
protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range)
private updateInterval = 10.0; // 100 updates per second
private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null;
// Polling configuration
private readonly FULL_RATE_MS = 10; // 100 UPS
private readonly IDLE_THRESHOLD = 100; // ms before considering idle/hands off controller
private readonly FULL_INTERVAL = 250; // ms before sending full state occassionally, to verify inputs are synced
// Polling state
private pollingState: PollState = PollState.IDLE;
private lastInputTime: number = Date.now();
private lastFullTime: number = Date.now();
private pollInterval: any = null;
// Controller batch vars
private sequence: number = 0;
private readonly CHANGED_BUTTONS_STATE = 1 << 0;
private readonly CHANGED_LEFT_STICK_X = 1 << 1;
private readonly CHANGED_LEFT_STICK_Y = 1 << 2;
private readonly CHANGED_RIGHT_STICK_X = 1 << 3;
private readonly CHANGED_RIGHT_STICK_Y = 1 << 4;
private readonly CHANGED_LEFT_TRIGGER = 1 << 5;
private readonly CHANGED_RIGHT_TRIGGER = 1 << 6;
private readonly CHANGED_DPAD_X = 1 << 7;
private readonly CHANGED_DPAD_Y = 1 << 8;
private _dcHandler: ((data: ArrayBuffer) => void) | null = null;
constructor({ webrtc, e }: Props) {
this.wrtc = webrtc;
this.slot = e.gamepad.index;
this.updateInterval = 1000 / webrtc.currentFrameRate;
// Gamepad connected
this.gamepad = e.gamepad;
// Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc")
const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/);
@@ -72,34 +88,49 @@ export class Controller {
const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/);
const productId = productMatch ? productMatch[1].toLowerCase() : "unknown";
const attachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAttach",
value: create(ProtoControllerAttachSchema, {
type: "ControllerAttach",
id: this.vendor_id_to_controller(vendorId, productId),
slot: this.slot,
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: attachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
// Listen to datachannel events from server
this._dcHandler = (data: ArrayBuffer) => {
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data);
// Listen to feedback rumble events from server
this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer);
this.wrtc.addDataChannelCallback(this._dcRumbleHandler);
if (messageWrapper.payload.case === "controllerRumble") {
this.rumbleCallback(messageWrapper.payload.value);
} else if (messageWrapper.payload.case === "controllerAttach") {
if (this.gamepad) return; // already attached
const attachMsg = messageWrapper.payload.value;
// Gamepad connected succesfully
this.gamepad = e.gamepad;
console.log(
`Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`,
);
this.run();
}
} catch (err) {
console.error("Error decoding datachannel message:", err);
}
};
this.wrtc.addDataChannelCallback(this._dcHandler);
const attachMsg = createMessage(
create(ProtoControllerAttachSchema, {
id: this.vendor_id_to_controller(vendorId, productId),
sessionSlot: e.gamepad.index,
sessionId: this.wrtc.getSessionID(),
}),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg));
this.run();
}
public getSlot(): number {
return this.gamepad.index;
}
// Maps vendor id and product id to supported controller type
// Currently supported: Sony (ps4, ps5), Microsoft (xbox360, xboxone), Nintendo (switchpro)
// Default fallback to xbox360
@@ -149,361 +180,352 @@ export class Controller {
return ((value - fromMin) * (toMax - toMin)) / (fromMax - fromMin) + toMin;
}
private pollGamepad() {
const gamepads = navigator.getGamepads();
if (this.slot < gamepads.length) {
const gamepad = gamepads[this.slot];
if (gamepad) {
/* Button handling */
gamepad.buttons.forEach((button, index) => {
// Ignore d-pad buttons (12-15) as we handle those as axis
if (index >= 12 && index <= 15) return;
// ignore trigger buttons (6-7) as we handle those as axis
if (index === 6 || index === 7) return;
// If state differs, send
if (button.pressed !== this.lastState.buttonState.get(index)) {
const linuxCode = this.controllerButtonToVirtualKeyCode(index);
if (linuxCode === undefined) {
// Skip unmapped button index
this.lastState.buttonState.set(index, button.pressed);
return;
}
const buttonProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerButton",
value: create(ProtoControllerButtonSchema, {
type: "ControllerButton",
slot: this.slot,
button: linuxCode,
pressed: button.pressed,
}),
},
});
const buttonMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: buttonProto,
};
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, buttonMessage),
);
// Store button state
this.lastState.buttonState.set(index, button.pressed);
}
});
/* Trigger handling */
// map trigger value from 0.0 to 1.0 to -32768 to 32767
const leftTrigger = Math.round(
this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767),
);
// If state differs, send
if (leftTrigger !== this.lastState.leftTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
trigger: 0, // 0 = left, 1 = right
value: leftTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.leftTrigger = leftTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
);
}
const rightTrigger = Math.round(
this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767),
);
// If state differs, send
if (rightTrigger !== this.lastState.rightTrigger) {
const triggerProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerTrigger",
value: create(ProtoControllerTriggerSchema, {
type: "ControllerTrigger",
slot: this.slot,
trigger: 1, // 0 = left, 1 = right
value: rightTrigger,
}),
},
});
const triggerMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: triggerProto,
};
this.lastState.rightTrigger = rightTrigger;
this.wrtc.sendBinary(
toBinary(ProtoMessageInputSchema, triggerMessage),
);
}
/* DPad handling */
// We send dpad buttons as axis values -1 to 1 for left/up, right/down
const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0;
const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0;
const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0;
if (dpadX !== this.lastState.dpadX) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
axis: 0, // 0 = dpadX, 1 = dpadY
value: dpadX,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
this.lastState.dpadX = dpadX;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
}
const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0;
const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0;
const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0;
if (dpadY !== this.lastState.dpadY) {
const dpadProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerAxis",
value: create(ProtoControllerAxisSchema, {
type: "ControllerAxis",
slot: this.slot,
axis: 1, // 0 = dpadX, 1 = dpadY
value: dpadY,
}),
},
});
const dpadMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: dpadProto,
};
this.lastState.dpadY = dpadY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
}
/* Stick handling */
// stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767
const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767);
const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767);
// Apply deadzone
const sendLeftX =
Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
const sendLeftY =
Math.abs(leftY) > this.stickDeadzone ? Math.round(leftY) : 0;
// if outside deadzone, send normally if changed
// if moves inside deadzone, zero it if not inside deadzone last time
if (
sendLeftX !== this.lastState.leftX ||
sendLeftY !== this.lastState.leftY
) {
// console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY);
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
stick: 0, // 0 = left, 1 = right
x: sendLeftX,
y: sendLeftY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
this.lastState.leftX = sendLeftX;
this.lastState.leftY = sendLeftY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767);
const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767);
// Apply deadzone
const sendRightX =
Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
const sendRightY =
Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0;
if (
sendRightX !== this.lastState.rightX ||
sendRightY !== this.lastState.rightY
) {
const stickProto = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerStick",
value: create(ProtoControllerStickSchema, {
type: "ControllerStick",
slot: this.slot,
stick: 1, // 0 = left, 1 = right
x: sendRightX,
y: sendRightY,
}),
},
});
const stickMessage: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: stickProto,
};
this.lastState.rightX = sendRightX;
this.lastState.rightY = sendRightY;
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
}
}
private restartPolling() {
// Clear existing interval
if (this.pollInterval) {
clearInterval(this.pollInterval);
this.pollInterval = null;
}
// Restart with active polling
this.pollingState = PollState.RUNNING;
this.lastInputTime = Date.now();
// Start interval
this.pollInterval = setInterval(
() => this.pollGamepad(),
this.FULL_RATE_MS,
);
}
private loopInterval: any = null;
private pollGamepad() {
if (!this.connected || !this.gamepad) return;
const gamepads = navigator.getGamepads();
if (!gamepads[this.gamepad.index]) return;
this.gamepad = gamepads[this.gamepad.index];
// Collect state changes
const changedFields = this.collectStateChanges();
// Send batched changes update if there's changes
if (changedFields > 0) {
let send_type = 1;
const timeSinceFull = Date.now() - this.lastFullTime;
if (timeSinceFull > this.FULL_INTERVAL) {
send_type = 0;
this.lastFullTime = Date.now();
}
this.sendBatchedState(changedFields, send_type);
this.lastInputTime = Date.now();
if (this.pollingState !== PollState.RUNNING) {
this.pollingState = PollState.RUNNING;
}
}
const timeSinceInput = Date.now() - this.lastInputTime;
if (timeSinceInput > this.IDLE_THRESHOLD) {
// Changing from running to idle..
if (this.pollingState === PollState.RUNNING) {
// Send full state on idle assumption
this.sendBatchedState(0xff, 0);
this.pollingState = PollState.IDLE;
}
}
this.state.buttonState.forEach((b, i) =>
this.state.previousButtonState.set(i, b),
);
}
private collectStateChanges(): number {
let changedFields = 0;
// Collect analog values
const leftTrigger = Math.round(
this.remapFromTo(
this.gamepad.buttons[6]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
const rightTrigger = Math.round(
this.remapFromTo(
this.gamepad.buttons[7]?.value ?? 0,
0,
1,
-32768,
32767,
),
);
const leftX = this.remapFromTo(
this.gamepad.axes[0] ?? 0,
-1,
1,
-32768,
32767,
);
const leftY = this.remapFromTo(
this.gamepad.axes[1] ?? 0,
-1,
1,
-32768,
32767,
);
const sendLeftX =
Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
const sendLeftY =
Math.abs(leftY) > this.stickDeadzone ? Math.round(leftY) : 0;
const rightX = this.remapFromTo(
this.gamepad.axes[2] ?? 0,
-1,
1,
-32768,
32767,
);
const rightY = this.remapFromTo(
this.gamepad.axes[3] ?? 0,
-1,
1,
-32768,
32767,
);
const sendRightX =
Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
const sendRightY =
Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0;
const dpadX =
(this.gamepad.buttons[14]?.pressed ? -1 : 0) +
(this.gamepad.buttons[15]?.pressed ? 1 : 0);
const dpadY =
(this.gamepad.buttons[12]?.pressed ? -1 : 0) +
(this.gamepad.buttons[13]?.pressed ? 1 : 0);
// Check what changed
for (let i = 0; i < this.gamepad.buttons.length; i++) {
if (i >= 6 && i <= 7) continue; // Skip triggers
if (i >= 12 && i <= 15) continue; // Skip d-pad
if (this.state.buttonState.get(i) !== this.gamepad.buttons[i].pressed) {
changedFields |= this.CHANGED_BUTTONS_STATE;
}
this.state.buttonState.set(i, this.gamepad.buttons[i].pressed);
}
if (leftTrigger !== this.state.leftTrigger) {
changedFields |= this.CHANGED_LEFT_TRIGGER;
}
this.state.leftTrigger = leftTrigger;
if (rightTrigger !== this.state.rightTrigger) {
changedFields |= this.CHANGED_RIGHT_TRIGGER;
}
this.state.rightTrigger = rightTrigger;
if (sendLeftX !== this.state.leftX) {
changedFields |= this.CHANGED_LEFT_STICK_X;
}
this.state.leftX = sendLeftX;
if (sendLeftY !== this.state.leftY) {
changedFields |= this.CHANGED_LEFT_STICK_Y;
}
this.state.leftY = sendLeftY;
if (sendRightX !== this.state.rightX) {
changedFields |= this.CHANGED_RIGHT_STICK_X;
}
this.state.rightX = sendRightX;
if (sendRightY !== this.state.rightY) {
changedFields |= this.CHANGED_RIGHT_STICK_Y;
}
this.state.rightY = sendRightY;
if (dpadX !== this.state.dpadX) {
changedFields |= this.CHANGED_DPAD_X;
}
this.state.dpadX = dpadX;
if (dpadY !== this.state.dpadY) {
changedFields |= this.CHANGED_DPAD_Y;
}
this.state.dpadY = dpadY;
return changedFields;
}
private sendBatchedState(changedFields: number, updateType: number) {
// @ts-ignore
let message: ProtoControllerStateBatch = {
sessionSlot: this.gamepad.index,
sessionId: this.wrtc.getSessionID(),
updateType: updateType,
sequence: this.sequence++,
};
// For FULL_STATE, include everything
if (updateType === 0) {
message.changedFields = 0xff;
message.buttonChangedMask = Object.fromEntries(
Array.from(this.state.buttonState)
.map(
([key, value]) =>
[this.controllerButtonToVirtualKeyCode(key), value] as const,
)
.filter(([code]) => code !== undefined),
);
message.leftStickX = this.state.leftX;
message.leftStickY = this.state.leftY;
message.rightStickX = this.state.rightX;
message.rightStickY = this.state.rightY;
message.leftTrigger = this.state.leftTrigger;
message.rightTrigger = this.state.rightTrigger;
message.dpadX = this.state.dpadX;
message.dpadY = this.state.dpadY;
}
// For DELTA, only include changed fields
else {
message.changedFields = changedFields;
if (changedFields & this.CHANGED_BUTTONS_STATE) {
const currentStateMap = this.state.buttonState;
const previousStateMap = this.state.previousButtonState;
const allKeys = new Set([
// @ts-ignore
...currentStateMap.keys(),
// @ts-ignore
...previousStateMap.keys(),
]);
message.buttonChangedMask = Object.fromEntries(
Array.from(allKeys)
.filter((key) => {
const newState = currentStateMap.get(key);
const oldState = previousStateMap.get(key);
return newState !== oldState;
})
.map((key) => {
const newValue = currentStateMap.get(key) ?? false;
return [
this.controllerButtonToVirtualKeyCode(key),
newValue,
] as const;
})
.filter(([code]) => code !== undefined),
);
}
if (changedFields & this.CHANGED_LEFT_STICK_X) {
message.leftStickX = this.state.leftX;
}
if (changedFields & this.CHANGED_LEFT_STICK_Y) {
message.leftStickY = this.state.leftY;
}
if (changedFields & this.CHANGED_RIGHT_STICK_X) {
message.rightStickX = this.state.rightX;
}
if (changedFields & this.CHANGED_RIGHT_STICK_Y) {
message.rightStickY = this.state.rightY;
}
if (changedFields & this.CHANGED_LEFT_TRIGGER) {
message.leftTrigger = this.state.leftTrigger;
}
if (changedFields & this.CHANGED_RIGHT_TRIGGER) {
message.rightTrigger = this.state.rightTrigger;
}
if (changedFields & this.CHANGED_DPAD_X) {
message.dpadX = this.state.dpadX;
}
if (changedFields & this.CHANGED_DPAD_Y) {
message.dpadY = this.state.dpadY;
}
}
// Send message
const batchMessage = createMessage(
create(
ProtoControllerStateBatchSchema,
message as ProtoControllerStateBatch,
),
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, batchMessage));
}
public run() {
if (this.connected)
this.stop();
if (this.connected) this.stop();
this.connected = true;
// Poll gamepads in setInterval loop
this.loopInterval = setInterval(() => {
if (this.connected) this.pollGamepad();
}, this.updateInterval);
// Start with active polling
this.restartPolling();
}
public stop() {
if (this.loopInterval) {
clearInterval(this.loopInterval);
this.loopInterval = null;
if (this.pollInterval) {
clearInterval(this.pollInterval);
this.pollInterval = null;
}
this.connected = false;
}
public getSlot() {
return this.slot;
}
public dispose() {
this.stop();
// Remove callback
if (this._dcRumbleHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcRumbleHandler);
this._dcRumbleHandler = null;
if (this._dcHandler !== null) {
this.wrtc.removeDataChannelCallback(this._dcHandler);
this._dcHandler = null;
}
// Gamepad disconnected
const detachMsg = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "controllerDetach",
value: create(ProtoControllerDetachSchema, {
type: "ControllerDetach",
slot: this.slot,
if (this.gamepad) {
// Gamepad disconnected
const detachMsg = createMessage(
create(ProtoControllerDetachSchema, {
sessionSlot: this.gamepad.index,
}),
},
});
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "controllerInput",
} as ProtoMessageBase,
data: detachMsg,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
"controllerInput",
);
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg));
}
}
private controllerButtonToVirtualKeyCode(code: number) {
private controllerButtonToVirtualKeyCode(code: number): number | undefined {
return controllerButtonToLinuxEventCode[code] || undefined;
}
private rumbleCallback(data: ArrayBuffer) {
// If not connected, ignore
if (!this.connected) return;
try {
// First decode the wrapper message
const uint8Data = new Uint8Array(data);
const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data);
private rumbleCallback(rumbleMsg: ProtoControllerRumble) {
if (!this.connected || !this.gamepad) return;
// Check if it contains controller rumble data
if (messageWrapper.data?.inputType?.case === "controllerRumble") {
const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble;
// Check if this rumble is for us
if (
rumbleMsg.sessionId !== this.wrtc.getSessionID() ||
rumbleMsg.sessionSlot !== this.gamepad.index
)
return;
// Check if aimed at this controller slot
if (rumbleMsg.slot !== this.slot) return;
// Trigger actual rumble
// Need to remap from 0-65535 to 0.0-1.0 ranges
const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
const rumbleLowFreq = this.remapFromTo(
clampedLowFreq,
0,
65535,
0.0,
1.0,
);
const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency));
const rumbleHighFreq = this.remapFromTo(
clampedHighFreq,
0,
65535,
0.0,
1.0,
);
// Cap to valid range (max 5000)
const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
if (this.gamepad.vibrationActuator) {
this.gamepad.vibrationActuator.playEffect("dual-rumble", {
startDelay: 0,
duration: rumbleDuration,
weakMagnitude: rumbleLowFreq,
strongMagnitude: rumbleHighFreq,
}).catch(console.error);
}
}
} catch (error) {
console.error("Failed to decode rumble message:", error);
// Trigger actual rumble
// Need to remap from 0-65535 to 0.0-1.0 ranges
const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0);
const clampedHighFreq = Math.max(
0,
Math.min(65535, rumbleMsg.highFrequency),
);
const rumbleHighFreq = this.remapFromTo(
clampedHighFreq,
0,
65535,
0.0,
1.0,
);
// Cap to valid range (max 5000)
const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
if (this.gamepad.vibrationActuator) {
this.gamepad.vibrationActuator
.playEffect("dual-rumble", {
startDelay: 0,
duration: rumbleDuration,
weakMagnitude: rumbleLowFreq,
strongMagnitude: rumbleHighFreq,
})
.catch(console.error);
}
}
}

101
packages/input/src/keyboard.ts
View File

@@ -1,16 +1,9 @@
import {keyCodeToLinuxEventCode} from "./codes"
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
import {
ProtoInput,
ProtoInputSchema,
ProtoKeyDownSchema,
ProtoKeyUpSchema,
} from "./proto/types_pb";
import {create, toBinary} from "@bufbuild/protobuf";
import { keyCodeToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,38 +17,29 @@ export class Keyboard {
private readonly keydownListener: (e: KeyboardEvent) => void;
private readonly keyupListener: (e: KeyboardEvent) => void;
constructor({webrtc}: Props) {
constructor({ webrtc }: Props) {
this.wrtc = webrtc;
this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyDown",
value: create(ProtoKeyDownSchema, {
type: "KeyDown",
key: this.keyToVirtualKeyCode(e.code)
}),
}
}));
this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyUp",
value: create(ProtoKeyUpSchema, {
type: "KeyUp",
key: this.keyToVirtualKeyCode(e.code)
}),
}
}));
this.run()
this.keydownListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
);
this.keyupListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
);
this.run();
}
private run() {
if (this.connected)
this.stop()
if (this.connected) this.stop();
this.connected = true
document.addEventListener("keydown", this.keydownListener, {passive: false});
document.addEventListener("keyup", this.keyupListener, {passive: false});
this.connected = true;
document.addEventListener("keydown", this.keydownListener, {
passive: false,
});
document.addEventListener("keyup", this.keyupListener, { passive: false });
}
private stop() {
@@ -65,42 +49,19 @@ export class Keyboard {
}
// Helper function to create and return mouse listeners
private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createKeyboardListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
// Prevent repeated key events from being sent (important for games)
if ((e as any).repeat)
return;
if ((e as any).repeat) return;
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-keyboard");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}
@@ -114,4 +75,4 @@ export class Keyboard {
if (code === "Home") return 1;
return keyCodeToLinuxEventCode[code] || undefined;
}
}
}

305
packages/input/src/messages.ts
View File

@@ -1,305 +0,0 @@
import { LatencyTracker } from "./latency";
import { Uint8ArrayList } from "uint8arraylist";
import { allocUnsafe } from "uint8arrays/alloc";
import { pipe } from "it-pipe";
import { decode, encode } from "it-length-prefixed";
import { Stream } from "@libp2p/interface";
export interface MessageBase {
payload_type: string;
latency?: LatencyTracker;
}
export interface MessageRaw extends MessageBase {
data: any;
}
export function NewMessageRaw(type: string, data: any): Uint8Array {
const msg = {
payload_type: type,
data: data,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageICE extends MessageBase {
candidate: RTCIceCandidateInit;
}
export function NewMessageICE(
type: string,
candidate: RTCIceCandidateInit,
): Uint8Array {
const msg = {
payload_type: type,
candidate: candidate,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
export interface MessageSDP extends MessageBase {
sdp: RTCSessionDescriptionInit;
}
export function NewMessageSDP(
type: string,
sdp: RTCSessionDescriptionInit,
): Uint8Array {
const msg = {
payload_type: type,
sdp: sdp,
};
return new TextEncoder().encode(JSON.stringify(msg));
}
const MAX_SIZE = 1024 * 1024; // 1MB
const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue
// Custom 4-byte length encoder
export const length4ByteEncoder = (length: number) => {
const buf = allocUnsafe(4);
// Write the length as a 32-bit unsigned integer (4 bytes)
buf[0] = length >>> 24;
buf[1] = (length >>> 16) & 0xff;
buf[2] = (length >>> 8) & 0xff;
buf[3] = length & 0xff;
// Set the bytes property to 4
length4ByteEncoder.bytes = 4;
return buf;
};
length4ByteEncoder.bytes = 4;
// Custom 4-byte length decoder
export const length4ByteDecoder = (data: Uint8ArrayList) => {
if (data.byteLength < 4) {
// Not enough bytes to read the length
return -1;
}
// Read the length from the first 4 bytes
let length = 0;
length =
(data.subarray(0, 1)[0] >>> 0) * 0x1000000 +
(data.subarray(1, 2)[0] >>> 0) * 0x10000 +
(data.subarray(2, 3)[0] >>> 0) * 0x100 +
(data.subarray(3, 4)[0] >>> 0);
// Set bytes read to 4
length4ByteDecoder.bytes = 4;
return length;
};
length4ByteDecoder.bytes = 4;
interface PromiseMessage {
data: Uint8Array;
resolve: () => void;
reject: (error: Error) => void;
}
export class SafeStream {
private stream: Stream;
private callbacks: Map<string, ((data: any) => void)[]> = new Map();
private isReading: boolean = false;
private isWriting: boolean = false;
private closed: boolean = false;
private messageQueue: PromiseMessage[] = [];
private writeLock = false;
private readRetries = 0;
private writeRetries = 0;
private readonly MAX_RETRIES = 5;
constructor(stream: Stream) {
this.stream = stream;
this.startReading();
this.startWriting();
}
private async startReading(): Promise<void> {
if (this.isReading || this.closed) return;
this.isReading = true;
try {
const source = this.stream.source;
const decodedSource = decode(source, {
maxDataLength: MAX_SIZE,
lengthDecoder: length4ByteDecoder,
});
for await (const chunk of decodedSource) {
if (this.closed) break;
this.readRetries = 0;
try {
const data = chunk.slice();
const message = JSON.parse(
new TextDecoder().decode(data),
) as MessageBase;
const msgType = message.payload_type;
if (this.callbacks.has(msgType)) {
const handlers = this.callbacks.get(msgType)!;
for (const handler of handlers) {
try {
handler(message);
} catch (err) {
console.error(`Error in message handler for ${msgType}:`, err);
}
}
}
} catch (err) {
console.error("Error processing message:", err);
}
}
} catch (err) {
console.error("Stream reading error:", err);
} finally {
this.isReading = false;
this.readRetries++;
// If not closed, try to restart reading
if (!this.closed && this.readRetries < this.MAX_RETRIES)
setTimeout(() => this.startReading(), 100);
else if (this.readRetries >= this.MAX_RETRIES)
console.error(
"Max retries reached for reading stream, stopping attempts",
);
}
}
public registerCallback(
msgType: string,
callback: (data: any) => void,
): void {
if (!this.callbacks.has(msgType)) {
this.callbacks.set(msgType, []);
}
this.callbacks.get(msgType)!.push(callback);
}
public removeCallback(msgType: string, callback: (data: any) => void): void {
if (this.callbacks.has(msgType)) {
const callbacks = this.callbacks.get(msgType)!;
const index = callbacks.indexOf(callback);
if (index !== -1) {
callbacks.splice(index, 1);
}
if (callbacks.length === 0) {
this.callbacks.delete(msgType);
}
}
}
private async startWriting(): Promise<void> {
if (this.isWriting || this.closed) return;
this.isWriting = true;
try {
// Create an async generator for real-time message processing
const messageSource = async function* (this: SafeStream) {
while (!this.closed) {
// Check if we have messages to send
if (this.messageQueue.length > 0) {
this.writeLock = true;
try {
const message = this.messageQueue[0];
// Encode the message
const encoded = encode([message.data], {
maxDataLength: MAX_SIZE,
lengthEncoder: length4ByteEncoder,
});
for await (const chunk of encoded) {
yield chunk;
}
// Remove message after successful sending
this.writeRetries = 0;
const sentMessage = this.messageQueue.shift();
if (sentMessage)
sentMessage.resolve();
} catch (err) {
console.error("Error encoding or sending message:", err);
const failedMessage = this.messageQueue.shift();
if (failedMessage)
failedMessage.reject(new Error(`Failed to send message: ${err}`));
} finally {
this.writeLock = false;
}
} else {
// No messages to send, wait for a short period
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
}.bind(this);
await pipe(messageSource(), this.stream.sink).catch((err) => {
console.error("Sink error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing to stream sink, stopping attempts");
}
});
} catch (err) {
console.error("Stream writing error:", err);
this.isWriting = false;
this.writeRetries++;
// Try to restart if not closed
if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
setTimeout(() => this.startWriting(), 1000);
} else if (this.writeRetries >= this.MAX_RETRIES) {
console.error("Max retries reached for writing stream, stopping attempts");
}
}
}
public async writeMessage(message: Uint8Array): Promise<void> {
if (this.closed) {
throw new Error("Cannot write to closed stream");
}
// Validate message size before queuing
if (message.length > MAX_SIZE) {
throw new Error("Message size exceeds maximum size limit");
}
// Check if the message queue is too large
if (this.messageQueue.length >= MAX_QUEUE_SIZE) {
throw new Error("Message queue is full, cannot write message");
}
// Create a promise to resolve when the message is sent
return new Promise((resolve, reject) => {
this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage);
});
}
public close(): void {
this.closed = true;
this.callbacks.clear();
// Reject pending messages
for (const msg of this.messageQueue)
msg.reject(new Error("Stream closed"));
this.messageQueue = [];
this.readRetries = 0;
this.writeRetries = 0;
}
}

174
packages/input/src/mouse.ts
View File

@@ -1,18 +1,14 @@
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
import { WebRTCStream } from "./webrtc-stream";
import {
ProtoInput, ProtoInputSchema,
ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
ProtoMouseMove,
ProtoMouseKeyDownSchema,
ProtoMouseKeyUpSchema,
ProtoMouseMoveSchema,
ProtoMouseWheel, ProtoMouseWheelSchema
ProtoMouseWheelSchema,
} from "./proto/types_pb";
import {mouseButtonToLinuxEventCode} from "./codes";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {create, toBinary} from "@bufbuild/protobuf";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import { mouseButtonToLinuxEventCode } from "./codes";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,7 +20,7 @@ export class Mouse {
protected canvas: HTMLCanvasElement;
protected connected!: boolean;
private sendInterval = 10 // 100 updates per second
private sendInterval = 10; // 100 updates per second
// Store references to event listeners
private readonly mousemoveListener: (e: MouseEvent) => void;
@@ -35,7 +31,7 @@ export class Mouse {
private readonly mouseupListener: (e: MouseEvent) => void;
private readonly mousewheelListener: (e: WheelEvent) => void;
constructor({webrtc, canvas}: Props) {
constructor({ webrtc, canvas }: Props) {
this.wrtc = webrtc;
this.canvas = canvas;
@@ -48,65 +44,56 @@ export class Mouse {
this.movementY += e.movementY;
};
this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyDown",
value: create(ProtoMouseKeyDownSchema, {
type: "MouseKeyDown",
key: this.keyToVirtualKeyCode(e.button)
}),
}
}));
this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseKeyUp",
value: create(ProtoMouseKeyUpSchema, {
type: "MouseKeyUp",
key: this.keyToVirtualKeyCode(e.button)
}),
}
}));
this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseWheel",
value: create(ProtoMouseWheelSchema, {
type: "MouseWheel",
x: Math.round(e.deltaX),
y: Math.round(e.deltaY),
}),
}
}));
this.mousedownListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
);
this.mouseupListener = this.createMouseListener((e: any) =>
create(ProtoMouseKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.button),
}),
);
this.mousewheelListener = this.createMouseListener((e: any) =>
create(ProtoMouseWheelSchema, {
x: Math.round(e.deltaX),
y: Math.round(e.deltaY),
}),
);
this.run()
this.run();
this.startProcessing();
}
private run() {
//calls all the other functions
if (!document.pointerLockElement) {
console.log("no pointerlock")
console.log("no pointerlock");
if (this.connected) {
this.stop()
this.stop();
}
return;
}
if (document.pointerLockElement == this.canvas) {
this.connected = true
this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
this.connected = true;
this.canvas.addEventListener("mousemove", this.mousemoveListener, {
passive: false,
});
this.canvas.addEventListener("mousedown", this.mousedownListener, {
passive: false,
});
this.canvas.addEventListener("mouseup", this.mouseupListener, {
passive: false,
});
this.canvas.addEventListener("wheel", this.mousewheelListener, {
passive: false,
});
} else {
if (this.connected) {
this.stop()
this.stop();
}
}
}
private stop() {
@@ -128,79 +115,26 @@ export class Mouse {
}
private sendAggregatedMouseMove() {
const data = create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "mouseMove",
value: create(ProtoMouseMoveSchema, {
type: "MouseMove",
x: Math.round(this.movementX),
y: Math.round(this.movementY),
}),
},
const data = create(ProtoMouseMoveSchema, {
x: Math.round(this.movementX),
y: Math.round(this.movementY),
});
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
}
// Helper function to create and return mouse listeners
private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createMouseListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}
@@ -213,4 +147,4 @@ export class Mouse {
private keyToVirtualKeyCode(code: number) {
return mouseButtonToLinuxEventCode[code] || undefined;
}
}
}

2
packages/input/src/proto/latency_tracker_pb.ts
View File

@@ -1,4 +1,4 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file latency_tracker.proto (package proto, syntax proto3)
/* eslint-disable */

128
packages/input/src/proto/messages_pb.ts
View File

@@ -1,10 +1,10 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file messages.proto (package proto, syntax proto3)
/* eslint-disable */
import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
import type { ProtoInput } from "./types_pb";
import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStateBatch, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb";
import { file_types } from "./types_pb";
import type { ProtoLatencyTracker } from "./latency_tracker_pb";
import { file_latency_tracker } from "./latency_tracker_pb";
@@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf";
* Describes the file messages.proto.
*/
export const file_messages: GenFile = /*@__PURE__*/
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIipQcKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9ydW1ibGUYCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIABJCChZjb250cm9sbGVyX3N0YXRlX2JhdGNoGAwgASgLMiAucHJvdG8uUHJvdG9Db250cm9sbGVyU3RhdGVCYXRjaEgAEh4KA2ljZRgUIAEoCzIPLnByb3RvLlByb3RvSUNFSAASHgoDc2RwGBUgASgLMg8ucHJvdG8uUHJvdG9TRFBIABIeCgNyYXcYFiABKAsyDy5wcm90by5Qcm90b1Jhd0gAEkkKGmNsaWVudF9yZXF1ZXN0X3Jvb21fc3RyZWFtGBcgASgLMiMucHJvdG8uUHJvdG9DbGllbnRSZXF1ZXN0Um9vbVN0cmVhbUgAEj0KE2NsaWVudF9kaXNjb25uZWN0ZWQYGCABKAsyHi5wcm90by5Qcm90b0NsaWVudERpc2Nvbm5lY3RlZEgAEjoKEnNlcnZlcl9wdXNoX3N0cmVhbRgZIAEoCzIcLnByb3RvLlByb3RvU2VydmVyUHVzaFN0cmVhbUgAQgkKB3BheWxvYWRCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw", [file_types, file_latency_tracker]);
/**
* @generated from message proto.ProtoMessageBase
@@ -39,24 +39,132 @@ export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*
messageDesc(file_messages, 0);
/**
* @generated from message proto.ProtoMessageInput
* @generated from message proto.ProtoMessage
*/
export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
export type ProtoMessage = Message<"proto.ProtoMessage"> & {
/**
* @generated from field: proto.ProtoMessageBase message_base = 1;
*/
messageBase?: ProtoMessageBase;
/**
* @generated from field: proto.ProtoInput data = 2;
* @generated from oneof proto.ProtoMessage.payload
*/
data?: ProtoInput;
payload: {
/**
* Input types
*
* @generated from field: proto.ProtoMouseMove mouse_move = 2;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 4;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 7;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 8;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* Controller input types
*
* @generated from field: proto.ProtoControllerAttach controller_attach = 9;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 10;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 11;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | {
/**
* @generated from field: proto.ProtoControllerStateBatch controller_state_batch = 12;
*/
value: ProtoControllerStateBatch;
case: "controllerStateBatch";
} | {
/**
* Signaling types
*
* @generated from field: proto.ProtoICE ice = 20;
*/
value: ProtoICE;
case: "ice";
} | {
/**
* @generated from field: proto.ProtoSDP sdp = 21;
*/
value: ProtoSDP;
case: "sdp";
} | {
/**
* @generated from field: proto.ProtoRaw raw = 22;
*/
value: ProtoRaw;
case: "raw";
} | {
/**
* @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23;
*/
value: ProtoClientRequestRoomStream;
case: "clientRequestRoomStream";
} | {
/**
* @generated from field: proto.ProtoClientDisconnected client_disconnected = 24;
*/
value: ProtoClientDisconnected;
case: "clientDisconnected";
} | {
/**
* @generated from field: proto.ProtoServerPushStream server_push_stream = 25;
*/
value: ProtoServerPushStream;
case: "serverPushStream";
} | { case: undefined; value?: undefined };
};
/**
* Describes the message proto.ProtoMessageInput.
* Use `create(ProtoMessageInputSchema)` to create a new message.
* Describes the message proto.ProtoMessage.
* Use `create(ProtoMessageSchema)` to create a new message.
*/
export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
export const ProtoMessageSchema: GenMessage<ProtoMessage> = /*@__PURE__*/
messageDesc(file_messages, 1);

697
packages/input/src/proto/types_pb.ts
View File

@@ -1,16 +1,16 @@
// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
// @generated from file types.proto (package proto, syntax proto3)
/* eslint-disable */
import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
import { enumDesc, fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
import type { Message } from "@bufbuild/protobuf";
/**
* Describes the file types.proto.
*/
export const file_types: GenFile = /*@__PURE__*/
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSKCAQoVUHJvdG9Db250cm9sbGVyUnVtYmxlEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi0AUKGVByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2gSFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSQAoLdXBkYXRlX3R5cGUYAyABKA4yKy5wcm90by5Qcm90b0NvbnRyb2xsZXJTdGF0ZUJhdGNoLlVwZGF0ZVR5cGUSEAoIc2VxdWVuY2UYBCABKA0SVAoTYnV0dG9uX2NoYW5nZWRfbWFzaxgFIAMoCzI3LnByb3RvLlByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2guQnV0dG9uQ2hhbmdlZE1hc2tFbnRyeRIZCgxsZWZ0X3N0aWNrX3gYBiABKAVIAIgBARIZCgxsZWZ0X3N0aWNrX3kYByABKAVIAYgBARIaCg1yaWdodF9zdGlja194GAggASgFSAKIAQESGgoNcmlnaHRfc3RpY2tfeRgJIAEoBUgDiAEBEhkKDGxlZnRfdHJpZ2dlchgKIAEoBUgEiAEBEhoKDXJpZ2h0X3RyaWdnZXIYCyABKAVIBYgBARITCgZkcGFkX3gYDCABKAVIBogBARITCgZkcGFkX3kYDSABKAVIB4gBARIbCg5jaGFuZ2VkX2ZpZWxkcxgOIAEoDUgIiAEBGjgKFkJ1dHRvbkNoYW5nZWRNYXNrRW50cnkSCwoDa2V5GAEgASgFEg0KBXZhbHVlGAIgASgIOgI4ASInCgpVcGRhdGVUeXBlEg4KCkZVTExfU1RBVEUQABIJCgVERUxUQRABQg8KDV9sZWZ0X3N0aWNrX3hCDwoNX2xlZnRfc3RpY2tfeUIQCg5fcmlnaHRfc3RpY2tfeEIQCg5fcmlnaHRfc3RpY2tfeUIPCg1fbGVmdF90cmlnZ2VyQhAKDl9yaWdodF90cmlnZ2VyQgkKB19kcGFkX3hCCQoHX2RwYWRfeUIRCg9fY2hhbmdlZF9maWVsZHMiqgEKE1JUQ0ljZUNhbmRpZGF0ZUluaXQSEQoJY2FuZGlkYXRlGAEgASgJEhoKDXNkcE1MaW5lSW5kZXgYAiABKA1IAIgBARITCgZzZHBNaWQYAyABKAlIAYgBARIdChB1c2VybmFtZUZyYWdtZW50GAQgASgJSAKIAQFCEAoOX3NkcE1MaW5lSW5kZXhCCQoHX3NkcE1pZEITChFfdXNlcm5hbWVGcmFnbWVudCI2ChlSVENTZXNzaW9uRGVzY3JpcHRpb25Jbml0EgsKA3NkcBgBIAEoCRIMCgR0eXBlGAIgASgJIjkKCFByb3RvSUNFEi0KCWNhbmRpZGF0ZRgBIAEoCzIaLnByb3RvLlJUQ0ljZUNhbmRpZGF0ZUluaXQiOQoIUHJvdG9TRFASLQoDc2RwGAEgASgLMiAucHJvdG8uUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdCIYCghQcm90b1JhdxIMCgRkYXRhGAEgASgJIkUKHFByb3RvQ2xpZW50UmVxdWVzdFJvb21TdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkiRwoXUHJvdG9DbGllbnREaXNjb25uZWN0ZWQSEgoKc2Vzc2lvbl9pZBgBIAEoCRIYChBjb250cm9sbGVyX3Nsb3RzGAIgAygFIioKFVByb3RvU2VydmVyUHVzaFN0cmVhbRIRCglyb29tX25hbWUYASABKAlCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw");
/**
* MouseMove message
@@ -19,19 +19,12 @@ export const file_types: GenFile = /*@__PURE__*/
*/
export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
/**
* Fixed value "MouseMove"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -50,19 +43,12 @@ export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
*/
export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
/**
* Fixed value "MouseMoveAbs"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -81,19 +67,12 @@ export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE_
*/
export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
/**
* Fixed value "MouseWheel"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 x = 2;
* @generated from field: int32 x = 1;
*/
x: number;
/**
* @generated from field: int32 y = 3;
* @generated from field: int32 y = 2;
*/
y: number;
};
@@ -112,14 +91,7 @@ export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
*/
export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
/**
* Fixed value "MouseKeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -138,14 +110,7 @@ export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE_
*/
export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
/**
* Fixed value "MouseKeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -164,14 +129,7 @@ export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
*/
export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
/**
* Fixed value "KeyDown"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -190,14 +148,7 @@ export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
*/
export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
/**
* Fixed value "KeyUp"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* @generated from field: int32 key = 2;
* @generated from field: int32 key = 1;
*/
key: number;
};
@@ -215,26 +166,26 @@ export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
* @generated from message proto.ProtoControllerAttach
*/
export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & {
/**
* Fixed value "ControllerAttach"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* One of the following enums: "ps", "xbox" or "switch"
*
* @generated from field: string id = 2;
* @generated from field: string id = 1;
*/
id: string;
/**
* Slot number (0-3)
* Session specific slot number (0-3)
*
* @generated from field: int32 slot = 3;
* @generated from field: int32 session_slot = 2;
*/
slot: number;
sessionSlot: number;
/**
* Session ID of the client
*
* @generated from field: string session_id = 3;
*/
sessionId: string;
};
/**
@@ -251,18 +202,18 @@ export const ProtoControllerAttachSchema: GenMessage<ProtoControllerAttach> = /*
*/
export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
/**
* Fixed value "ControllerDetach"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
};
/**
@@ -272,181 +223,6 @@ export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*@__PURE__*/
messageDesc(file_types, 8);
/**
* ControllerButton message
*
* @generated from message proto.ProtoControllerButton
*/
export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & {
/**
* Fixed value "ControllerButtons"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
*/
slot: number;
/**
* Button code (linux input event code)
*
* @generated from field: int32 button = 3;
*/
button: number;
/**
* true if pressed, false if released
*
* @generated from field: bool pressed = 4;
*/
pressed: boolean;
};
/**
* Describes the message proto.ProtoControllerButton.
* Use `create(ProtoControllerButtonSchema)` to create a new message.
*/
export const ProtoControllerButtonSchema: GenMessage<ProtoControllerButton> = /*@__PURE__*/
messageDesc(file_types, 9);
/**
* ControllerTriggers message
*
* @generated from message proto.ProtoControllerTrigger
*/
export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & {
/**
* Fixed value "ControllerTriggers"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
*/
slot: number;
/**
* Trigger number (0 for left, 1 for right)
*
* @generated from field: int32 trigger = 3;
*/
trigger: number;
/**
* trigger value (-32768 to 32767)
*
* @generated from field: int32 value = 4;
*/
value: number;
};
/**
* Describes the message proto.ProtoControllerTrigger.
* Use `create(ProtoControllerTriggerSchema)` to create a new message.
*/
export const ProtoControllerTriggerSchema: GenMessage<ProtoControllerTrigger> = /*@__PURE__*/
messageDesc(file_types, 10);
/**
* ControllerSticks message
*
* @generated from message proto.ProtoControllerStick
*/
export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & {
/**
* Fixed value "ControllerStick"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
*/
slot: number;
/**
* Stick number (0 for left, 1 for right)
*
* @generated from field: int32 stick = 3;
*/
stick: number;
/**
* X axis value (-32768 to 32767)
*
* @generated from field: int32 x = 4;
*/
x: number;
/**
* Y axis value (-32768 to 32767)
*
* @generated from field: int32 y = 5;
*/
y: number;
};
/**
* Describes the message proto.ProtoControllerStick.
* Use `create(ProtoControllerStickSchema)` to create a new message.
*/
export const ProtoControllerStickSchema: GenMessage<ProtoControllerStick> = /*@__PURE__*/
messageDesc(file_types, 11);
/**
* ControllerAxis message
*
* @generated from message proto.ProtoControllerAxis
*/
export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & {
/**
* Fixed value "ControllerAxis"
*
* @generated from field: string type = 1;
*/
type: string;
/**
* Slot number (0-3)
*
* @generated from field: int32 slot = 2;
*/
slot: number;
/**
* Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
*
* @generated from field: int32 axis = 3;
*/
axis: number;
/**
* axis value (-1 to 1)
*
* @generated from field: int32 value = 4;
*/
value: number;
};
/**
* Describes the message proto.ProtoControllerAxis.
* Use `create(ProtoControllerAxisSchema)` to create a new message.
*/
export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__PURE__*/
messageDesc(file_types, 12);
/**
* ControllerRumble message
*
@@ -454,18 +230,18 @@ export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__P
*/
export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
/**
* Fixed value "ControllerRumble"
* Session specific slot number (0-3)
*
* @generated from field: string type = 1;
* @generated from field: int32 session_slot = 1;
*/
type: string;
sessionSlot: number;
/**
* Slot number (0-3)
* Session ID of the client
*
* @generated from field: int32 slot = 2;
* @generated from field: string session_id = 2;
*/
slot: number;
sessionId: string;
/**
* Low frequency rumble (0-65535)
@@ -494,108 +270,321 @@ export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
* Use `create(ProtoControllerRumbleSchema)` to create a new message.
*/
export const ProtoControllerRumbleSchema: GenMessage<ProtoControllerRumble> = /*@__PURE__*/
messageDesc(file_types, 13);
messageDesc(file_types, 9);
/**
* Union of all Input types
* ControllerStateBatch - single message containing full or partial controller state
*
* @generated from message proto.ProtoInput
* @generated from message proto.ProtoControllerStateBatch
*/
export type ProtoInput = Message<"proto.ProtoInput"> & {
export type ProtoControllerStateBatch = Message<"proto.ProtoControllerStateBatch"> & {
/**
* @generated from oneof proto.ProtoInput.input_type
* Session specific slot number (0-3)
*
* @generated from field: int32 session_slot = 1;
*/
inputType: {
/**
* @generated from field: proto.ProtoMouseMove mouse_move = 1;
*/
value: ProtoMouseMove;
case: "mouseMove";
} | {
/**
* @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
*/
value: ProtoMouseMoveAbs;
case: "mouseMoveAbs";
} | {
/**
* @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
*/
value: ProtoMouseWheel;
case: "mouseWheel";
} | {
/**
* @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
*/
value: ProtoMouseKeyDown;
case: "mouseKeyDown";
} | {
/**
* @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
*/
value: ProtoMouseKeyUp;
case: "mouseKeyUp";
} | {
/**
* @generated from field: proto.ProtoKeyDown key_down = 6;
*/
value: ProtoKeyDown;
case: "keyDown";
} | {
/**
* @generated from field: proto.ProtoKeyUp key_up = 7;
*/
value: ProtoKeyUp;
case: "keyUp";
} | {
/**
* @generated from field: proto.ProtoControllerAttach controller_attach = 8;
*/
value: ProtoControllerAttach;
case: "controllerAttach";
} | {
/**
* @generated from field: proto.ProtoControllerDetach controller_detach = 9;
*/
value: ProtoControllerDetach;
case: "controllerDetach";
} | {
/**
* @generated from field: proto.ProtoControllerButton controller_button = 10;
*/
value: ProtoControllerButton;
case: "controllerButton";
} | {
/**
* @generated from field: proto.ProtoControllerTrigger controller_trigger = 11;
*/
value: ProtoControllerTrigger;
case: "controllerTrigger";
} | {
/**
* @generated from field: proto.ProtoControllerStick controller_stick = 12;
*/
value: ProtoControllerStick;
case: "controllerStick";
} | {
/**
* @generated from field: proto.ProtoControllerAxis controller_axis = 13;
*/
value: ProtoControllerAxis;
case: "controllerAxis";
} | {
/**
* @generated from field: proto.ProtoControllerRumble controller_rumble = 14;
*/
value: ProtoControllerRumble;
case: "controllerRumble";
} | { case: undefined; value?: undefined };
sessionSlot: number;
/**
* Session ID of the client
*
* @generated from field: string session_id = 2;
*/
sessionId: string;
/**
* @generated from field: proto.ProtoControllerStateBatch.UpdateType update_type = 3;
*/
updateType: ProtoControllerStateBatch_UpdateType;
/**
* Sequence number for packet loss detection
*
* @generated from field: uint32 sequence = 4;
*/
sequence: number;
/**
* Button state map (Linux event codes)
*
* @generated from field: map<int32, bool> button_changed_mask = 5;
*/
buttonChangedMask: { [key: number]: boolean };
/**
* Analog inputs
*
* -32768 to 32767
*
* @generated from field: optional int32 left_stick_x = 6;
*/
leftStickX?: number;
/**
* -32768 to 32767
*
* @generated from field: optional int32 left_stick_y = 7;
*/
leftStickY?: number;
/**
* -32768 to 32767
*
* @generated from field: optional int32 right_stick_x = 8;
*/
rightStickX?: number;
/**
* -32768 to 32767
*
* @generated from field: optional int32 right_stick_y = 9;
*/
rightStickY?: number;
/**
* -32768 to 32767
*
* @generated from field: optional int32 left_trigger = 10;
*/
leftTrigger?: number;
/**
* -32768 to 32767
*
* @generated from field: optional int32 right_trigger = 11;
*/
rightTrigger?: number;
/**
* -1, 0, or 1
*
* @generated from field: optional int32 dpad_x = 12;
*/
dpadX?: number;
/**
* -1, 0, or 1
*
* @generated from field: optional int32 dpad_y = 13;
*/
dpadY?: number;
/**
* Bitmask indicating which fields have changed
* Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc.
*
* @generated from field: optional uint32 changed_fields = 14;
*/
changedFields?: number;
};
/**
* Describes the message proto.ProtoInput.
* Use `create(ProtoInputSchema)` to create a new message.
* Describes the message proto.ProtoControllerStateBatch.
* Use `create(ProtoControllerStateBatchSchema)` to create a new message.
*/
export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
export const ProtoControllerStateBatchSchema: GenMessage<ProtoControllerStateBatch> = /*@__PURE__*/
messageDesc(file_types, 10);
/**
* @generated from enum proto.ProtoControllerStateBatch.UpdateType
*/
export enum ProtoControllerStateBatch_UpdateType {
/**
* Complete controller state
*
* @generated from enum value: FULL_STATE = 0;
*/
FULL_STATE = 0,
/**
* Only changed fields
*
* @generated from enum value: DELTA = 1;
*/
DELTA = 1,
}
/**
* Describes the enum proto.ProtoControllerStateBatch.UpdateType.
*/
export const ProtoControllerStateBatch_UpdateTypeSchema: GenEnum<ProtoControllerStateBatch_UpdateType> = /*@__PURE__*/
enumDesc(file_types, 10, 0);
/**
* @generated from message proto.RTCIceCandidateInit
*/
export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & {
/**
* @generated from field: string candidate = 1;
*/
candidate: string;
/**
* @generated from field: optional uint32 sdpMLineIndex = 2;
*/
sdpMLineIndex?: number;
/**
* @generated from field: optional string sdpMid = 3;
*/
sdpMid?: string;
/**
* @generated from field: optional string usernameFragment = 4;
*/
usernameFragment?: string;
};
/**
* Describes the message proto.RTCIceCandidateInit.
* Use `create(RTCIceCandidateInitSchema)` to create a new message.
*/
export const RTCIceCandidateInitSchema: GenMessage<RTCIceCandidateInit> = /*@__PURE__*/
messageDesc(file_types, 11);
/**
* @generated from message proto.RTCSessionDescriptionInit
*/
export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & {
/**
* @generated from field: string sdp = 1;
*/
sdp: string;
/**
* @generated from field: string type = 2;
*/
type: string;
};
/**
* Describes the message proto.RTCSessionDescriptionInit.
* Use `create(RTCSessionDescriptionInitSchema)` to create a new message.
*/
export const RTCSessionDescriptionInitSchema: GenMessage<RTCSessionDescriptionInit> = /*@__PURE__*/
messageDesc(file_types, 12);
/**
* ProtoICE message
*
* @generated from message proto.ProtoICE
*/
export type ProtoICE = Message<"proto.ProtoICE"> & {
/**
* @generated from field: proto.RTCIceCandidateInit candidate = 1;
*/
candidate?: RTCIceCandidateInit;
};
/**
* Describes the message proto.ProtoICE.
* Use `create(ProtoICESchema)` to create a new message.
*/
export const ProtoICESchema: GenMessage<ProtoICE> = /*@__PURE__*/
messageDesc(file_types, 13);
/**
* ProtoSDP message
*
* @generated from message proto.ProtoSDP
*/
export type ProtoSDP = Message<"proto.ProtoSDP"> & {
/**
* @generated from field: proto.RTCSessionDescriptionInit sdp = 1;
*/
sdp?: RTCSessionDescriptionInit;
};
/**
* Describes the message proto.ProtoSDP.
* Use `create(ProtoSDPSchema)` to create a new message.
*/
export const ProtoSDPSchema: GenMessage<ProtoSDP> = /*@__PURE__*/
messageDesc(file_types, 14);
/**
* ProtoRaw message
*
* @generated from message proto.ProtoRaw
*/
export type ProtoRaw = Message<"proto.ProtoRaw"> & {
/**
* @generated from field: string data = 1;
*/
data: string;
};
/**
* Describes the message proto.ProtoRaw.
* Use `create(ProtoRawSchema)` to create a new message.
*/
export const ProtoRawSchema: GenMessage<ProtoRaw> = /*@__PURE__*/
messageDesc(file_types, 15);
/**
* ProtoClientRequestRoomStream message
*
* @generated from message proto.ProtoClientRequestRoomStream
*/
export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
/**
* @generated from field: string session_id = 2;
*/
sessionId: string;
};
/**
* Describes the message proto.ProtoClientRequestRoomStream.
* Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message.
*/
export const ProtoClientRequestRoomStreamSchema: GenMessage<ProtoClientRequestRoomStream> = /*@__PURE__*/
messageDesc(file_types, 16);
/**
* ProtoClientDisconnected message
*
* @generated from message proto.ProtoClientDisconnected
*/
export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & {
/**
* @generated from field: string session_id = 1;
*/
sessionId: string;
/**
* @generated from field: repeated int32 controller_slots = 2;
*/
controllerSlots: number[];
};
/**
* Describes the message proto.ProtoClientDisconnected.
* Use `create(ProtoClientDisconnectedSchema)` to create a new message.
*/
export const ProtoClientDisconnectedSchema: GenMessage<ProtoClientDisconnected> = /*@__PURE__*/
messageDesc(file_types, 17);
/**
* ProtoServerPushStream message
*
* @generated from message proto.ProtoServerPushStream
*/
export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & {
/**
* @generated from field: string room_name = 1;
*/
roomName: string;
};
/**
* Describes the message proto.ProtoServerPushStream.
* Use `create(ProtoServerPushStreamSchema)` to create a new message.
*/
export const ProtoServerPushStreamSchema: GenMessage<ProtoServerPushStream> = /*@__PURE__*/
messageDesc(file_types, 18);

81
packages/input/src/streamwrapper.ts Normal file
View File

@@ -0,0 +1,81 @@
import { pbStream, type ProtobufStream } from "@libp2p/utils";
import type { Stream } from "@libp2p/interface";
import { bufbuildAdapter } from "./utils";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBase,
} from "./proto/messages_pb";
type MessageHandler = (
data: any,
base: ProtoMessageBase,
) => void | Promise<void>;
export class P2PMessageStream {
private pb: ProtobufStream;
private handlers = new Map<string, MessageHandler[]>();
private closed = false;
private readLoopRunning = false;
constructor(stream: Stream) {
this.pb = pbStream(stream);
}
public on(payloadType: string, handler: MessageHandler): void {
if (!this.handlers.has(payloadType)) {
this.handlers.set(payloadType, []);
}
this.handlers.get(payloadType)!.push(handler);
if (!this.readLoopRunning) this.startReading().catch(console.error);
}
private async startReading(): Promise<void> {
if (this.readLoopRunning || this.closed) return;
this.readLoopRunning = true;
while (!this.closed) {
try {
const msg: ProtoMessage = await this.pb.read(
bufbuildAdapter(ProtoMessageSchema),
);
const payloadType = msg.messageBase?.payloadType;
if (payloadType && this.handlers.has(payloadType)) {
const handlers = this.handlers.get(payloadType)!;
if (msg.payload.value) {
for (const handler of handlers) {
try {
await handler(msg.payload.value, msg.messageBase);
} catch (err) {
console.error(`Error in handler for ${payloadType}:`, err);
}
}
}
}
} catch (err) {
if (this.closed) break;
console.error("Stream read error:", err);
this.close();
}
}
this.readLoopRunning = false;
}
public async write(
message: ProtoMessage,
options?: { signal?: AbortSignal },
): Promise<void> {
if (this.closed)
throw new Error("Cannot write to closed stream");
await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options);
}
public close(): void {
this.closed = true;
this.handlers.clear();
}
}

95
packages/input/src/utils.ts Normal file
View File

@@ -0,0 +1,95 @@
import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
import type { Message } from "@bufbuild/protobuf";
import { Uint8ArrayList } from "uint8arraylist";
import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
import { timestampFromDate } from "@bufbuild/protobuf/wkt";
import {
ProtoLatencyTracker,
ProtoLatencyTrackerSchema,
ProtoTimestampEntrySchema,
} from "./proto/latency_tracker_pb";
import {
ProtoMessage,
ProtoMessageSchema,
ProtoMessageBaseSchema,
} from "./proto/messages_pb";
export function bufbuildAdapter<T extends Message>(schema: GenMessage<T>) {
return {
encode: (data: T): Uint8Array => {
return toBinary(schema, data);
},
decode: (data: Uint8Array | Uint8ArrayList): T => {
// Convert Uint8ArrayList to Uint8Array if needed
const bytes = data instanceof Uint8ArrayList ? data.subarray() : data;
return fromBinary(schema, bytes);
},
};
}
// Latency tracker helpers
export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker {
return create(ProtoLatencyTrackerSchema, {
sequenceId: sequenceId || crypto.randomUUID(),
timestamps: [],
});
}
export function addLatencyTimestamp(
tracker: ProtoLatencyTracker,
stage: string,
): ProtoLatencyTracker {
const entry = create(ProtoTimestampEntrySchema, {
stage,
time: timestampFromDate(new Date()),
});
return {
...tracker,
timestamps: [...tracker.timestamps, entry],
};
}
interface CreateMessageOptions {
sequenceId?: string;
}
function derivePayloadCase(data: Message): string {
// Extract case from $typeName: "proto.ProtoICE" -> "ice"
// "proto.ProtoControllerAttach" -> "controllerAttach"
const typeName = data.$typeName;
if (!typeName)
throw new Error("Message has no $typeName");
// Remove "proto.Proto" prefix and convert first char to lowercase
const caseName = typeName.replace(/^proto\.Proto/, "");
// Convert PascalCase to camelCase
// If it's all caps (like SDP, ICE), lowercase everything
// Otherwise, just lowercase the first character
if (caseName === caseName.toUpperCase()) {
return caseName.toLowerCase();
}
return caseName.charAt(0).toLowerCase() + caseName.slice(1);
}
export function createMessage(
data: Message,
payloadType: string,
options?: CreateMessageOptions,
): ProtoMessage {
const payloadCase = derivePayloadCase(data);
return create(ProtoMessageSchema, {
messageBase: create(ProtoMessageBaseSchema, {
payloadType,
latency: options?.sequenceId
? createLatencyTracker(options.sequenceId)
: undefined,
}),
payload: {
case: payloadCase,
value: data,
} as any, // Type assertion needed for dynamic case
});
}

129
packages/input/src/webrtc-stream.ts
View File

@@ -1,9 +1,3 @@
import {
NewMessageRaw,
NewMessageSDP,
NewMessageICE,
SafeStream,
} from "./messages";
import { webSockets } from "@libp2p/websockets";
import { webTransport } from "@libp2p/webtransport";
import { createLibp2p, Libp2p } from "libp2p";
@@ -13,19 +7,33 @@ import { identify } from "@libp2p/identify";
import { multiaddr } from "@multiformats/multiaddr";
import { Connection } from "@libp2p/interface";
import { ping } from "@libp2p/ping";
import { createMessage } from "./utils";
import { create } from "@bufbuild/protobuf";
import {
ProtoClientRequestRoomStream,
ProtoClientRequestRoomStreamSchema,
ProtoICE,
ProtoICESchema, ProtoRaw,
ProtoSDP,
ProtoSDPSchema
} from "./proto/types_pb";
import { P2PMessageStream } from "./streamwrapper";
const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
export class WebRTCStream {
private _sessionId: string | null = null;
private _p2p: Libp2p | undefined = undefined;
private _p2pConn: Connection | undefined = undefined;
private _p2pSafeStream: SafeStream | undefined = undefined;
private _msgStream: P2PMessageStream | undefined = undefined;
private _pc: RTCPeerConnection | undefined = undefined;
private _audioTrack: MediaStreamTrack | undefined = undefined;
private _videoTrack: MediaStreamTrack | undefined = undefined;
private _dataChannel: RTCDataChannel | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined =
undefined;
private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined =
undefined;
private _serverURL: string | undefined = undefined;
private _roomName: string | undefined = undefined;
private _isConnected: boolean = false;
@@ -89,14 +97,20 @@ export class WebRTCStream {
.newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
.catch(console.error);
if (stream) {
this._p2pSafeStream = new SafeStream(stream);
this._msgStream = new P2PMessageStream(stream);
console.log("Stream opened with peer");
let iceHolder: RTCIceCandidateInit[] = [];
this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
this._msgStream.on("ice-candidate", (data: ProtoICE) => {
const cand: RTCIceCandidateInit = {
candidate: data.candidate.candidate,
sdpMLineIndex: data.candidate.sdpMLineIndex,
sdpMid: data.candidate.sdpMid,
usernameFragment: data.candidate.usernameFragment,
};
if (this._pc) {
if (this._pc.remoteDescription) {
this._pc.addIceCandidate(data.candidate).catch((err) => {
this._pc.addIceCandidate(cand).catch((err) => {
console.error("Error adding ICE candidate:", err);
});
// Add held candidates
@@ -107,45 +121,78 @@ export class WebRTCStream {
});
iceHolder = [];
} else {
iceHolder.push(data.candidate);
iceHolder.push(cand);
}
} else {
iceHolder.push(data.candidate);
}
});
this._p2pSafeStream.registerCallback("offer", async (data) => {
this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => {
this._sessionId = data.sessionId;
localStorage.setItem("nestri-session-id", this._sessionId);
console.log("Session ID assigned:", this._sessionId, "for room:", data.roomName);
});
this._msgStream.on("offer", async (data: ProtoSDP) => {
if (!this._pc) {
// Setup peer connection now
this._setupPeerConnection();
}
await this._pc!.setRemoteDescription(data.sdp);
await this._pc!.setRemoteDescription({
sdp: data.sdp.sdp,
type: data.sdp.type as RTCSdpType,
});
// Add held candidates
iceHolder.forEach((candidate) => {
this._pc!.addIceCandidate(candidate).catch((err) => {
console.error("Error adding held ICE candidate:", err);
});
});
iceHolder = [];
// Create our answer
const answer = await this._pc!.createAnswer();
// Force stereo in Chromium browsers
answer.sdp = this.forceOpusStereo(answer.sdp!);
await this._pc!.setLocalDescription(answer);
// Send answer back
const answerMsg = NewMessageSDP("answer", answer);
await this._p2pSafeStream?.writeMessage(answerMsg);
const answerMsg = createMessage(
create(ProtoSDPSchema, {
sdp: answer,
}),
"answer",
);
await this._msgStream?.write(answerMsg);
});
this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
console.warn("Stream is offline for room:", data.roomName);
this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => {
console.warn("Stream is offline for room:", msg.data);
this._onConnected?.(null);
});
const clientId = this.getSessionID();
if (clientId) {
console.debug("Using existing session ID:", clientId);
}
// Send stream request
// marshal room name into json
const request = NewMessageRaw(
const requestMsg = createMessage(
create(ProtoClientRequestRoomStreamSchema, {
roomName: roomName,
sessionId: clientId ?? "",
}),
"request-stream-room",
roomName,
);
await this._p2pSafeStream.writeMessage(request);
await this._msgStream.write(requestMsg);
}
}
}
public getSessionID(): string | null {
if (this._sessionId === null)
this._sessionId = localStorage.getItem("nestri-session-id");
return this._sessionId;
}
// Forces opus to stereo in Chromium browsers, because of course
private forceOpusStereo(SDP: string): string {
// Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
@@ -200,11 +247,16 @@ export class WebRTCStream {
this._pc.onicecandidate = (e) => {
if (e.candidate) {
const iceMsg = NewMessageICE("ice-candidate", e.candidate);
if (this._p2pSafeStream) {
this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
console.error("Error sending ICE candidate:", err),
);
const iceMsg = createMessage(
create(ProtoICESchema, {
candidate: e.candidate,
}),
"ice-candidate",
);
if (this._msgStream) {
this._msgStream
.write(iceMsg)
.catch((err) => console.error("Error sending ICE candidate:", err));
} else {
console.warn("P2P stream not established, cannot send ICE candidate");
}
@@ -218,8 +270,7 @@ export class WebRTCStream {
}
private _checkConnectionState() {
if (!this._pc || !this._p2p || !this._p2pConn)
return;
if (!this._pc || !this._p2p || !this._p2pConn) return;
console.debug("Checking connection state:", {
connectionState: this._pc.connectionState,
@@ -256,7 +307,7 @@ export class WebRTCStream {
// @ts-ignore
receiver.jitterBufferTarget = receiver.jitterBufferDelayHint = receiver.playoutDelayHint = 0;
}
}, 15);
}, 50);
});
}
}
@@ -286,7 +337,9 @@ export class WebRTCStream {
// Attempt to reconnect only if not already connected
if (!this._isConnected && this._serverURL && this._roomName) {
this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
this._setup(this._serverURL, this._roomName).catch((err) =>
console.error("Reconnection failed:", err),
);
}
}
@@ -335,7 +388,9 @@ export class WebRTCStream {
}
public removeDataChannelCallback(callback: (data: any) => void) {
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback);
this._dataChannelCallbacks = this._dataChannelCallbacks.filter(
(cb) => cb !== callback,
);
}
private _setupDataChannelEvents() {
@@ -343,7 +398,7 @@ export class WebRTCStream {
this._dataChannel.onclose = () => console.log("sendChannel has closed");
this._dataChannel.onopen = () => console.log("sendChannel has opened");
this._dataChannel.onmessage = (event => {
this._dataChannel.onmessage = (event) => {
// Parse as ProtoBuf message
const data = event.data;
// Call registered callback if exists
@@ -354,7 +409,7 @@ export class WebRTCStream {
console.error("Error in data channel callback:", err);
}
});
});
};
}
private _gatherFrameRate() {