diff --git a/packages/input/src/mouse.ts b/packages/input/src/mouse.ts index e1848b94..db0388bd 100644 --- a/packages/input/src/mouse.ts +++ b/packages/input/src/mouse.ts @@ -25,7 +25,13 @@ export class Mouse { protected connected!: boolean; // Store references to event listeners + private sendInterval = 16 //60fps + private readonly mousemoveListener: (e: MouseEvent) => void; + private movementX: number = 0; + private movementY: number = 0; + private isProcessing: boolean = false; + private readonly mousedownListener: (e: MouseEvent) => void; private readonly mouseupListener: (e: MouseEvent) => void; private readonly mousewheelListener: (e: WheelEvent) => void; @@ -34,17 +40,15 @@ export class Mouse { this.wrtc = webrtc; this.canvas = canvas; - this.mousemoveListener = this.createMouseListener((e: any) => create(ProtoInputSchema, { - $typeName: "proto.ProtoInput", - inputType: { - case: "mouseMove", - value: create(ProtoMouseMoveSchema, { - type: "MouseMove", - x: e.movementX, - y: e.movementY - }), - } - })); + this.sendInterval = 1000 / webrtc.currentFrameRate + + this.mousemoveListener = (e: MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + this.movementX += e.movementX; + this.movementY += e.movementY; + }; + this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, { $typeName: "proto.ProtoInput", inputType: { @@ -78,6 +82,7 @@ export class Mouse { })); this.run() + this.startProcessing(); } private run() { @@ -113,6 +118,57 @@ export class Mouse { this.connected = false; } + private startProcessing() { + setInterval(() => { + if (this.connected && (this.movementX !== 0 || this.movementY !== 0)) { + this.sendAggregatedMouseMove(); + this.movementX = 0; + this.movementY = 0; + } + }, this.sendInterval); + } + + private sendAggregatedMouseMove() { + const data = create(ProtoInputSchema, { + $typeName: "proto.ProtoInput", + inputType: { + case: "mouseMove", + value: create(ProtoMouseMoveSchema, { + type: "MouseMove", + x: this.movementX, + y: this.movementY, + }), + }, + }); + + // Latency tracking + const tracker = new LatencyTracker("input-mouse"); + tracker.addTimestamp("client_send"); + const protoTracker: ProtoLatencyTracker = { + $typeName: "proto.ProtoLatencyTracker", + sequenceId: tracker.sequence_id, + timestamps: [], + }; + for (const t of tracker.timestamps) { + protoTracker.timestamps.push({ + $typeName: "proto.ProtoTimestampEntry", + stage: t.stage, + time: timestampFromDate(t.time), + } as ProtoTimestampEntry); + } + + const message: ProtoMessageInput = { + $typeName: "proto.ProtoMessageInput", + messageBase: { + $typeName: "proto.ProtoMessageBase", + payloadType: "input", + latency: protoTracker, + } as ProtoMessageBase, + data: data, + }; + this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message)); + } + // Helper function to create and return mouse listeners private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void { return (e: Event) => { diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts index 51b8d4f1..333ab759 100644 --- a/packages/input/src/webrtc-stream.ts +++ b/packages/input/src/webrtc-stream.ts @@ -22,6 +22,7 @@ export class WebRTCStream { private _serverURL: string | undefined = undefined; private _roomName: string | undefined = undefined; private _isConnected: boolean = false; // Add flag to track connection state + currentFrameRate: number = 60; constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) { if (roomName.length <= 0) { @@ -219,6 +220,8 @@ export class WebRTCStream { this._onConnected(this._mediaStream); } } + + this._gatherFrameRate(); } else if (this._pc.connectionState === "failed" || this._pc.connectionState === "closed" || this._pc.iceConnectionState === "failed") { @@ -297,6 +300,35 @@ export class WebRTCStream { this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`) } + private _gatherFrameRate() { + if (this._pc === undefined || this._mediaStream === undefined) + return; + + const videoInfoPromise = new Promise<{ fps: number}>((resolve) => { + const track = this._mediaStream!.getVideoTracks()[0]; + // Keep trying to get fps until it's found + const interval = setInterval(async () => { + if (this._pc === undefined) { + clearInterval(interval); + return; + } + + const stats = await this._pc!.getStats(track); + stats.forEach((report) => { + if (report.type === "inbound-rtp") { + clearInterval(interval); + + resolve({ fps: report.framesPerSecond }); + } + }); + }, 250); + }); + + videoInfoPromise.then((value) => { + this.currentFrameRate = value.fps + }) + } + // Send binary message through the data channel public sendBinary(data: Uint8Array) { if (this._dataChannel && this._dataChannel.readyState === "open")