From 2bbd705af93b18b9fdeec0336dcf9a76d015b50e Mon Sep 17 00:00:00 2001
From: Philipp Neumann <3daquawolf@gmail.com>
Date: Fri, 14 Feb 2025 16:53:41 +0100
Subject: [PATCH] =?UTF-8?q?=E2=AD=90=20feat:=20Aggregated=20mouse=20moveme?=
 =?UTF-8?q?nts=20(#185)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mouse Movements are aggreagated to send out with an fixed interval

This reduces sending mouse movement events especially when the mouse is
moved really fast. The mouse events are not lost they will be summed and
aggregated to one single mouse movement and then send with the fixed
interval
---
 packages/input/src/mouse.ts         | 78 +++++++++++++++++++++++++----
 packages/input/src/webrtc-stream.ts | 32 ++++++++++++
 2 files changed, 99 insertions(+), 11 deletions(-)

diff --git a/packages/input/src/mouse.ts b/packages/input/src/mouse.ts
index e1848b94..db0388bd 100644
--- a/packages/input/src/mouse.ts
+++ b/packages/input/src/mouse.ts
@@ -25,7 +25,13 @@ export class Mouse {
   protected connected!: boolean;
 
   // Store references to event listeners
+  private sendInterval = 16 //60fps
+
   private readonly mousemoveListener: (e: MouseEvent) => void;
+  private movementX: number = 0;
+  private movementY: number = 0;
+  private isProcessing: boolean = false;
+
   private readonly mousedownListener: (e: MouseEvent) => void;
   private readonly mouseupListener: (e: MouseEvent) => void;
   private readonly mousewheelListener: (e: WheelEvent) => void;
@@ -34,17 +40,15 @@ export class Mouse {
     this.wrtc = webrtc;
     this.canvas = canvas;
 
-    this.mousemoveListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
-      $typeName: "proto.ProtoInput",
-      inputType: {
-        case: "mouseMove",
-        value: create(ProtoMouseMoveSchema, {
-          type: "MouseMove",
-          x: e.movementX,
-          y: e.movementY
-        }),
-      }
-    }));
+    this.sendInterval = 1000 / webrtc.currentFrameRate
+
+    this.mousemoveListener = (e: MouseEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      this.movementX += e.movementX;
+      this.movementY += e.movementY;
+    };
+
     this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
       $typeName: "proto.ProtoInput",
       inputType: {
@@ -78,6 +82,7 @@ export class Mouse {
     }));
 
     this.run()
+    this.startProcessing();
   }
 
   private run() {
@@ -113,6 +118,57 @@ export class Mouse {
     this.connected = false;
   }
 
+  private startProcessing() {
+    setInterval(() => {
+      if (this.connected && (this.movementX !== 0 || this.movementY !== 0)) {
+        this.sendAggregatedMouseMove();
+        this.movementX = 0;
+        this.movementY = 0;
+      }
+    }, this.sendInterval);
+  }
+
+  private sendAggregatedMouseMove() {
+    const data = create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseMove",
+        value: create(ProtoMouseMoveSchema, {
+          type: "MouseMove",
+          x: this.movementX,
+          y: this.movementY,
+        }),
+      },
+    });
+
+    // Latency tracking
+    const tracker = new LatencyTracker("input-mouse");
+    tracker.addTimestamp("client_send");
+    const protoTracker: ProtoLatencyTracker = {
+      $typeName: "proto.ProtoLatencyTracker",
+      sequenceId: tracker.sequence_id,
+      timestamps: [],
+    };
+    for (const t of tracker.timestamps) {
+      protoTracker.timestamps.push({
+        $typeName: "proto.ProtoTimestampEntry",
+        stage: t.stage,
+        time: timestampFromDate(t.time),
+      } as ProtoTimestampEntry);
+    }
+
+    const message: ProtoMessageInput = {
+      $typeName: "proto.ProtoMessageInput",
+      messageBase: {
+        $typeName: "proto.ProtoMessageBase",
+        payloadType: "input",
+        latency: protoTracker,
+      } as ProtoMessageBase,
+      data: data,
+    };
+    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
+  }
+
   // Helper function to create and return mouse listeners
   private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
diff --git a/packages/input/src/webrtc-stream.ts b/packages/input/src/webrtc-stream.ts
index 51b8d4f1..333ab759 100644
--- a/packages/input/src/webrtc-stream.ts
+++ b/packages/input/src/webrtc-stream.ts
@@ -22,6 +22,7 @@ export class WebRTCStream {
   private _serverURL: string | undefined = undefined;
   private _roomName: string | undefined = undefined;
   private _isConnected: boolean = false; // Add flag to track connection state
+  currentFrameRate: number = 60;
 
   constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) {
     if (roomName.length <= 0) {
@@ -219,6 +220,8 @@ export class WebRTCStream {
           this._onConnected(this._mediaStream);
         }
       }
+
+      this._gatherFrameRate();
     } else if (this._pc.connectionState === "failed" ||
       this._pc.connectionState === "closed" ||
       this._pc.iceConnectionState === "failed") {
@@ -297,6 +300,35 @@ export class WebRTCStream {
     this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`)
   }
 
+  private _gatherFrameRate() {
+    if (this._pc === undefined || this._mediaStream === undefined)
+      return;
+
+    const videoInfoPromise = new Promise<{ fps: number}>((resolve) => {
+      const track = this._mediaStream!.getVideoTracks()[0];
+      // Keep trying to get fps until it's found
+      const interval = setInterval(async () => {
+        if (this._pc === undefined) {
+          clearInterval(interval);
+          return;
+        }
+
+        const stats = await this._pc!.getStats(track);
+        stats.forEach((report) => {
+          if (report.type === "inbound-rtp") {
+            clearInterval(interval);
+            
+            resolve({ fps: report.framesPerSecond });
+          }
+        });
+      }, 250);
+    });
+
+    videoInfoPromise.then((value) => {
+      this.currentFrameRate = value.fps
+    }) 
+  }
+
   // Send binary message through the data channel
   public sendBinary(data: Uint8Array) {
     if (this._dataChannel && this._dataChannel.readyState === "open")