⭐ feat: Aggregated mouse movements (#185)

Mouse Movements are aggreagated to send out with an fixed interval

This reduces sending mouse movement events especially when the mouse is
moved really fast. The mouse events are not lost they will be summed and
aggregated to one single mouse movement and then send with the fixed
interval

packages/input/src/mouse.ts

@@ -25,7 +25,13 @@ export class Mouse {
   protected connected!: boolean;
 
   // Store references to event listeners
+  private sendInterval = 16 //60fps
+
   private readonly mousemoveListener: (e: MouseEvent) => void;
+  private movementX: number = 0;
+  private movementY: number = 0;
+  private isProcessing: boolean = false;
+
   private readonly mousedownListener: (e: MouseEvent) => void;
   private readonly mouseupListener: (e: MouseEvent) => void;
   private readonly mousewheelListener: (e: WheelEvent) => void;
@@ -34,17 +40,15 @@ export class Mouse {
     this.wrtc = webrtc;
     this.canvas = canvas;
 
-    this.mousemoveListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+    this.sendInterval = 1000 / webrtc.currentFrameRate
-      $typeName: "proto.ProtoInput",
+
-      inputType: {
+    this.mousemoveListener = (e: MouseEvent) => {
-        case: "mouseMove",
+      e.preventDefault();
-        value: create(ProtoMouseMoveSchema, {
+      e.stopPropagation();
-          type: "MouseMove",
+      this.movementX += e.movementX;
-          x: e.movementX,
+      this.movementY += e.movementY;
-          y: e.movementY
+    };
-        }),
+
-      }
-    }));
     this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
       $typeName: "proto.ProtoInput",
       inputType: {
@@ -78,6 +82,7 @@ export class Mouse {
     }));
 
     this.run()
+    this.startProcessing();
   }
 
   private run() {
@@ -113,6 +118,57 @@ export class Mouse {
     this.connected = false;
   }
 
+  private startProcessing() {
+    setInterval(() => {
+      if (this.connected && (this.movementX !== 0 || this.movementY !== 0)) {
+        this.sendAggregatedMouseMove();
+        this.movementX = 0;
+        this.movementY = 0;
+      }
+    }, this.sendInterval);
+  }
+
+  private sendAggregatedMouseMove() {
+    const data = create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseMove",
+        value: create(ProtoMouseMoveSchema, {
+          type: "MouseMove",
+          x: this.movementX,
+          y: this.movementY,
+        }),
+      },
+    });
+
+    // Latency tracking
+    const tracker = new LatencyTracker("input-mouse");
+    tracker.addTimestamp("client_send");
+    const protoTracker: ProtoLatencyTracker = {
+      $typeName: "proto.ProtoLatencyTracker",
+      sequenceId: tracker.sequence_id,
+      timestamps: [],
+    };
+    for (const t of tracker.timestamps) {
+      protoTracker.timestamps.push({
+        $typeName: "proto.ProtoTimestampEntry",
+        stage: t.stage,
+        time: timestampFromDate(t.time),
+      } as ProtoTimestampEntry);
+    }
+
+    const message: ProtoMessageInput = {
+      $typeName: "proto.ProtoMessageInput",
+      messageBase: {
+        $typeName: "proto.ProtoMessageBase",
+        payloadType: "input",
+        latency: protoTracker,
+      } as ProtoMessageBase,
+      data: data,
+    };
+    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
+  }
+
   // Helper function to create and return mouse listeners
   private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {

packages/input/src/webrtc-stream.ts

@@ -22,6 +22,7 @@ export class WebRTCStream {
   private _serverURL: string | undefined = undefined;
   private _roomName: string | undefined = undefined;
   private _isConnected: boolean = false; // Add flag to track connection state
+  currentFrameRate: number = 60;
 
   constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) {
     if (roomName.length <= 0) {
@@ -219,6 +220,8 @@ export class WebRTCStream {
           this._onConnected(this._mediaStream);
         }
       }
+
+      this._gatherFrameRate();
     } else if (this._pc.connectionState === "failed" ||
       this._pc.connectionState === "closed" ||
       this._pc.iceConnectionState === "failed") {
@@ -297,6 +300,35 @@ export class WebRTCStream {
     this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`)
   }
 
+  private _gatherFrameRate() {
+    if (this._pc === undefined || this._mediaStream === undefined)
+      return;
+
+    const videoInfoPromise = new Promise<{ fps: number}>((resolve) => {
+      const track = this._mediaStream!.getVideoTracks()[0];
+      // Keep trying to get fps until it's found
+      const interval = setInterval(async () => {
+        if (this._pc === undefined) {
+          clearInterval(interval);
+          return;
+        }
+
+        const stats = await this._pc!.getStats(track);
+        stats.forEach((report) => {
+          if (report.type === "inbound-rtp") {
+            clearInterval(interval);
+            
+            resolve({ fps: report.framesPerSecond });
+          }
+        });
+      }, 250);
+    });
+
+    videoInfoPromise.then((value) => {
+      this.currentFrameRate = value.fps
+    }) 
+  }
+
   // Send binary message through the data channel
   public sendBinary(data: Uint8Array) {
     if (this._dataChannel && this._dataChannel.readyState === "open")