⭐ feat: protobuf input messaging (#165)

Replace json protocol by protobuf
generate protobuf files with `bun buf generate` or just `buf generate`

- [x]  Implement all datatypes with proto files

- [x] Map to ts types or use the generated proto types directly with:
   - [x] web frontend
   - [x] relay
   - [x] runner

- [ ] final performance test (to be done when CI builds new images)

---------

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>

packages/input/src/keyboard.ts

@@ -1,8 +1,17 @@
-import {type Input} from "./types"
 import {keyCodeToLinuxEventCode} from "./codes"
-import {MessageInput, encodeMessage} from "./messages";
 import {WebRTCStream} from "./webrtc-stream";
 import {LatencyTracker} from "./latency";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput,
+  ProtoInputSchema,
+  ProtoKeyDownSchema,
+  ProtoKeyUpSchema,
+  ProtoMouseMoveSchema
+} from "./proto/types_pb";
+import {create, toBinary} from "@bufbuild/protobuf";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -15,19 +24,31 @@ export class Keyboard {
   protected connected!: boolean;
 
   // Store references to event listeners
-  private keydownListener: (e: KeyboardEvent) => void;
-  private keyupListener: (e: KeyboardEvent) => void;
+  private readonly keydownListener: (e: KeyboardEvent) => void;
+  private readonly keyupListener: (e: KeyboardEvent) => void;
 
   constructor({webrtc, canvas}: Props) {
     this.wrtc = webrtc;
     this.canvas = canvas;
-    this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
-      type: "KeyDown",
-      key: this.keyToVirtualKeyCode(e.code)
+    this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyDown",
+        value: create(ProtoKeyDownSchema, {
+          type: "KeyDown",
+          key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
     }));
-    this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
-      type: "KeyUp",
-      key: this.keyToVirtualKeyCode(e.code)
+    this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyUp",
+        value: create(ProtoKeyUpSchema, {
+          type: "KeyUp",
+          key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
     }));
     this.run()
   }
@@ -59,7 +80,7 @@ export class Keyboard {
   }
 
   // Helper function to create and return mouse listeners
-  private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+  private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
@@ -67,18 +88,34 @@ export class Keyboard {
       if ((e as any).repeat)
         return;
 
-      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
-      const dataString = JSON.stringify({...data, type} as Input);
+      const data = dataCreator(e as any);
 
       // Latency tracking
       const tracker = new LatencyTracker("input-keyboard");
       tracker.addTimestamp("client_send");
-      const message: MessageInput = {
-        payload_type: "input",
-        data: dataString,
-        latency: tracker,
+      const protoTracker: ProtoLatencyTracker = {
+        $typeName: "proto.ProtoLatencyTracker",
+        sequenceId: tracker.sequence_id,
+        timestamps: [],
       };
-      this.wrtc.sendBinary(encodeMessage(message));
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }