⭐ feat: Add protobuf (#171)

This is a second attempt to add protobuf to Nestri, after the first one failed --------- Co-authored-by: Philipp Neumann <3daquawolf@gmail.com> Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
2025-12-12 16:55:37 +02:00 · 2025-01-29 04:16:27 +03:00
parent be6ea11052
commit c2363b0bce
42 changed files with 3114 additions and 854 deletions
--- a/packages/input/src/keyboard.ts
+++ b/packages/input/src/keyboard.ts
@@ -1,8 +1,17 @@
-import {type Input} from "./types"
 import {keyCodeToLinuxEventCode} from "./codes"
-import {MessageInput, encodeMessage} from "./messages";
 import {WebRTCStream} from "./webrtc-stream";
 import {LatencyTracker} from "./latency";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput,
+  ProtoInputSchema,
+  ProtoKeyDownSchema,
+  ProtoKeyUpSchema,
+  ProtoMouseMoveSchema
+} from "./proto/types_pb";
+import {create, toBinary} from "@bufbuild/protobuf";

 interface Props {
  webrtc: WebRTCStream;
@@ -15,19 +24,31 @@ export class Keyboard {
  protected connected!: boolean;

  // Store references to event listeners
-  private keydownListener: (e: KeyboardEvent) => void;
-  private keyupListener: (e: KeyboardEvent) => void;
+  private readonly keydownListener: (e: KeyboardEvent) => void;
+  private readonly keyupListener: (e: KeyboardEvent) => void;

  constructor({webrtc, canvas}: Props) {
    this.wrtc = webrtc;
    this.canvas = canvas;
-    this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
-      type: "KeyDown",
-      key: this.keyToVirtualKeyCode(e.code)
+    this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyDown",
+        value: create(ProtoKeyDownSchema, {
+          type: "KeyDown",
+          key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
    }));
-    this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
-      type: "KeyUp",
-      key: this.keyToVirtualKeyCode(e.code)
+    this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyUp",
+        value: create(ProtoKeyUpSchema, {
+          type: "KeyUp",
+          key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
    }));
    this.run()
  }
@@ -59,7 +80,7 @@ export class Keyboard {
  }

  // Helper function to create and return mouse listeners
-  private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+  private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
    return (e: Event) => {
      e.preventDefault();
      e.stopPropagation();
@@ -67,18 +88,34 @@ export class Keyboard {
      if ((e as any).repeat)
        return;

-      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
-      const dataString = JSON.stringify({...data, type} as Input);
+      const data = dataCreator(e as any);

      // Latency tracking
      const tracker = new LatencyTracker("input-keyboard");
      tracker.addTimestamp("client_send");
-      const message: MessageInput = {
-        payload_type: "input",
-        data: dataString,
-        latency: tracker,
+      const protoTracker: ProtoLatencyTracker = {
+        $typeName: "proto.ProtoLatencyTracker",
+        sequenceId: tracker.sequence_id,
+        timestamps: [],
      };
-      this.wrtc.sendBinary(encodeMessage(message));
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
    };
  }

--- a/packages/input/src/latency.ts
+++ b/packages/input/src/latency.ts
@@ -6,12 +6,10 @@ type TimestampEntry = {
 export class LatencyTracker {
  sequence_id: string;
  timestamps: TimestampEntry[];
-  metadata?: Record<string, any>;

-  constructor(sequence_id: string, timestamps: TimestampEntry[] = [], metadata: Record<string, any> = {}) {
+  constructor(sequence_id: string, timestamps: TimestampEntry[] = []) {
    this.sequence_id = sequence_id;
    this.timestamps = timestamps;
-    this.metadata = metadata;
  }

  addTimestamp(stage: string): void {
@@ -40,7 +38,6 @@ export class LatencyTracker {
        // Fill nanoseconds with zeros to match the expected format
        time: entry.time.toISOString().replace(/\.(\d+)Z$/, ".$1000000Z"),
      })),
-      metadata: this.metadata,
    };
  }

@@ -49,6 +46,6 @@ export class LatencyTracker {
      stage: ts.stage,
      time: new Date(ts.time),
    }));
-    return new LatencyTracker(json.sequence_id, timestamps, json.metadata);
+    return new LatencyTracker(json.sequence_id, timestamps);
  }
 }
--- a/packages/input/src/messages.ts
+++ b/packages/input/src/messages.ts
@@ -1,13 +1,7 @@
-import {gzip, ungzip} from "pako";
 import {LatencyTracker} from "./latency";

 export interface MessageBase {
  payload_type: string;
-}
-
-export interface MessageInput extends MessageBase {
-  payload_type: "input";
-  data: string;
  latency?: LatencyTracker;
 }

@@ -41,33 +35,3 @@ export interface MessageAnswer extends MessageBase {
  payload_type: "answer";
  answer_type: AnswerType;
 }
-
-function blobToUint8Array(blob: Blob): Promise<Uint8Array> {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onloadend = () => {
-      const arrayBuffer = reader.result as ArrayBuffer;
-      resolve(new Uint8Array(arrayBuffer));
-    };
-    reader.onerror = reject;
-    reader.readAsArrayBuffer(blob);
-  });
-}
-
-export function encodeMessage<T>(message: T): Uint8Array {
-  // Convert the message to JSON string
-  const json = JSON.stringify(message);
-  // Compress the JSON string using gzip
-  return gzip(json);
-}
-
-export async function decodeMessage<T>(data: Blob): Promise<T> {
-  // Convert the Blob to Uint8Array
-  const array = await blobToUint8Array(data);
-  // Decompress the gzip data
-  const decompressed = ungzip(array);
-  // Convert the Uint8Array to JSON string
-  const json = new TextDecoder().decode(decompressed);
-  // Parse the JSON string
-  return JSON.parse(json);
-}
--- a/packages/input/src/mouse.ts
+++ b/packages/input/src/mouse.ts
@@ -1,8 +1,18 @@
-import {type Input} from "./types"
-import {mouseButtonToLinuxEventCode} from "./codes"
-import {MessageInput, encodeMessage} from "./messages";
 import {WebRTCStream} from "./webrtc-stream";
 import {LatencyTracker} from "./latency";
+import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput, ProtoInputSchema,
+  ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
+  ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
+  ProtoMouseMove,
+  ProtoMouseMoveSchema,
+  ProtoMouseWheel, ProtoMouseWheelSchema
+} from "./proto/types_pb";
+import {mouseButtonToLinuxEventCode} from "./codes";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {create, toBinary} from "@bufbuild/protobuf";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";

 interface Props {
  webrtc: WebRTCStream;
@@ -15,33 +25,56 @@ export class Mouse {
  protected connected!: boolean;

  // Store references to event listeners
-  private mousemoveListener: (e: MouseEvent) => void;
-  private mousedownListener: (e: MouseEvent) => void;
-  private mouseupListener: (e: MouseEvent) => void;
-  private mousewheelListener: (e: WheelEvent) => void;
+  private readonly mousemoveListener: (e: MouseEvent) => void;
+  private readonly mousedownListener: (e: MouseEvent) => void;
+  private readonly mouseupListener: (e: MouseEvent) => void;
+  private readonly mousewheelListener: (e: WheelEvent) => void;

  constructor({webrtc, canvas}: Props) {
    this.wrtc = webrtc;
    this.canvas = canvas;

-    this.mousemoveListener = this.createMouseListener("mousemove", (e: any) => ({
-      type: "MouseMove",
-      x: e.movementX,
-      y: e.movementY
+    this.mousemoveListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseMove",
+        value: create(ProtoMouseMoveSchema, {
+          type: "MouseMove",
+          x: e.movementX,
+          y: e.movementY
+        }),
+      }
    }));
-    this.mousedownListener = this.createMouseListener("mousedown", (e: any) => ({
-      type: "MouseKeyDown",
-      key: this.keyToVirtualKeyCode(e.button)
+    this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyDown",
+        value: create(ProtoMouseKeyDownSchema, {
+          type: "MouseKeyDown",
+          key: this.keyToVirtualKeyCode(e.button)
+        }),
+      }
    }));
-
-    this.mouseupListener = this.createMouseListener("mouseup", (e: any) => ({
-      type: "MouseKeyUp",
-      key: this.keyToVirtualKeyCode(e.button)
+    this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyUp",
+        value: create(ProtoMouseKeyUpSchema, {
+          type: "MouseKeyUp",
+          key: this.keyToVirtualKeyCode(e.button)
+        }),
+      }
    }));
-    this.mousewheelListener = this.createMouseListener("wheel", (e: any) => ({
-      type: "MouseWheel",
-      x: e.deltaX,
-      y: e.deltaY
+    this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseWheel",
+        value: create(ProtoMouseWheelSchema, {
+          type: "MouseWheel",
+          x: e.deltaX,
+          y: e.deltaY
+        }),
+      }
    }));

    this.run()
@@ -59,10 +92,10 @@ export class Mouse {

    if (document.pointerLockElement == this.canvas) {
      this.connected = true
-      this.canvas.addEventListener("mousemove", this.mousemoveListener, { passive: false });
-      this.canvas.addEventListener("mousedown", this.mousedownListener, { passive: false });
-      this.canvas.addEventListener("mouseup", this.mouseupListener, { passive: false });
-      this.canvas.addEventListener("wheel", this.mousewheelListener, { passive: false });
+      this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
+      this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
+      this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
+      this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});

    } else {
      if (this.connected) {
@@ -81,22 +114,38 @@ export class Mouse {
  }

  // Helper function to create and return mouse listeners
-  private createMouseListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+  private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
    return (e: Event) => {
      e.preventDefault();
      e.stopPropagation();
-      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
-      const dataString = JSON.stringify({...data, type} as Input);
+      const data = dataCreator(e as any);

      // Latency tracking
      const tracker = new LatencyTracker("input-mouse");
      tracker.addTimestamp("client_send");
-      const message: MessageInput = {
-        payload_type: "input",
-        data: dataString,
-        latency: tracker,
+      const protoTracker: ProtoLatencyTracker = {
+        $typeName: "proto.ProtoLatencyTracker",
+        sequenceId: tracker.sequence_id,
+        timestamps: [],
      };
-      this.wrtc.sendBinary(encodeMessage(message));
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
    };
  }

--- a/packages/input/src/proto/latency_tracker_pb.ts
+++ b/packages/input/src/proto/latency_tracker_pb.ts
@@ -0,0 +1,60 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file latency_tracker.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { Timestamp } from "@bufbuild/protobuf/wkt";
+import { file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file latency_tracker.proto.
+ */
+export const file_latency_tracker: GenFile = /*@__PURE__*/
+  fileDesc("ChVsYXRlbmN5X3RyYWNrZXIucHJvdG8SBXByb3RvIk4KE1Byb3RvVGltZXN0YW1wRW50cnkSDQoFc3RhZ2UYASABKAkSKAoEdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiWgoTUHJvdG9MYXRlbmN5VHJhY2tlchITCgtzZXF1ZW5jZV9pZBgBIAEoCRIuCgp0aW1lc3RhbXBzGAIgAygLMhoucHJvdG8uUHJvdG9UaW1lc3RhbXBFbnRyeUIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_google_protobuf_timestamp]);
+
+/**
+ * @generated from message proto.ProtoTimestampEntry
+ */
+export type ProtoTimestampEntry = Message<"proto.ProtoTimestampEntry"> & {
+  /**
+   * @generated from field: string stage = 1;
+   */
+  stage: string;
+
+  /**
+   * @generated from field: google.protobuf.Timestamp time = 2;
+   */
+  time?: Timestamp;
+};
+
+/**
+ * Describes the message proto.ProtoTimestampEntry.
+ * Use `create(ProtoTimestampEntrySchema)` to create a new message.
+ */
+export const ProtoTimestampEntrySchema: GenMessage<ProtoTimestampEntry> = /*@__PURE__*/
+  messageDesc(file_latency_tracker, 0);
+
+/**
+ * @generated from message proto.ProtoLatencyTracker
+ */
+export type ProtoLatencyTracker = Message<"proto.ProtoLatencyTracker"> & {
+  /**
+   * @generated from field: string sequence_id = 1;
+   */
+  sequenceId: string;
+
+  /**
+   * @generated from field: repeated proto.ProtoTimestampEntry timestamps = 2;
+   */
+  timestamps: ProtoTimestampEntry[];
+};
+
+/**
+ * Describes the message proto.ProtoLatencyTracker.
+ * Use `create(ProtoLatencyTrackerSchema)` to create a new message.
+ */
+export const ProtoLatencyTrackerSchema: GenMessage<ProtoLatencyTracker> = /*@__PURE__*/
+  messageDesc(file_latency_tracker, 1);
+
--- a/packages/input/src/proto/messages_pb.ts
+++ b/packages/input/src/proto/messages_pb.ts
@@ -0,0 +1,62 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file messages.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { ProtoInput } from "./types_pb";
+import { file_types } from "./types_pb";
+import type { ProtoLatencyTracker } from "./latency_tracker_pb";
+import { file_latency_tracker } from "./latency_tracker_pb";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file messages.proto.
+ */
+export const file_messages: GenFile = /*@__PURE__*/
+  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
+
+/**
+ * @generated from message proto.ProtoMessageBase
+ */
+export type ProtoMessageBase = Message<"proto.ProtoMessageBase"> & {
+  /**
+   * @generated from field: string payload_type = 1;
+   */
+  payloadType: string;
+
+  /**
+   * @generated from field: proto.ProtoLatencyTracker latency = 2;
+   */
+  latency?: ProtoLatencyTracker;
+};
+
+/**
+ * Describes the message proto.ProtoMessageBase.
+ * Use `create(ProtoMessageBaseSchema)` to create a new message.
+ */
+export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*/
+  messageDesc(file_messages, 0);
+
+/**
+ * @generated from message proto.ProtoMessageInput
+ */
+export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
+  /**
+   * @generated from field: proto.ProtoMessageBase message_base = 1;
+   */
+  messageBase?: ProtoMessageBase;
+
+  /**
+   * @generated from field: proto.ProtoInput data = 2;
+   */
+  data?: ProtoInput;
+};
+
+/**
+ * Describes the message proto.ProtoMessageInput.
+ * Use `create(ProtoMessageInputSchema)` to create a new message.
+ */
+export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
+  messageDesc(file_messages, 1);
+
--- a/packages/input/src/proto/types_pb.ts
+++ b/packages/input/src/proto/types_pb.ts
@@ -0,0 +1,272 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file types.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file types.proto.
+ */
+export const file_types: GenFile = /*@__PURE__*/
+  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSLcAgoKUHJvdG9JbnB1dBIrCgptb3VzZV9tb3ZlGAEgASgLMhUucHJvdG8uUHJvdG9Nb3VzZU1vdmVIABIyCg5tb3VzZV9tb3ZlX2FicxgCIAEoCzIYLnByb3RvLlByb3RvTW91c2VNb3ZlQWJzSAASLQoLbW91c2Vfd2hlZWwYAyABKAsyFi5wcm90by5Qcm90b01vdXNlV2hlZWxIABIyCg5tb3VzZV9rZXlfZG93bhgEIAEoCzIYLnByb3RvLlByb3RvTW91c2VLZXlEb3duSAASLgoMbW91c2Vfa2V5X3VwGAUgASgLMhYucHJvdG8uUHJvdG9Nb3VzZUtleVVwSAASJwoIa2V5X2Rvd24YBiABKAsyEy5wcm90by5Qcm90b0tleURvd25IABIjCgZrZXlfdXAYByABKAsyES5wcm90by5Qcm90b0tleVVwSABCDAoKaW5wdXRfdHlwZUIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z");
+
+/**
+ * MouseMove message
+ *
+ * @generated from message proto.ProtoMouseMove
+ */
+export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
+  /**
+   * Fixed value "MouseMove"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseMove.
+ * Use `create(ProtoMouseMoveSchema)` to create a new message.
+ */
+export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
+  messageDesc(file_types, 0);
+
+/**
+ * MouseMoveAbs message
+ *
+ * @generated from message proto.ProtoMouseMoveAbs
+ */
+export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
+  /**
+   * Fixed value "MouseMoveAbs"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseMoveAbs.
+ * Use `create(ProtoMouseMoveAbsSchema)` to create a new message.
+ */
+export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE__*/
+  messageDesc(file_types, 1);
+
+/**
+ * MouseWheel message
+ *
+ * @generated from message proto.ProtoMouseWheel
+ */
+export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
+  /**
+   * Fixed value "MouseWheel"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseWheel.
+ * Use `create(ProtoMouseWheelSchema)` to create a new message.
+ */
+export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
+  messageDesc(file_types, 2);
+
+/**
+ * MouseKeyDown message
+ *
+ * @generated from message proto.ProtoMouseKeyDown
+ */
+export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
+  /**
+   * Fixed value "MouseKeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseKeyDown.
+ * Use `create(ProtoMouseKeyDownSchema)` to create a new message.
+ */
+export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE__*/
+  messageDesc(file_types, 3);
+
+/**
+ * MouseKeyUp message
+ *
+ * @generated from message proto.ProtoMouseKeyUp
+ */
+export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
+  /**
+   * Fixed value "MouseKeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseKeyUp.
+ * Use `create(ProtoMouseKeyUpSchema)` to create a new message.
+ */
+export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
+  messageDesc(file_types, 4);
+
+/**
+ * KeyDown message
+ *
+ * @generated from message proto.ProtoKeyDown
+ */
+export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
+  /**
+   * Fixed value "KeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoKeyDown.
+ * Use `create(ProtoKeyDownSchema)` to create a new message.
+ */
+export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
+  messageDesc(file_types, 5);
+
+/**
+ * KeyUp message
+ *
+ * @generated from message proto.ProtoKeyUp
+ */
+export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
+  /**
+   * Fixed value "KeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoKeyUp.
+ * Use `create(ProtoKeyUpSchema)` to create a new message.
+ */
+export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
+  messageDesc(file_types, 6);
+
+/**
+ * Union of all Input types
+ *
+ * @generated from message proto.ProtoInput
+ */
+export type ProtoInput = Message<"proto.ProtoInput"> & {
+  /**
+   * @generated from oneof proto.ProtoInput.input_type
+   */
+  inputType: {
+    /**
+     * @generated from field: proto.ProtoMouseMove mouse_move = 1;
+     */
+    value: ProtoMouseMove;
+    case: "mouseMove";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
+     */
+    value: ProtoMouseMoveAbs;
+    case: "mouseMoveAbs";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
+     */
+    value: ProtoMouseWheel;
+    case: "mouseWheel";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
+     */
+    value: ProtoMouseKeyDown;
+    case: "mouseKeyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
+     */
+    value: ProtoMouseKeyUp;
+    case: "mouseKeyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyDown key_down = 6;
+     */
+    value: ProtoKeyDown;
+    case: "keyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyUp key_up = 7;
+     */
+    value: ProtoKeyUp;
+    case: "keyUp";
+  } | { case: undefined; value?: undefined };
+};
+
+/**
+ * Describes the message proto.ProtoInput.
+ * Use `create(ProtoInputSchema)` to create a new message.
+ */
+export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
+  messageDesc(file_types, 7);
+
--- a/packages/input/src/types.ts
+++ b/packages/input/src/types.ts
@@ -1,52 +0,0 @@
-interface BaseInput {
-    timestamp?: number; // Add a timestamp for better context (optional)
-}
-
-interface MouseMove extends BaseInput {
-    type: "MouseMove";
-    x: number;
-    y: number;
-}
-
-interface MouseMoveAbs extends BaseInput {
-    type: "MouseMoveAbs";
-    x: number;
-    y: number;
-}
-
-interface MouseWheel extends BaseInput {
-    type: "MouseWheel";
-    x: number;
-    y: number;
-}
-
-interface MouseKeyDown extends BaseInput {
-    type: "MouseKeyDown";
-    key: number;
-}
-
-interface MouseKeyUp extends BaseInput {
-    type: "MouseKeyUp";
-    key: number;
-}
-
-interface KeyDown extends BaseInput {
-    type: "KeyDown";
-    key: number;
-}
-
-interface KeyUp extends BaseInput {
-    type: "KeyUp";
-    key: number;
-}
-
-
-export type Input =
-    | MouseMove
-    | MouseMoveAbs
-    | MouseWheel
-    | MouseKeyDown
-    | MouseKeyUp
-    | KeyDown
-    | KeyUp;
-
--- a/packages/input/src/webrtc-stream.ts
+++ b/packages/input/src/webrtc-stream.ts
@@ -6,8 +6,6 @@ import {
  MessageAnswer,
  JoinerType,
  AnswerType,
-  decodeMessage,
-  encodeMessage
 } from "./messages";

 export class WebRTCStream {
@@ -40,16 +38,16 @@ export class WebRTCStream {
        payload_type: "join",
        joiner_type: JoinerType.JoinerClient
      };
-      this._ws!.send(encodeMessage(joinMessage));
+      this._ws!.send(JSON.stringify(joinMessage));
    }

    let iceHolder: RTCIceCandidateInit[] = [];

    this._ws.onmessage = async (e) => {
-      // allow only binary
-      if (typeof e.data !== "object") return;
+      // allow only JSON
+      if (typeof e.data === "object") return;
      if (!e.data) return;
-      const message = await decodeMessage<MessageBase>(e.data);
+      const message = JSON.parse(e.data) as MessageBase;
      switch (message.payload_type) {
        case "sdp":
          if (!this._pc) {
@@ -63,7 +61,7 @@ export class WebRTCStream {
          // Force stereo in Chromium browsers
          answer.sdp = this.forceOpusStereo(answer.sdp!);
          await this._pc!.setLocalDescription(answer);
-          this._ws!.send(encodeMessage({
+          this._ws!.send(JSON.stringify({
            payload_type: "sdp",
            sdp: answer
          }));
@@ -154,7 +152,7 @@ export class WebRTCStream {
          payload_type: "ice",
          candidate: e.candidate
        };
-        this._ws!.send(encodeMessage(message));
+        this._ws!.send(JSON.stringify(message));
      }
    }