Merge a54cf759fa into 32341574dc

containerfiles/runner-builder.Containerfile

@@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \
     pacman -Sy --noconfirm lib32-gcc-libs
 
 # Clone repository
-RUN git clone --depth 1 --rev "9e8bfd0217eeab011c5afc368d3ea67a4c239e81" https://github.com/DatCaptainHorse/vimputti.git
+RUN git clone --depth 1 --rev "f2f21561ddcb814d74455311969d3e8934b052c6" https://github.com/DatCaptainHorse/vimputti.git
 
 #--------------------------------------------------------------------
 FROM vimputti-manager-deps AS vimputti-manager-planner
@@ -129,23 +129,8 @@ RUN --mount=type=cache,target=/var/cache/pacman/pkg \
 RUN --mount=type=cache,target=${CARGO_HOME}/registry \
     cargo install cargo-c
 
-# Grab cudart from NVIDIA..
-RUN wget https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-13.0.96-archive.tar.xz -O cuda_cudart.tar.xz && \
-    mkdir cuda_cudart && tar -xf cuda_cudart.tar.xz -C cuda_cudart --strip-components=1 && \
-    cp cuda_cudart/lib/libcudart.so cuda_cudart/lib/libcudart.so.* /usr/lib/ && \
-    rm -r cuda_cudart && \
-    rm cuda_cudart.tar.xz
-
-# Grab cuda lib from NVIDIA (it's in driver package of all things..)
-RUN wget https://developer.download.nvidia.com/compute/cuda/redist/nvidia_driver/linux-x86_64/nvidia_driver-linux-x86_64-580.95.05-archive.tar.xz -O nvidia_driver.tar.xz && \
-    mkdir nvidia_driver && tar -xf nvidia_driver.tar.xz -C nvidia_driver --strip-components=1 && \
-    cp nvidia_driver/lib/libcuda.so.* /usr/lib/libcuda.so && \
-    ln -s /usr/lib/libcuda.so /usr/lib/libcuda.so.1 && \
-    rm -r nvidia_driver && \
-    rm nvidia_driver.tar.xz
-
 # Clone repository
-RUN git clone --depth 1 --rev "afa853fa03e8403c83bbb3bc0cf39147ad46c266" https://github.com/games-on-whales/gst-wayland-display.git
+RUN git clone --depth 1 --rev "a4abcfe2cffe2d33b564d1308b58504a5e3012b1" https://github.com/games-on-whales/gst-wayland-display.git
 
 #--------------------------------------------------------------------
 FROM gst-wayland-deps AS gst-wayland-planner
@@ -214,5 +199,4 @@ COPY --from=gst-wayland-cached-builder /artifacts/include/ /artifacts/include/
 COPY --from=vimputti-manager-cached-builder /artifacts/vimputti-manager /artifacts/bin/
 COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_64.so /artifacts/lib64/libvimputti_shim.so
 COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_32.so /artifacts/lib32/libvimputti_shim.so
-COPY --from=gst-wayland-deps /usr/lib/libcuda.so /usr/lib/libcuda.so.* /artifacts/lib/
 COPY --from=bubblewrap-builder /artifacts/bin/bwrap /artifacts/bin/

packages/input/package.json

@@ -7,24 +7,22 @@
         ".": "./src/index.ts"
     },
     "devDependencies": {
-        "@bufbuild/buf": "^1.57.2",
+        "@bufbuild/buf": "^1.59.0",
-        "@bufbuild/protoc-gen-es": "^2.9.0"
+        "@bufbuild/protoc-gen-es": "^2.10.0"
     },
     "dependencies": {
-        "@bufbuild/protobuf": "^2.9.0",
+        "@bufbuild/protobuf": "^2.10.0",
-        "@chainsafe/libp2p-noise": "^16.1.4",
+        "@chainsafe/libp2p-noise": "^17.0.0",
         "@chainsafe/libp2p-quic": "^1.1.3",
-        "@chainsafe/libp2p-yamux": "^7.0.4",
+        "@chainsafe/libp2p-yamux": "^8.0.1",
-        "@libp2p/identify": "^3.0.39",
+        "@libp2p/identify": "^4.0.5",
-        "@libp2p/interface": "^2.11.0",
+        "@libp2p/interface": "^3.0.2",
-        "@libp2p/ping": "^2.0.37",
+        "@libp2p/ping": "^3.0.5",
-        "@libp2p/websockets": "^9.2.19",
+        "@libp2p/websockets": "^10.0.6",
-        "@libp2p/webtransport": "^5.0.51",
+        "@libp2p/webtransport": "^6.0.7",
-        "@multiformats/multiaddr": "^12.5.1",
+        "@libp2p/utils": "^7.0.5",
-        "it-length-prefixed": "^10.0.1",
+        "@multiformats/multiaddr": "^13.0.1",
-        "it-pipe": "^3.0.1",
+        "libp2p": "^3.0.6",
-        "libp2p": "^2.10.0",
+        "uint8arraylist": "^2.4.8"
-        "uint8arraylist": "^2.4.8",
-        "uint8arrays": "^5.1.0"
     }
 }

packages/input/src/controller.ts

@@ -1,12 +1,6 @@
 import { controllerButtonToLinuxEventCode } from "./codes";
 import { WebRTCStream } from "./webrtc-stream";
 import {
-  ProtoMessageBase,
-  ProtoMessageInput,
-  ProtoMessageInputSchema,
-} from "./proto/messages_pb";
-import {
-  ProtoInputSchema,
   ProtoControllerAttachSchema,
   ProtoControllerDetachSchema,
   ProtoControllerButtonSchema,
@@ -16,6 +10,8 @@ import {
   ProtoControllerRumble,
 } from "./proto/types_pb";
 import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
+import { createMessage } from "./utils";
+import { ProtoMessageSchema } from "./proto/messages_pb";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -36,7 +32,6 @@ interface GamepadState {
 
 export class Controller {
   protected wrtc: WebRTCStream;
-  protected slot: number;
   protected connected: boolean = false;
   protected gamepad: Gamepad | null = null;
   protected lastState: GamepadState = {
@@ -54,17 +49,21 @@ export class Controller {
   protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range)
 
   private updateInterval = 10.0; // 100 updates per second
-  private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null;
+  private isIdle: boolean = true;
+  private lastInputTime: number = Date.now();
+  private idleUpdateInterval: number = 150.0; // ~6-7 updates per second for keep-alive packets
+  private inputDetected: boolean = false;
+  private lastFullStateSend: number = Date.now();
+  private fullStateSendInterval: number = 500.0; // send full state every 0.5 seconds (helps packet loss)
+  private forceFullStateSend: boolean = false;
+
+  private _dcHandler: ((data: ArrayBuffer) => void) | null = null;
 
   constructor({ webrtc, e }: Props) {
     this.wrtc = webrtc;
-    this.slot = e.gamepad.index;
 
     this.updateInterval = 1000 / webrtc.currentFrameRate;
 
-    // Gamepad connected
-    this.gamepad = e.gamepad;
-
     // Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc")
     const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/);
     const vendorId = vendorMatch ? vendorMatch[1].toLowerCase() : "unknown";
@@ -72,34 +71,48 @@ export class Controller {
     const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/);
     const productId = productMatch ? productMatch[1].toLowerCase() : "unknown";
 
-    const attachMsg = create(ProtoInputSchema, {
+    // Listen to datachannel events from server
-      $typeName: "proto.ProtoInput",
+    this._dcHandler = (data: ArrayBuffer) => {
-      inputType: {
+      if (!this.connected) return;
-        case: "controllerAttach",
+      try {
-        value: create(ProtoControllerAttachSchema, {
+        // First decode the wrapper message
-          type: "ControllerAttach",
+        const uint8Data = new Uint8Array(data);
-          id: this.vendor_id_to_controller(vendorId, productId),
+        const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data);
-          slot: this.slot,
-        }),
-      },
-    });
-    const message: ProtoMessageInput = {
-      $typeName: "proto.ProtoMessageInput",
-      messageBase: {
-        $typeName: "proto.ProtoMessageBase",
-        payloadType: "controllerInput",
-      } as ProtoMessageBase,
-      data: attachMsg,
-    };
-    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
 
-    // Listen to feedback rumble events from server
+        if (messageWrapper.payload.case === "controllerRumble") {
-    this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer);
+          this.rumbleCallback(messageWrapper.payload.value);
-    this.wrtc.addDataChannelCallback(this._dcRumbleHandler);
+        } else if (messageWrapper.payload.case === "controllerAttach") {
+          if (this.gamepad) return; // already attached
+          const attachMsg = messageWrapper.payload.value;
+          // Gamepad connected succesfully
+          this.gamepad = e.gamepad;
+          console.log(
+            `Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`,
+          );
+        }
+      } catch (err) {
+        console.error("Error decoding datachannel message:", err);
+      }
+    };
+    this.wrtc.addDataChannelCallback(this._dcHandler);
+
+    const attachMsg = createMessage(
+      create(ProtoControllerAttachSchema, {
+        id: this.vendor_id_to_controller(vendorId, productId),
+        sessionSlot: e.gamepad.index,
+        sessionId: this.wrtc.getSessionID(),
+      }),
+      "controllerInput",
+    );
+    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg));
 
     this.run();
   }
 
+  public getSlot(): number {
+    return this.gamepad.index;
+  }
+
   // Maps vendor id and product id to supported controller type
   // Currently supported: Sony (ps4, ps5), Microsoft (xbox360, xboxone), Nintendo (switchpro)
   // Default fallback to xbox360
@@ -150,18 +163,26 @@ export class Controller {
   }
 
   private pollGamepad() {
+    // Get updated gamepad state
     const gamepads = navigator.getGamepads();
-    if (this.slot < gamepads.length) {
+
-      const gamepad = gamepads[this.slot];
+    // Periodically force send full state to clear stuck inputs
-      if (gamepad) {
+    if (Date.now() - this.lastFullStateSend > this.fullStateSendInterval) {
+      this.forceFullStateSend = true;
+      this.lastFullStateSend = Date.now();
+    }
+
+    if (this.gamepad) {
+      if (gamepads[this.gamepad.index]) {
+        this.gamepad = gamepads[this.gamepad!.index];
         /* Button handling */
-        gamepad.buttons.forEach((button, index) => {
+        this.gamepad.buttons.forEach((button, index) => {
           // Ignore d-pad buttons (12-15) as we handle those as axis
           if (index >= 12 && index <= 15) return;
           // ignore trigger buttons (6-7) as we handle those as axis
           if (index === 6 || index === 7) return;
           // If state differs, send
-          if (button.pressed !== this.lastState.buttonState.get(index)) {
+          if (button.pressed !== this.lastState.buttonState.get(index) || this.forceFullStateSend) {
             const linuxCode = this.controllerButtonToVirtualKeyCode(index);
             if (linuxCode === undefined) {
               // Skip unmapped button index
@@ -169,29 +190,17 @@ export class Controller {
               return;
             }
 
-            const buttonProto = create(ProtoInputSchema, {
+            const buttonMessage = createMessage(
-              $typeName: "proto.ProtoInput",
+              create(ProtoControllerButtonSchema, {
-              inputType: {
+                sessionSlot: this.gamepad.index,
-                case: "controllerButton",
+                sessionId: this.wrtc.getSessionID(),
-                value: create(ProtoControllerButtonSchema, {
-                  type: "ControllerButton",
-                  slot: this.slot,
                 button: linuxCode,
                 pressed: button.pressed,
               }),
-              },
+              "controllerInput",
-            });
-            const buttonMessage: ProtoMessageInput = {
-              $typeName: "proto.ProtoMessageInput",
-              messageBase: {
-                $typeName: "proto.ProtoMessageBase",
-                payloadType: "controllerInput",
-              } as ProtoMessageBase,
-              data: buttonProto,
-            };
-            this.wrtc.sendBinary(
-              toBinary(ProtoMessageInputSchema, buttonMessage),
             );
+            this.wrtc.sendBinary(toBinary(ProtoMessageSchema, buttonMessage));
+            this.inputDetected = true;
             // Store button state
             this.lastState.buttonState.set(index, button.pressed);
           }
@@ -200,128 +209,108 @@ export class Controller {
         /* Trigger handling */
         // map trigger value from 0.0 to 1.0 to -32768 to 32767
         const leftTrigger = Math.round(
-          this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767),
+          this.remapFromTo(
+            this.gamepad.buttons[6]?.value ?? 0,
+            0,
+            1,
+            -32768,
+            32767,
+          ),
         );
         // If state differs, send
-        if (leftTrigger !== this.lastState.leftTrigger) {
+        if (leftTrigger !== this.lastState.leftTrigger || this.forceFullStateSend) {
-          const triggerProto = create(ProtoInputSchema, {
+          const triggerMessage = createMessage(
-            $typeName: "proto.ProtoInput",
+            create(ProtoControllerTriggerSchema, {
-            inputType: {
+              sessionSlot: this.gamepad.index,
-              case: "controllerTrigger",
+              sessionId: this.wrtc.getSessionID(),
-              value: create(ProtoControllerTriggerSchema, {
-                type: "ControllerTrigger",
-                slot: this.slot,
               trigger: 0, // 0 = left, 1 = right
               value: leftTrigger,
             }),
-            },
+            "controllerInput",
-          });
-          const triggerMessage: ProtoMessageInput = {
-            $typeName: "proto.ProtoMessageInput",
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: triggerProto,
-          };
-          this.lastState.leftTrigger = leftTrigger;
-          this.wrtc.sendBinary(
-            toBinary(ProtoMessageInputSchema, triggerMessage),
           );
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
+          this.inputDetected = true;
+          this.lastState.leftTrigger = leftTrigger;
         }
         const rightTrigger = Math.round(
-          this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767),
+          this.remapFromTo(
+            this.gamepad.buttons[7]?.value ?? 0,
+            0,
+            1,
+            -32768,
+            32767,
+          ),
         );
         // If state differs, send
-        if (rightTrigger !== this.lastState.rightTrigger) {
+        if (rightTrigger !== this.lastState.rightTrigger || this.forceFullStateSend) {
-          const triggerProto = create(ProtoInputSchema, {
+          const triggerMessage = createMessage(
-            $typeName: "proto.ProtoInput",
+            create(ProtoControllerTriggerSchema, {
-            inputType: {
+              sessionSlot: this.gamepad.index,
-              case: "controllerTrigger",
+              sessionId: this.wrtc.getSessionID(),
-              value: create(ProtoControllerTriggerSchema, {
-                type: "ControllerTrigger",
-                slot: this.slot,
               trigger: 1, // 0 = left, 1 = right
               value: rightTrigger,
             }),
-            },
+            "controllerInput",
-          });
-          const triggerMessage: ProtoMessageInput = {
-            $typeName: "proto.ProtoMessageInput",
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: triggerProto,
-          };
-          this.lastState.rightTrigger = rightTrigger;
-          this.wrtc.sendBinary(
-            toBinary(ProtoMessageInputSchema, triggerMessage),
           );
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, triggerMessage));
+          this.inputDetected = true;
+          this.lastState.rightTrigger = rightTrigger;
         }
 
         /* DPad handling */
         // We send dpad buttons as axis values -1 to 1 for left/up, right/down
-        const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0;
+        const dpadLeft = this.gamepad.buttons[14]?.pressed ? 1 : 0;
-        const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0;
+        const dpadRight = this.gamepad.buttons[15]?.pressed ? 1 : 0;
         const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0;
-        if (dpadX !== this.lastState.dpadX) {
+        if (dpadX !== this.lastState.dpadX || this.forceFullStateSend) {
-          const dpadProto = create(ProtoInputSchema, {
+          const dpadMessage = createMessage(
-            $typeName: "proto.ProtoInput",
+            create(ProtoControllerAxisSchema, {
-            inputType: {
+              sessionSlot: this.gamepad.index,
-              case: "controllerAxis",
+              sessionId: this.wrtc.getSessionID(),
-              value: create(ProtoControllerAxisSchema, {
-                type: "ControllerAxis",
-                slot: this.slot,
               axis: 0, // 0 = dpadX, 1 = dpadY
               value: dpadX,
             }),
-            },
+            "controllerInput",
-          });
+          );
-          const dpadMessage: ProtoMessageInput = {
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
-            $typeName: "proto.ProtoMessageInput",
+          this.inputDetected = true;
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: dpadProto,
-          };
           this.lastState.dpadX = dpadX;
-          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
         }
 
-        const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0;
+        const dpadUp = this.gamepad.buttons[12]?.pressed ? 1 : 0;
-        const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0;
+        const dpadDown = this.gamepad.buttons[13]?.pressed ? 1 : 0;
         const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0;
-        if (dpadY !== this.lastState.dpadY) {
+        if (dpadY !== this.lastState.dpadY || this.forceFullStateSend) {
-          const dpadProto = create(ProtoInputSchema, {
+          const dpadMessage = createMessage(
-            $typeName: "proto.ProtoInput",
+            create(ProtoControllerAxisSchema, {
-            inputType: {
+              sessionSlot: this.gamepad.index,
-              case: "controllerAxis",
+              sessionId: this.wrtc.getSessionID(),
-              value: create(ProtoControllerAxisSchema, {
-                type: "ControllerAxis",
-                slot: this.slot,
               axis: 1, // 0 = dpadX, 1 = dpadY
               value: dpadY,
             }),
-            },
+            "controllerInput",
-          });
+          );
-          const dpadMessage: ProtoMessageInput = {
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, dpadMessage));
-            $typeName: "proto.ProtoMessageInput",
+          this.inputDetected = true;
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: dpadProto,
-          };
           this.lastState.dpadY = dpadY;
-          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
         }
 
         /* Stick handling */
         // stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767
-        const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767);
+        const leftX = this.remapFromTo(
-        const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767);
+          this.gamepad.axes[0] ?? 0,
+          -1,
+          1,
+          -32768,
+          32767,
+        );
+        const leftY = this.remapFromTo(
+          this.gamepad.axes[1] ?? 0,
+          -1,
+          1,
+          -32768,
+          32767,
+        );
         // Apply deadzone
         const sendLeftX =
           Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
@@ -331,37 +320,38 @@ export class Controller {
         // if moves inside deadzone, zero it if not inside deadzone last time
         if (
           sendLeftX !== this.lastState.leftX ||
-          sendLeftY !== this.lastState.leftY
+          sendLeftY !== this.lastState.leftY || this.forceFullStateSend
         ) {
-          // console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY);
+          const stickMessage = createMessage(
-          const stickProto = create(ProtoInputSchema, {
+            create(ProtoControllerStickSchema, {
-            $typeName: "proto.ProtoInput",
+              sessionSlot: this.gamepad.index,
-            inputType: {
+              sessionId: this.wrtc.getSessionID(),
-              case: "controllerStick",
-              value: create(ProtoControllerStickSchema, {
-                type: "ControllerStick",
-                slot: this.slot,
               stick: 0, // 0 = left, 1 = right
               x: sendLeftX,
               y: sendLeftY,
             }),
-            },
+            "controllerInput",
-          });
+          );
-          const stickMessage: ProtoMessageInput = {
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
-            $typeName: "proto.ProtoMessageInput",
+          this.inputDetected = true;
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: stickProto,
-          };
           this.lastState.leftX = sendLeftX;
           this.lastState.leftY = sendLeftY;
-          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
         }
 
-        const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767);
+        const rightX = this.remapFromTo(
-        const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767);
+          this.gamepad.axes[2] ?? 0,
+          -1,
+          1,
+          -32768,
+          32767,
+        );
+        const rightY = this.remapFromTo(
+          this.gamepad.axes[3] ?? 0,
+          -1,
+          1,
+          -32768,
+          32767,
+        );
         // Apply deadzone
         const sendRightX =
           Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
@@ -369,49 +359,64 @@ export class Controller {
           Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0;
         if (
           sendRightX !== this.lastState.rightX ||
-          sendRightY !== this.lastState.rightY
+          sendRightY !== this.lastState.rightY || this.forceFullStateSend
         ) {
-          const stickProto = create(ProtoInputSchema, {
+          const stickMessage = createMessage(
-            $typeName: "proto.ProtoInput",
+            create(ProtoControllerStickSchema, {
-            inputType: {
+              sessionSlot: this.gamepad.index,
-              case: "controllerStick",
+              sessionId: this.wrtc.getSessionID(),
-              value: create(ProtoControllerStickSchema, {
-                type: "ControllerStick",
-                slot: this.slot,
               stick: 1, // 0 = left, 1 = right
               x: sendRightX,
               y: sendRightY,
             }),
-            },
+            "controllerInput",
-          });
+          );
-          const stickMessage: ProtoMessageInput = {
+          this.wrtc.sendBinary(toBinary(ProtoMessageSchema, stickMessage));
-            $typeName: "proto.ProtoMessageInput",
+          this.inputDetected = true;
-            messageBase: {
-              $typeName: "proto.ProtoMessageBase",
-              payloadType: "controllerInput",
-            } as ProtoMessageBase,
-            data: stickProto,
-          };
           this.lastState.rightX = sendRightX;
           this.lastState.rightY = sendRightY;
-          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
         }
       }
     }
+
+    this.forceFullStateSend = false;
   }
 
   private loopInterval: any = null;
 
   public run() {
-    if (this.connected)
+    if (this.connected) this.stop();
-      this.stop();
 
     this.connected = true;
-    // Poll gamepads in setInterval loop
+    this.isIdle = true;
+    this.lastInputTime = Date.now();
+
+    this.loopInterval = setInterval(() => {
+      if (this.connected) {
+        this.inputDetected = false; // Reset before poll
+        this.pollGamepad();
+
+        // Switch polling rate based on input
+        if (this.inputDetected) {
+          this.lastInputTime = Date.now();
+          if (this.isIdle) {
+            this.isIdle = false;
+            clearInterval(this.loopInterval);
             this.loopInterval = setInterval(() => {
               if (this.connected) this.pollGamepad();
             }, this.updateInterval);
           }
+        } else if (!this.isIdle && Date.now() - this.lastInputTime > 200) {
+          // Switch to idle polling after 200ms of no input
+          this.isIdle = true;
+          clearInterval(this.loopInterval);
+          this.loopInterval = setInterval(() => {
+            if (this.connected) this.pollGamepad();
+          }, this.idleUpdateInterval);
+        }
+      }
+    }, this.isIdle ? this.idleUpdateInterval : this.updateInterval);
+  }
 
   public stop() {
     if (this.loopInterval) {
@@ -421,69 +426,44 @@ export class Controller {
     this.connected = false;
   }
 
-  public getSlot() {
-    return this.slot;
-  }
-
   public dispose() {
     this.stop();
     // Remove callback
-    if (this._dcRumbleHandler !== null) {
+    if (this._dcHandler !== null) {
-      this.wrtc.removeDataChannelCallback(this._dcRumbleHandler);
+      this.wrtc.removeDataChannelCallback(this._dcHandler);
-      this._dcRumbleHandler = null;
+      this._dcHandler = null;
     }
     // Gamepad disconnected
-    const detachMsg = create(ProtoInputSchema, {
+    const detachMsg = createMessage(
-      $typeName: "proto.ProtoInput",
+      create(ProtoControllerDetachSchema, {
-      inputType: {
+        sessionSlot: this.gamepad.index,
-        case: "controllerDetach",
-        value: create(ProtoControllerDetachSchema, {
-          type: "ControllerDetach",
-          slot: this.slot,
       }),
-      },
+      "controllerInput",
-    });
+    );
-    const message: ProtoMessageInput = {
+    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg));
-      $typeName: "proto.ProtoMessageInput",
-      messageBase: {
-        $typeName: "proto.ProtoMessageBase",
-        payloadType: "controllerInput",
-      } as ProtoMessageBase,
-      data: detachMsg,
-    };
-    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
   }
 
   private controllerButtonToVirtualKeyCode(code: number) {
     return controllerButtonToLinuxEventCode[code] || undefined;
   }
 
-  private rumbleCallback(data: ArrayBuffer) {
+  private rumbleCallback(rumbleMsg: ProtoControllerRumble) {
     // If not connected, ignore
     if (!this.connected) return;
-    try {
-      // First decode the wrapper message
-      const uint8Data = new Uint8Array(data);
-      const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data);
-
-      // Check if it contains controller rumble data
-      if (messageWrapper.data?.inputType?.case === "controllerRumble") {
-        const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble;
 
     // Check if aimed at this controller slot
-        if (rumbleMsg.slot !== this.slot) return;
+    if (rumbleMsg.sessionId !== this.wrtc.getSessionID() &&
+        rumbleMsg.sessionSlot !== this.gamepad.index)
+      return;
 
     // Trigger actual rumble
     // Need to remap from 0-65535 to 0.0-1.0 ranges
     const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
-        const rumbleLowFreq = this.remapFromTo(
+    const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0);
-          clampedLowFreq,
+    const clampedHighFreq = Math.max(
       0,
-          65535,
+      Math.min(65535, rumbleMsg.highFrequency),
-          0.0,
-          1.0,
     );
-        const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency));
     const rumbleHighFreq = this.remapFromTo(
       clampedHighFreq,
       0,
@@ -494,16 +474,14 @@ export class Controller {
     // Cap to valid range (max 5000)
     const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
     if (this.gamepad.vibrationActuator) {
-          this.gamepad.vibrationActuator.playEffect("dual-rumble", {
+      this.gamepad.vibrationActuator
+        .playEffect("dual-rumble", {
           startDelay: 0,
           duration: rumbleDuration,
           weakMagnitude: rumbleLowFreq,
           strongMagnitude: rumbleHighFreq,
-          }).catch(console.error);
+        })
-        }
+        .catch(console.error);
-      }
-    } catch (error) {
-      console.error("Failed to decode rumble message:", error);
     }
   }
 }

packages/input/src/keyboard.ts

@@ -1,16 +1,9 @@
-import {keyCodeToLinuxEventCode} from "./codes"
+import { keyCodeToLinuxEventCode } from "./codes";
-import {WebRTCStream} from "./webrtc-stream";
+import { WebRTCStream } from "./webrtc-stream";
-import {LatencyTracker} from "./latency";
+import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
-import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import { create, toBinary } from "@bufbuild/protobuf";
-import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import { createMessage } from "./utils";
-import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
+import { ProtoMessageSchema } from "./proto/messages_pb";
-import {
-  ProtoInput,
-  ProtoInputSchema,
-  ProtoKeyDownSchema,
-  ProtoKeyUpSchema,
-} from "./proto/types_pb";
-import {create, toBinary} from "@bufbuild/protobuf";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -24,38 +17,29 @@ export class Keyboard {
   private readonly keydownListener: (e: KeyboardEvent) => void;
   private readonly keyupListener: (e: KeyboardEvent) => void;
 
-  constructor({webrtc}: Props) {
+  constructor({ webrtc }: Props) {
     this.wrtc = webrtc;
-    this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+    this.keydownListener = this.createKeyboardListener((e: any) =>
-      $typeName: "proto.ProtoInput",
+      create(ProtoKeyDownSchema, {
-      inputType: {
+        key: this.keyToVirtualKeyCode(e.code),
-        case: "keyDown",
-        value: create(ProtoKeyDownSchema, {
-          type: "KeyDown",
-          key: this.keyToVirtualKeyCode(e.code)
       }),
-      }
+    );
-    }));
+    this.keyupListener = this.createKeyboardListener((e: any) =>
-    this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      create(ProtoKeyUpSchema, {
-      $typeName: "proto.ProtoInput",
+        key: this.keyToVirtualKeyCode(e.code),
-      inputType: {
-        case: "keyUp",
-        value: create(ProtoKeyUpSchema, {
-          type: "KeyUp",
-          key: this.keyToVirtualKeyCode(e.code)
       }),
-      }
+    );
-    }));
+    this.run();
-    this.run()
   }
 
   private run() {
-    if (this.connected)
+    if (this.connected) this.stop();
-      this.stop()
 
-    this.connected = true
+    this.connected = true;
-    document.addEventListener("keydown", this.keydownListener, {passive: false});
+    document.addEventListener("keydown", this.keydownListener, {
-    document.addEventListener("keyup", this.keyupListener, {passive: false});
+      passive: false,
+    });
+    document.addEventListener("keyup", this.keyupListener, { passive: false });
   }
 
   private stop() {
@@ -65,42 +49,19 @@ export class Keyboard {
   }
 
   // Helper function to create and return mouse listeners
-  private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
+  private createKeyboardListener(
+    dataCreator: (e: Event) => any,
+  ): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
       // Prevent repeated key events from being sent (important for games)
-      if ((e as any).repeat)
+      if ((e as any).repeat) return;
-        return;
 
       const data = dataCreator(e as any);
 
-      // Latency tracking
+      const message = createMessage(data, "input");
-      const tracker = new LatencyTracker("input-keyboard");
+      this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
-      tracker.addTimestamp("client_send");
-      const protoTracker: ProtoLatencyTracker = {
-        $typeName: "proto.ProtoLatencyTracker",
-        sequenceId: tracker.sequence_id,
-        timestamps: [],
-      };
-      for (const t of tracker.timestamps) {
-        protoTracker.timestamps.push({
-          $typeName: "proto.ProtoTimestampEntry",
-          stage: t.stage,
-          time: timestampFromDate(t.time),
-        } as ProtoTimestampEntry);
-      }
-
-      const message: ProtoMessageInput = {
-        $typeName: "proto.ProtoMessageInput",
-        messageBase: {
-          $typeName: "proto.ProtoMessageBase",
-          payloadType: "input",
-          latency: protoTracker,
-        } as ProtoMessageBase,
-        data: data,
-      };
-      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 

packages/input/src/messages.ts

@@ -1,305 +0,0 @@
-import { LatencyTracker } from "./latency";
-import { Uint8ArrayList } from "uint8arraylist";
-import { allocUnsafe } from "uint8arrays/alloc";
-import { pipe } from "it-pipe";
-import { decode, encode } from "it-length-prefixed";
-import { Stream } from "@libp2p/interface";
-
-export interface MessageBase {
-  payload_type: string;
-  latency?: LatencyTracker;
-}
-
-export interface MessageRaw extends MessageBase {
-  data: any;
-}
-
-export function NewMessageRaw(type: string, data: any): Uint8Array {
-  const msg = {
-    payload_type: type,
-    data: data,
-  };
-  return new TextEncoder().encode(JSON.stringify(msg));
-}
-
-export interface MessageICE extends MessageBase {
-  candidate: RTCIceCandidateInit;
-}
-
-export function NewMessageICE(
-  type: string,
-  candidate: RTCIceCandidateInit,
-): Uint8Array {
-  const msg = {
-    payload_type: type,
-    candidate: candidate,
-  };
-  return new TextEncoder().encode(JSON.stringify(msg));
-}
-
-export interface MessageSDP extends MessageBase {
-  sdp: RTCSessionDescriptionInit;
-}
-
-export function NewMessageSDP(
-  type: string,
-  sdp: RTCSessionDescriptionInit,
-): Uint8Array {
-  const msg = {
-    payload_type: type,
-    sdp: sdp,
-  };
-  return new TextEncoder().encode(JSON.stringify(msg));
-}
-
-const MAX_SIZE = 1024 * 1024; // 1MB
-const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue
-
-// Custom 4-byte length encoder
-export const length4ByteEncoder = (length: number) => {
-  const buf = allocUnsafe(4);
-
-  // Write the length as a 32-bit unsigned integer (4 bytes)
-  buf[0] = length >>> 24;
-  buf[1] = (length >>> 16) & 0xff;
-  buf[2] = (length >>> 8) & 0xff;
-  buf[3] = length & 0xff;
-
-  // Set the bytes property to 4
-  length4ByteEncoder.bytes = 4;
-
-  return buf;
-};
-length4ByteEncoder.bytes = 4;
-
-// Custom 4-byte length decoder
-export const length4ByteDecoder = (data: Uint8ArrayList) => {
-  if (data.byteLength < 4) {
-    // Not enough bytes to read the length
-    return -1;
-  }
-
-  // Read the length from the first 4 bytes
-  let length = 0;
-  length =
-    (data.subarray(0, 1)[0] >>> 0) * 0x1000000 +
-    (data.subarray(1, 2)[0] >>> 0) * 0x10000 +
-    (data.subarray(2, 3)[0] >>> 0) * 0x100 +
-    (data.subarray(3, 4)[0] >>> 0);
-
-  // Set bytes read to 4
-  length4ByteDecoder.bytes = 4;
-
-  return length;
-};
-length4ByteDecoder.bytes = 4;
-
-interface PromiseMessage {
-  data: Uint8Array;
-  resolve: () => void;
-  reject: (error: Error) => void;
-}
-
-export class SafeStream {
-  private stream: Stream;
-  private callbacks: Map<string, ((data: any) => void)[]> = new Map();
-  private isReading: boolean = false;
-  private isWriting: boolean = false;
-  private closed: boolean = false;
-  private messageQueue: PromiseMessage[] = [];
-  private writeLock = false;
-  private readRetries = 0;
-  private writeRetries = 0;
-  private readonly MAX_RETRIES = 5;
-
-  constructor(stream: Stream) {
-    this.stream = stream;
-    this.startReading();
-    this.startWriting();
-  }
-
-  private async startReading(): Promise<void> {
-    if (this.isReading || this.closed) return;
-
-    this.isReading = true;
-
-    try {
-      const source = this.stream.source;
-      const decodedSource = decode(source, {
-        maxDataLength: MAX_SIZE,
-        lengthDecoder: length4ByteDecoder,
-      });
-
-      for await (const chunk of decodedSource) {
-        if (this.closed) break;
-
-        this.readRetries = 0;
-
-        try {
-          const data = chunk.slice();
-          const message = JSON.parse(
-            new TextDecoder().decode(data),
-          ) as MessageBase;
-          const msgType = message.payload_type;
-
-          if (this.callbacks.has(msgType)) {
-            const handlers = this.callbacks.get(msgType)!;
-            for (const handler of handlers) {
-              try {
-                handler(message);
-              } catch (err) {
-                console.error(`Error in message handler for ${msgType}:`, err);
-              }
-            }
-          }
-        } catch (err) {
-          console.error("Error processing message:", err);
-        }
-      }
-    } catch (err) {
-      console.error("Stream reading error:", err);
-    } finally {
-      this.isReading = false;
-      this.readRetries++;
-
-      // If not closed, try to restart reading
-      if (!this.closed && this.readRetries < this.MAX_RETRIES)
-        setTimeout(() => this.startReading(), 100);
-      else if (this.readRetries >= this.MAX_RETRIES)
-        console.error(
-          "Max retries reached for reading stream, stopping attempts",
-        );
-    }
-  }
-
-  public registerCallback(
-    msgType: string,
-    callback: (data: any) => void,
-  ): void {
-    if (!this.callbacks.has(msgType)) {
-      this.callbacks.set(msgType, []);
-    }
-
-    this.callbacks.get(msgType)!.push(callback);
-  }
-
-  public removeCallback(msgType: string, callback: (data: any) => void): void {
-    if (this.callbacks.has(msgType)) {
-      const callbacks = this.callbacks.get(msgType)!;
-      const index = callbacks.indexOf(callback);
-
-      if (index !== -1) {
-        callbacks.splice(index, 1);
-      }
-
-      if (callbacks.length === 0) {
-        this.callbacks.delete(msgType);
-      }
-    }
-  }
-
-  private async startWriting(): Promise<void> {
-    if (this.isWriting || this.closed) return;
-
-    this.isWriting = true;
-
-    try {
-      // Create an async generator for real-time message processing
-      const messageSource = async function* (this: SafeStream) {
-        while (!this.closed) {
-          // Check if we have messages to send
-          if (this.messageQueue.length > 0) {
-            this.writeLock = true;
-
-            try {
-              const message = this.messageQueue[0];
-
-              // Encode the message
-              const encoded = encode([message.data], {
-                maxDataLength: MAX_SIZE,
-                lengthEncoder: length4ByteEncoder,
-              });
-
-              for await (const chunk of encoded) {
-                yield chunk;
-              }
-
-              // Remove message after successful sending
-              this.writeRetries = 0;
-              const sentMessage = this.messageQueue.shift();
-              if (sentMessage)
-                sentMessage.resolve();
-            } catch (err) {
-              console.error("Error encoding or sending message:", err);
-              const failedMessage = this.messageQueue.shift();
-              if (failedMessage)
-                failedMessage.reject(new Error(`Failed to send message: ${err}`));
-            } finally {
-              this.writeLock = false;
-            }
-          } else {
-            // No messages to send, wait for a short period
-            await new Promise((resolve) => setTimeout(resolve, 100));
-          }
-        }
-      }.bind(this);
-
-      await pipe(messageSource(), this.stream.sink).catch((err) => {
-        console.error("Sink error:", err);
-        this.isWriting = false;
-        this.writeRetries++;
-
-        // Try to restart if not closed
-        if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
-          setTimeout(() => this.startWriting(), 1000);
-        } else if (this.writeRetries >= this.MAX_RETRIES) {
-          console.error("Max retries reached for writing to stream sink, stopping attempts");
-        }
-      });
-    } catch (err) {
-      console.error("Stream writing error:", err);
-      this.isWriting = false;
-      this.writeRetries++;
-
-      // Try to restart if not closed
-      if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
-        setTimeout(() => this.startWriting(), 1000);
-      } else if (this.writeRetries >= this.MAX_RETRIES) {
-        console.error("Max retries reached for writing stream, stopping attempts");
-      }
-    }
-  }
-
-  public async writeMessage(message: Uint8Array): Promise<void> {
-    if (this.closed) {
-      throw new Error("Cannot write to closed stream");
-    }
-
-    // Validate message size before queuing
-    if (message.length > MAX_SIZE) {
-      throw new Error("Message size exceeds maximum size limit");
-    }
-
-    // Check if the message queue is too large
-    if (this.messageQueue.length >= MAX_QUEUE_SIZE) {
-      throw new Error("Message queue is full, cannot write message");
-    }
-
-    // Create a promise to resolve when the message is sent
-    return new Promise((resolve, reject) => {
-      this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage);
-    });
-  }
-
-  public close(): void {
-    this.closed = true;
-    this.callbacks.clear();
-    // Reject pending messages
-    for (const msg of this.messageQueue)
-      msg.reject(new Error("Stream closed"));
-
-    this.messageQueue = [];
-    this.readRetries = 0;
-    this.writeRetries = 0;
-  }
-}

packages/input/src/mouse.ts

@@ -1,18 +1,14 @@
-import {WebRTCStream} from "./webrtc-stream";
+import { WebRTCStream } from "./webrtc-stream";
-import {LatencyTracker} from "./latency";
-import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
 import {
-  ProtoInput, ProtoInputSchema,
+  ProtoMouseKeyDownSchema,
-  ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
+  ProtoMouseKeyUpSchema,
-  ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
-  ProtoMouseMove,
   ProtoMouseMoveSchema,
-  ProtoMouseWheel, ProtoMouseWheelSchema
+  ProtoMouseWheelSchema,
 } from "./proto/types_pb";
-import {mouseButtonToLinuxEventCode} from "./codes";
+import { mouseButtonToLinuxEventCode } from "./codes";
-import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import { create, toBinary } from "@bufbuild/protobuf";
-import {create, toBinary} from "@bufbuild/protobuf";
+import { createMessage } from "./utils";
-import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import { ProtoMessageSchema } from "./proto/messages_pb";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -24,7 +20,7 @@ export class Mouse {
   protected canvas: HTMLCanvasElement;
   protected connected!: boolean;
 
-  private sendInterval = 10 // 100 updates per second
+  private sendInterval = 10; // 100 updates per second
 
   // Store references to event listeners
   private readonly mousemoveListener: (e: MouseEvent) => void;
@@ -35,7 +31,7 @@ export class Mouse {
   private readonly mouseupListener: (e: MouseEvent) => void;
   private readonly mousewheelListener: (e: WheelEvent) => void;
 
-  constructor({webrtc, canvas}: Props) {
+  constructor({ webrtc, canvas }: Props) {
     this.wrtc = webrtc;
     this.canvas = canvas;
 
@@ -48,65 +44,56 @@ export class Mouse {
       this.movementY += e.movementY;
     };
 
-    this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+    this.mousedownListener = this.createMouseListener((e: any) =>
-      $typeName: "proto.ProtoInput",
+      create(ProtoMouseKeyDownSchema, {
-      inputType: {
+        key: this.keyToVirtualKeyCode(e.button),
-        case: "mouseKeyDown",
-        value: create(ProtoMouseKeyDownSchema, {
-          type: "MouseKeyDown",
-          key: this.keyToVirtualKeyCode(e.button)
       }),
-      }
+    );
-    }));
+    this.mouseupListener = this.createMouseListener((e: any) =>
-    this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      create(ProtoMouseKeyUpSchema, {
-      $typeName: "proto.ProtoInput",
+        key: this.keyToVirtualKeyCode(e.button),
-      inputType: {
-        case: "mouseKeyUp",
-        value: create(ProtoMouseKeyUpSchema, {
-          type: "MouseKeyUp",
-          key: this.keyToVirtualKeyCode(e.button)
       }),
-      }
+    );
-    }));
+    this.mousewheelListener = this.createMouseListener((e: any) =>
-    this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      create(ProtoMouseWheelSchema, {
-      $typeName: "proto.ProtoInput",
-      inputType: {
-        case: "mouseWheel",
-        value: create(ProtoMouseWheelSchema, {
-          type: "MouseWheel",
         x: Math.round(e.deltaX),
         y: Math.round(e.deltaY),
       }),
-      }
+    );
-    }));
 
-    this.run()
+    this.run();
     this.startProcessing();
   }
 
   private run() {
     //calls all the other functions
     if (!document.pointerLockElement) {
-      console.log("no pointerlock")
+      console.log("no pointerlock");
       if (this.connected) {
-        this.stop()
+        this.stop();
       }
       return;
     }
 
     if (document.pointerLockElement == this.canvas) {
-      this.connected = true
+      this.connected = true;
-      this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
+      this.canvas.addEventListener("mousemove", this.mousemoveListener, {
-      this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
+        passive: false,
-      this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
+      });
-      this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
+      this.canvas.addEventListener("mousedown", this.mousedownListener, {
-
+        passive: false,
+      });
+      this.canvas.addEventListener("mouseup", this.mouseupListener, {
+        passive: false,
+      });
+      this.canvas.addEventListener("wheel", this.mousewheelListener, {
+        passive: false,
+      });
     } else {
       if (this.connected) {
-        this.stop()
+        this.stop();
       }
     }
-
   }
 
   private stop() {
@@ -128,79 +115,26 @@ export class Mouse {
   }
 
   private sendAggregatedMouseMove() {
-    const data = create(ProtoInputSchema, {
+    const data = create(ProtoMouseMoveSchema, {
-      $typeName: "proto.ProtoInput",
-      inputType: {
-        case: "mouseMove",
-        value: create(ProtoMouseMoveSchema, {
-          type: "MouseMove",
       x: Math.round(this.movementX),
       y: Math.round(this.movementY),
-        }),
-      },
     });
 
-    // Latency tracking
+    const message = createMessage(data, "input");
-    const tracker = new LatencyTracker("input-mouse");
+    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
-    tracker.addTimestamp("client_send");
-    const protoTracker: ProtoLatencyTracker = {
-      $typeName: "proto.ProtoLatencyTracker",
-      sequenceId: tracker.sequence_id,
-      timestamps: [],
-    };
-    for (const t of tracker.timestamps) {
-      protoTracker.timestamps.push({
-        $typeName: "proto.ProtoTimestampEntry",
-        stage: t.stage,
-        time: timestampFromDate(t.time),
-      } as ProtoTimestampEntry);
-    }
-
-    const message: ProtoMessageInput = {
-      $typeName: "proto.ProtoMessageInput",
-      messageBase: {
-        $typeName: "proto.ProtoMessageBase",
-        payloadType: "input",
-        latency: protoTracker,
-      } as ProtoMessageBase,
-      data: data,
-    };
-    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
   }
 
   // Helper function to create and return mouse listeners
-  private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
+  private createMouseListener(
+    dataCreator: (e: Event) => any,
+  ): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
       const data = dataCreator(e as any);
 
-      // Latency tracking
+      const message = createMessage(data, "input");
-      const tracker = new LatencyTracker("input-mouse");
+      this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
-      tracker.addTimestamp("client_send");
-      const protoTracker: ProtoLatencyTracker = {
-        $typeName: "proto.ProtoLatencyTracker",
-        sequenceId: tracker.sequence_id,
-        timestamps: [],
-      };
-      for (const t of tracker.timestamps) {
-        protoTracker.timestamps.push({
-          $typeName: "proto.ProtoTimestampEntry",
-          stage: t.stage,
-          time: timestampFromDate(t.time),
-        } as ProtoTimestampEntry);
-      }
-
-      const message: ProtoMessageInput = {
-        $typeName: "proto.ProtoMessageInput",
-        messageBase: {
-          $typeName: "proto.ProtoMessageBase",
-          payloadType: "input",
-          latency: protoTracker,
-        } as ProtoMessageBase,
-        data: data,
-      };
-      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 

packages/input/src/proto/latency_tracker_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
+// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
 // @generated from file latency_tracker.proto (package proto, syntax proto3)
 /* eslint-disable */
 

packages/input/src/proto/messages_pb.ts

@@ -1,10 +1,10 @@
-// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
+// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
 // @generated from file messages.proto (package proto, syntax proto3)
 /* eslint-disable */
 
 import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
 import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
-import type { ProtoInput } from "./types_pb";
+import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerAxis, ProtoControllerButton, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStick, ProtoControllerTrigger, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb";
 import { file_types } from "./types_pb";
 import type { ProtoLatencyTracker } from "./latency_tracker_pb";
 import { file_latency_tracker } from "./latency_tracker_pb";
@@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf";
  * Describes the file messages.proto.
  */
 export const file_messages: GenFile = /*@__PURE__*/
-  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
+  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiyQgKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9idXR0b24YCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJCdXR0b25IABI7ChJjb250cm9sbGVyX3RyaWdnZXIYDCABKAsyHS5wcm90by5Qcm90b0NvbnRyb2xsZXJUcmlnZ2VySAASNwoQY29udHJvbGxlcl9zdGljaxgNIAEoCzIbLnByb3RvLlByb3RvQ29udHJvbGxlclN0aWNrSAASNQoPY29udHJvbGxlcl9heGlzGA4gASgLMhoucHJvdG8uUHJvdG9Db250cm9sbGVyQXhpc0gAEjkKEWNvbnRyb2xsZXJfcnVtYmxlGA8gASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyUnVtYmxlSAASHgoDaWNlGBQgASgLMg8ucHJvdG8uUHJvdG9JQ0VIABIeCgNzZHAYFSABKAsyDy5wcm90by5Qcm90b1NEUEgAEh4KA3JhdxgWIAEoCzIPLnByb3RvLlByb3RvUmF3SAASSQoaY2xpZW50X3JlcXVlc3Rfcm9vbV9zdHJlYW0YFyABKAsyIy5wcm90by5Qcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtSAASPQoTY2xpZW50X2Rpc2Nvbm5lY3RlZBgYIAEoCzIeLnByb3RvLlByb3RvQ2xpZW50RGlzY29ubmVjdGVkSAASOgoSc2VydmVyX3B1c2hfc3RyZWFtGBkgASgLMhwucHJvdG8uUHJvdG9TZXJ2ZXJQdXNoU3RyZWFtSABCCQoHcGF5bG9hZEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
 
 /**
  * @generated from message proto.ProtoMessageBase
@@ -39,24 +39,148 @@ export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*
   messageDesc(file_messages, 0);
 
 /**
- * @generated from message proto.ProtoMessageInput
+ * @generated from message proto.ProtoMessage
  */
-export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
+export type ProtoMessage = Message<"proto.ProtoMessage"> & {
   /**
    * @generated from field: proto.ProtoMessageBase message_base = 1;
    */
   messageBase?: ProtoMessageBase;
 
   /**
-   * @generated from field: proto.ProtoInput data = 2;
+   * @generated from oneof proto.ProtoMessage.payload
    */
-  data?: ProtoInput;
+  payload: {
+    /**
+     * Input types
+     *
+     * @generated from field: proto.ProtoMouseMove mouse_move = 2;
+     */
+    value: ProtoMouseMove;
+    case: "mouseMove";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3;
+     */
+    value: ProtoMouseMoveAbs;
+    case: "mouseMoveAbs";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 4;
+     */
+    value: ProtoMouseWheel;
+    case: "mouseWheel";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5;
+     */
+    value: ProtoMouseKeyDown;
+    case: "mouseKeyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6;
+     */
+    value: ProtoMouseKeyUp;
+    case: "mouseKeyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyDown key_down = 7;
+     */
+    value: ProtoKeyDown;
+    case: "keyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyUp key_up = 8;
+     */
+    value: ProtoKeyUp;
+    case: "keyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerAttach controller_attach = 9;
+     */
+    value: ProtoControllerAttach;
+    case: "controllerAttach";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerDetach controller_detach = 10;
+     */
+    value: ProtoControllerDetach;
+    case: "controllerDetach";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerButton controller_button = 11;
+     */
+    value: ProtoControllerButton;
+    case: "controllerButton";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerTrigger controller_trigger = 12;
+     */
+    value: ProtoControllerTrigger;
+    case: "controllerTrigger";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerStick controller_stick = 13;
+     */
+    value: ProtoControllerStick;
+    case: "controllerStick";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerAxis controller_axis = 14;
+     */
+    value: ProtoControllerAxis;
+    case: "controllerAxis";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerRumble controller_rumble = 15;
+     */
+    value: ProtoControllerRumble;
+    case: "controllerRumble";
+  } | {
+    /**
+     * Signaling types
+     *
+     * @generated from field: proto.ProtoICE ice = 20;
+     */
+    value: ProtoICE;
+    case: "ice";
+  } | {
+    /**
+     * @generated from field: proto.ProtoSDP sdp = 21;
+     */
+    value: ProtoSDP;
+    case: "sdp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoRaw raw = 22;
+     */
+    value: ProtoRaw;
+    case: "raw";
+  } | {
+    /**
+     * @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23;
+     */
+    value: ProtoClientRequestRoomStream;
+    case: "clientRequestRoomStream";
+  } | {
+    /**
+     * @generated from field: proto.ProtoClientDisconnected client_disconnected = 24;
+     */
+    value: ProtoClientDisconnected;
+    case: "clientDisconnected";
+  } | {
+    /**
+     * @generated from field: proto.ProtoServerPushStream server_push_stream = 25;
+     */
+    value: ProtoServerPushStream;
+    case: "serverPushStream";
+  } | { case: undefined; value?: undefined };
 };
 
 /**
- * Describes the message proto.ProtoMessageInput.
+ * Describes the message proto.ProtoMessage.
- * Use `create(ProtoMessageInputSchema)` to create a new message.
+ * Use `create(ProtoMessageSchema)` to create a new message.
  */
-export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
+export const ProtoMessageSchema: GenMessage<ProtoMessage> = /*@__PURE__*/
   messageDesc(file_messages, 1);
 

packages/input/src/proto/types_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
+// @generated by protoc-gen-es v2.10.0 with parameter "target=ts"
 // @generated from file types.proto (package proto, syntax proto3)
 /* eslint-disable */
 
@@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf";
  * Describes the file types.proto.
  */
 export const file_types: GenFile = /*@__PURE__*/
-  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
+  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSJiChVQcm90b0NvbnRyb2xsZXJCdXR0b24SFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSDgoGYnV0dG9uGAMgASgFEg8KB3ByZXNzZWQYBCABKAgiYgoWUHJvdG9Db250cm9sbGVyVHJpZ2dlchIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFImUKFFByb3RvQ29udHJvbGxlclN0aWNrEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEg0KBXN0aWNrGAMgASgFEgkKAXgYBCABKAUSCQoBeRgFIAEoBSJcChNQcm90b0NvbnRyb2xsZXJBeGlzEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEgwKBGF4aXMYAyABKAUSDQoFdmFsdWUYBCABKAUiggEKFVByb3RvQ29udHJvbGxlclJ1bWJsZRIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCRIVCg1sb3dfZnJlcXVlbmN5GAMgASgFEhYKDmhpZ2hfZnJlcXVlbmN5GAQgASgFEhAKCGR1cmF0aW9uGAUgASgFIqoBChNSVENJY2VDYW5kaWRhdGVJbml0EhEKCWNhbmRpZGF0ZRgBIAEoCRIaCg1zZHBNTGluZUluZGV4GAIgASgNSACIAQESEwoGc2RwTWlkGAMgASgJSAGIAQESHQoQdXNlcm5hbWVGcmFnbWVudBgEIAEoCUgCiAEBQhAKDl9zZHBNTGluZUluZGV4QgkKB19zZHBNaWRCEwoRX3VzZXJuYW1lRnJhZ21lbnQiNgoZUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdBILCgNzZHAYASABKAkSDAoEdHlwZRgCIAEoCSI5CghQcm90b0lDRRItCgljYW5kaWRhdGUYASABKAsyGi5wcm90by5SVENJY2VDYW5kaWRhdGVJbml0IjkKCFByb3RvU0RQEi0KA3NkcBgBIAEoCzIgLnByb3RvLlJUQ1Nlc3Npb25EZXNjcmlwdGlvbkluaXQiGAoIUHJvdG9SYXcSDAoEZGF0YRgBIAEoCSJFChxQcm90b0NsaWVudFJlcXVlc3RSb29tU3RyZWFtEhEKCXJvb21fbmFtZRgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJIkcKF1Byb3RvQ2xpZW50RGlzY29ubmVjdGVkEhIKCnNlc3Npb25faWQYASABKAkSGAoQY29udHJvbGxlcl9zbG90cxgCIAMoBSIqChVQcm90b1NlcnZlclB1c2hTdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
 
 /**
  * MouseMove message
@@ -19,19 +19,12 @@ export const file_types: GenFile = /*@__PURE__*/
  */
 export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
   /**
-   * Fixed value "MouseMove"
+   * @generated from field: int32 x = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 3;
+   * @generated from field: int32 y = 2;
    */
   y: number;
 };
@@ -50,19 +43,12 @@ export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
  */
 export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
   /**
-   * Fixed value "MouseMoveAbs"
+   * @generated from field: int32 x = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 3;
+   * @generated from field: int32 y = 2;
    */
   y: number;
 };
@@ -81,19 +67,12 @@ export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE_
  */
 export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
   /**
-   * Fixed value "MouseWheel"
+   * @generated from field: int32 x = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 3;
+   * @generated from field: int32 y = 2;
    */
   y: number;
 };
@@ -112,14 +91,7 @@ export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
  */
 export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
   /**
-   * Fixed value "MouseKeyDown"
+   * @generated from field: int32 key = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -138,14 +110,7 @@ export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE_
  */
 export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
   /**
-   * Fixed value "MouseKeyUp"
+   * @generated from field: int32 key = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -164,14 +129,7 @@ export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
  */
 export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
   /**
-   * Fixed value "KeyDown"
+   * @generated from field: int32 key = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -190,14 +148,7 @@ export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
  */
 export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
   /**
-   * Fixed value "KeyUp"
+   * @generated from field: int32 key = 1;
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
-  /**
-   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -215,26 +166,26 @@ export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
  * @generated from message proto.ProtoControllerAttach
  */
 export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & {
-  /**
-   * Fixed value "ControllerAttach"
-   *
-   * @generated from field: string type = 1;
-   */
-  type: string;
-
   /**
    * One of the following enums: "ps", "xbox" or "switch"
    *
-   * @generated from field: string id = 2;
+   * @generated from field: string id = 1;
    */
   id: string;
 
   /**
-   * Slot number (0-3)
+   * Session specific slot number (0-3)
    *
-   * @generated from field: int32 slot = 3;
+   * @generated from field: int32 session_slot = 2;
    */
-  slot: number;
+  sessionSlot: number;
+
+  /**
+   * Session ID of the client
+   *
+   * @generated from field: string session_id = 3;
+   */
+  sessionId: string;
 };
 
 /**
@@ -251,18 +202,18 @@ export const ProtoControllerAttachSchema: GenMessage<ProtoControllerAttach> = /*
  */
 export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
   /**
-   * Fixed value "ControllerDetach"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 };
 
 /**
@@ -279,18 +230,18 @@ export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*
  */
 export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & {
   /**
-   * Fixed value "ControllerButtons"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 
   /**
    * Button code (linux input event code)
@@ -321,18 +272,18 @@ export const ProtoControllerButtonSchema: GenMessage<ProtoControllerButton> = /*
  */
 export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & {
   /**
-   * Fixed value "ControllerTriggers"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 
   /**
    * Trigger number (0 for left, 1 for right)
@@ -363,18 +314,18 @@ export const ProtoControllerTriggerSchema: GenMessage<ProtoControllerTrigger> =
  */
 export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & {
   /**
-   * Fixed value "ControllerStick"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 
   /**
    * Stick number (0 for left, 1 for right)
@@ -412,18 +363,18 @@ export const ProtoControllerStickSchema: GenMessage<ProtoControllerStick> = /*@_
  */
 export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & {
   /**
-   * Fixed value "ControllerAxis"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 
   /**
    * Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
@@ -454,18 +405,18 @@ export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__P
  */
 export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
   /**
-   * Fixed value "ControllerRumble"
+   * Session specific slot number (0-3)
    *
-   * @generated from field: string type = 1;
+   * @generated from field: int32 session_slot = 1;
    */
-  type: string;
+  sessionSlot: number;
 
   /**
-   * Slot number (0-3)
+   * Session ID of the client
    *
-   * @generated from field: int32 slot = 2;
+   * @generated from field: string session_id = 2;
    */
-  slot: number;
+  sessionId: string;
 
   /**
    * Low frequency rumble (0-65535)
@@ -497,105 +448,180 @@ export const ProtoControllerRumbleSchema: GenMessage<ProtoControllerRumble> = /*
   messageDesc(file_types, 13);
 
 /**
- * Union of all Input types
+ * @generated from message proto.RTCIceCandidateInit
- *
- * @generated from message proto.ProtoInput
  */
-export type ProtoInput = Message<"proto.ProtoInput"> & {
+export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & {
   /**
-   * @generated from oneof proto.ProtoInput.input_type
+   * @generated from field: string candidate = 1;
    */
-  inputType: {
+  candidate: string;
+
   /**
-     * @generated from field: proto.ProtoMouseMove mouse_move = 1;
+   * @generated from field: optional uint32 sdpMLineIndex = 2;
    */
-    value: ProtoMouseMove;
+  sdpMLineIndex?: number;
-    case: "mouseMove";
+
-  } | {
   /**
-     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
+   * @generated from field: optional string sdpMid = 3;
    */
-    value: ProtoMouseMoveAbs;
+  sdpMid?: string;
-    case: "mouseMoveAbs";
+
-  } | {
   /**
-     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
+   * @generated from field: optional string usernameFragment = 4;
    */
-    value: ProtoMouseWheel;
+  usernameFragment?: string;
-    case: "mouseWheel";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
-     */
-    value: ProtoMouseKeyDown;
-    case: "mouseKeyDown";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
-     */
-    value: ProtoMouseKeyUp;
-    case: "mouseKeyUp";
-  } | {
-    /**
-     * @generated from field: proto.ProtoKeyDown key_down = 6;
-     */
-    value: ProtoKeyDown;
-    case: "keyDown";
-  } | {
-    /**
-     * @generated from field: proto.ProtoKeyUp key_up = 7;
-     */
-    value: ProtoKeyUp;
-    case: "keyUp";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerAttach controller_attach = 8;
-     */
-    value: ProtoControllerAttach;
-    case: "controllerAttach";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerDetach controller_detach = 9;
-     */
-    value: ProtoControllerDetach;
-    case: "controllerDetach";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerButton controller_button = 10;
-     */
-    value: ProtoControllerButton;
-    case: "controllerButton";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerTrigger controller_trigger = 11;
-     */
-    value: ProtoControllerTrigger;
-    case: "controllerTrigger";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerStick controller_stick = 12;
-     */
-    value: ProtoControllerStick;
-    case: "controllerStick";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerAxis controller_axis = 13;
-     */
-    value: ProtoControllerAxis;
-    case: "controllerAxis";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerRumble controller_rumble = 14;
-     */
-    value: ProtoControllerRumble;
-    case: "controllerRumble";
-  } | { case: undefined; value?: undefined };
 };
 
 /**
- * Describes the message proto.ProtoInput.
+ * Describes the message proto.RTCIceCandidateInit.
- * Use `create(ProtoInputSchema)` to create a new message.
+ * Use `create(RTCIceCandidateInitSchema)` to create a new message.
  */
-export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
+export const RTCIceCandidateInitSchema: GenMessage<RTCIceCandidateInit> = /*@__PURE__*/
   messageDesc(file_types, 14);
 
+/**
+ * @generated from message proto.RTCSessionDescriptionInit
+ */
+export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & {
+  /**
+   * @generated from field: string sdp = 1;
+   */
+  sdp: string;
+
+  /**
+   * @generated from field: string type = 2;
+   */
+  type: string;
+};
+
+/**
+ * Describes the message proto.RTCSessionDescriptionInit.
+ * Use `create(RTCSessionDescriptionInitSchema)` to create a new message.
+ */
+export const RTCSessionDescriptionInitSchema: GenMessage<RTCSessionDescriptionInit> = /*@__PURE__*/
+  messageDesc(file_types, 15);
+
+/**
+ * ProtoICE message
+ *
+ * @generated from message proto.ProtoICE
+ */
+export type ProtoICE = Message<"proto.ProtoICE"> & {
+  /**
+   * @generated from field: proto.RTCIceCandidateInit candidate = 1;
+   */
+  candidate?: RTCIceCandidateInit;
+};
+
+/**
+ * Describes the message proto.ProtoICE.
+ * Use `create(ProtoICESchema)` to create a new message.
+ */
+export const ProtoICESchema: GenMessage<ProtoICE> = /*@__PURE__*/
+  messageDesc(file_types, 16);
+
+/**
+ * ProtoSDP message
+ *
+ * @generated from message proto.ProtoSDP
+ */
+export type ProtoSDP = Message<"proto.ProtoSDP"> & {
+  /**
+   * @generated from field: proto.RTCSessionDescriptionInit sdp = 1;
+   */
+  sdp?: RTCSessionDescriptionInit;
+};
+
+/**
+ * Describes the message proto.ProtoSDP.
+ * Use `create(ProtoSDPSchema)` to create a new message.
+ */
+export const ProtoSDPSchema: GenMessage<ProtoSDP> = /*@__PURE__*/
+  messageDesc(file_types, 17);
+
+/**
+ * ProtoRaw message
+ *
+ * @generated from message proto.ProtoRaw
+ */
+export type ProtoRaw = Message<"proto.ProtoRaw"> & {
+  /**
+   * @generated from field: string data = 1;
+   */
+  data: string;
+};
+
+/**
+ * Describes the message proto.ProtoRaw.
+ * Use `create(ProtoRawSchema)` to create a new message.
+ */
+export const ProtoRawSchema: GenMessage<ProtoRaw> = /*@__PURE__*/
+  messageDesc(file_types, 18);
+
+/**
+ * ProtoClientRequestRoomStream message
+ *
+ * @generated from message proto.ProtoClientRequestRoomStream
+ */
+export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & {
+  /**
+   * @generated from field: string room_name = 1;
+   */
+  roomName: string;
+
+  /**
+   * @generated from field: string session_id = 2;
+   */
+  sessionId: string;
+};
+
+/**
+ * Describes the message proto.ProtoClientRequestRoomStream.
+ * Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message.
+ */
+export const ProtoClientRequestRoomStreamSchema: GenMessage<ProtoClientRequestRoomStream> = /*@__PURE__*/
+  messageDesc(file_types, 19);
+
+/**
+ * ProtoClientDisconnected message
+ *
+ * @generated from message proto.ProtoClientDisconnected
+ */
+export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & {
+  /**
+   * @generated from field: string session_id = 1;
+   */
+  sessionId: string;
+
+  /**
+   * @generated from field: repeated int32 controller_slots = 2;
+   */
+  controllerSlots: number[];
+};
+
+/**
+ * Describes the message proto.ProtoClientDisconnected.
+ * Use `create(ProtoClientDisconnectedSchema)` to create a new message.
+ */
+export const ProtoClientDisconnectedSchema: GenMessage<ProtoClientDisconnected> = /*@__PURE__*/
+  messageDesc(file_types, 20);
+
+/**
+ * ProtoServerPushStream message
+ *
+ * @generated from message proto.ProtoServerPushStream
+ */
+export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & {
+  /**
+   * @generated from field: string room_name = 1;
+   */
+  roomName: string;
+};
+
+/**
+ * Describes the message proto.ProtoServerPushStream.
+ * Use `create(ProtoServerPushStreamSchema)` to create a new message.
+ */
+export const ProtoServerPushStreamSchema: GenMessage<ProtoServerPushStream> = /*@__PURE__*/
+  messageDesc(file_types, 21);
+

packages/input/src/streamwrapper.ts

@@ -0,0 +1,81 @@
+import { pbStream, type ProtobufStream } from "@libp2p/utils";
+import type { Stream } from "@libp2p/interface";
+import { bufbuildAdapter } from "./utils";
+import {
+  ProtoMessage,
+  ProtoMessageSchema,
+  ProtoMessageBase,
+} from "./proto/messages_pb";
+
+type MessageHandler = (
+  data: any,
+  base: ProtoMessageBase,
+) => void | Promise<void>;
+
+export class P2PMessageStream {
+  private pb: ProtobufStream;
+  private handlers = new Map<string, MessageHandler[]>();
+  private closed = false;
+  private readLoopRunning = false;
+
+  constructor(stream: Stream) {
+    this.pb = pbStream(stream);
+  }
+
+  public on(payloadType: string, handler: MessageHandler): void {
+    if (!this.handlers.has(payloadType)) {
+      this.handlers.set(payloadType, []);
+    }
+    this.handlers.get(payloadType)!.push(handler);
+
+    if (!this.readLoopRunning) this.startReading().catch(console.error);
+  }
+
+  private async startReading(): Promise<void> {
+    if (this.readLoopRunning || this.closed) return;
+    this.readLoopRunning = true;
+
+    while (!this.closed) {
+      try {
+        const msg: ProtoMessage = await this.pb.read(
+          bufbuildAdapter(ProtoMessageSchema),
+        );
+
+        const payloadType = msg.messageBase?.payloadType;
+        if (payloadType && this.handlers.has(payloadType)) {
+          const handlers = this.handlers.get(payloadType)!;
+          if (msg.payload.value) {
+            for (const handler of handlers) {
+              try {
+                await handler(msg.payload.value, msg.messageBase);
+              } catch (err) {
+                console.error(`Error in handler for ${payloadType}:`, err);
+              }
+            }
+          }
+        }
+      } catch (err) {
+        if (this.closed) break;
+        console.error("Stream read error:", err);
+        this.close();
+      }
+    }
+
+    this.readLoopRunning = false;
+  }
+
+  public async write(
+    message: ProtoMessage,
+    options?: { signal?: AbortSignal },
+  ): Promise<void> {
+    if (this.closed)
+      throw new Error("Cannot write to closed stream");
+
+    await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options);
+  }
+
+  public close(): void {
+    this.closed = true;
+    this.handlers.clear();
+  }
+}

packages/input/src/utils.ts

@@ -0,0 +1,95 @@
+import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
+import type { Message } from "@bufbuild/protobuf";
+import { Uint8ArrayList } from "uint8arraylist";
+import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
+import { timestampFromDate } from "@bufbuild/protobuf/wkt";
+import {
+  ProtoLatencyTracker,
+  ProtoLatencyTrackerSchema,
+  ProtoTimestampEntrySchema,
+} from "./proto/latency_tracker_pb";
+import {
+  ProtoMessage,
+  ProtoMessageSchema,
+  ProtoMessageBaseSchema,
+} from "./proto/messages_pb";
+
+export function bufbuildAdapter<T extends Message>(schema: GenMessage<T>) {
+  return {
+    encode: (data: T): Uint8Array => {
+      return toBinary(schema, data);
+    },
+    decode: (data: Uint8Array | Uint8ArrayList): T => {
+      // Convert Uint8ArrayList to Uint8Array if needed
+      const bytes = data instanceof Uint8ArrayList ? data.subarray() : data;
+      return fromBinary(schema, bytes);
+    },
+  };
+}
+
+// Latency tracker helpers
+export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker {
+  return create(ProtoLatencyTrackerSchema, {
+    sequenceId: sequenceId || crypto.randomUUID(),
+    timestamps: [],
+  });
+}
+
+export function addLatencyTimestamp(
+  tracker: ProtoLatencyTracker,
+  stage: string,
+): ProtoLatencyTracker {
+  const entry = create(ProtoTimestampEntrySchema, {
+    stage,
+    time: timestampFromDate(new Date()),
+  });
+
+  return {
+    ...tracker,
+    timestamps: [...tracker.timestamps, entry],
+  };
+}
+
+interface CreateMessageOptions {
+  sequenceId?: string;
+}
+
+function derivePayloadCase(data: Message): string {
+  // Extract case from $typeName: "proto.ProtoICE" -> "ice"
+  // "proto.ProtoControllerAttach" -> "controllerAttach"
+  const typeName = data.$typeName;
+  if (!typeName)
+    throw new Error("Message has no $typeName");
+
+  // Remove "proto.Proto" prefix and convert first char to lowercase
+  const caseName = typeName.replace(/^proto\.Proto/, "");
+
+  // Convert PascalCase to camelCase
+  // If it's all caps (like SDP, ICE), lowercase everything
+  // Otherwise, just lowercase the first character
+  if (caseName === caseName.toUpperCase()) {
+    return caseName.toLowerCase();
+  }
+  return caseName.charAt(0).toLowerCase() + caseName.slice(1);
+}
+
+export function createMessage(
+  data: Message,
+  payloadType: string,
+  options?: CreateMessageOptions,
+): ProtoMessage {
+  const payloadCase = derivePayloadCase(data);
+
+  return create(ProtoMessageSchema, {
+    messageBase: create(ProtoMessageBaseSchema, {
+      payloadType,
+      latency: options?.sequenceId
+        ? createLatencyTracker(options.sequenceId)
+        : undefined,
+    }),
+    payload: {
+      case: payloadCase,
+      value: data,
+    } as any, // Type assertion needed for dynamic case
+  });
+}

packages/input/src/webrtc-stream.ts

@@ -1,9 +1,3 @@
-import {
-  NewMessageRaw,
-  NewMessageSDP,
-  NewMessageICE,
-  SafeStream,
-} from "./messages";
 import { webSockets } from "@libp2p/websockets";
 import { webTransport } from "@libp2p/webtransport";
 import { createLibp2p, Libp2p } from "libp2p";
@@ -13,19 +7,33 @@ import { identify } from "@libp2p/identify";
 import { multiaddr } from "@multiformats/multiaddr";
 import { Connection } from "@libp2p/interface";
 import { ping } from "@libp2p/ping";
+import { createMessage } from "./utils";
+import { create } from "@bufbuild/protobuf";
+import {
+  ProtoClientRequestRoomStream,
+  ProtoClientRequestRoomStreamSchema,
+  ProtoICE,
+  ProtoICESchema, ProtoRaw,
+  ProtoSDP,
+  ProtoSDPSchema
+} from "./proto/types_pb";
+import { P2PMessageStream } from "./streamwrapper";
 
 const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
 
 export class WebRTCStream {
+  private _sessionId: string | null = null;
   private _p2p: Libp2p | undefined = undefined;
   private _p2pConn: Connection | undefined = undefined;
-  private _p2pSafeStream: SafeStream | undefined = undefined;
+  private _msgStream: P2PMessageStream | undefined = undefined;
   private _pc: RTCPeerConnection | undefined = undefined;
   private _audioTrack: MediaStreamTrack | undefined = undefined;
   private _videoTrack: MediaStreamTrack | undefined = undefined;
   private _dataChannel: RTCDataChannel | undefined = undefined;
-  private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
+  private _onConnected: ((stream: MediaStream | null) => void) | undefined =
-  private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined;
+    undefined;
+  private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined =
+    undefined;
   private _serverURL: string | undefined = undefined;
   private _roomName: string | undefined = undefined;
   private _isConnected: boolean = false;
@@ -89,14 +97,20 @@ export class WebRTCStream {
         .newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
         .catch(console.error);
       if (stream) {
-        this._p2pSafeStream = new SafeStream(stream);
+        this._msgStream = new P2PMessageStream(stream);
         console.log("Stream opened with peer");
 
         let iceHolder: RTCIceCandidateInit[] = [];
-        this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
+        this._msgStream.on("ice-candidate", (data: ProtoICE) => {
+          const cand: RTCIceCandidateInit = {
+            candidate: data.candidate.candidate,
+            sdpMLineIndex: data.candidate.sdpMLineIndex,
+            sdpMid: data.candidate.sdpMid,
+            usernameFragment: data.candidate.usernameFragment,
+          };
           if (this._pc) {
             if (this._pc.remoteDescription) {
-              this._pc.addIceCandidate(data.candidate).catch((err) => {
+              this._pc.addIceCandidate(cand).catch((err) => {
                 console.error("Error adding ICE candidate:", err);
               });
               // Add held candidates
@@ -107,45 +121,72 @@ export class WebRTCStream {
               });
               iceHolder = [];
             } else {
-              iceHolder.push(data.candidate);
+              iceHolder.push(cand);
             }
           } else {
-            iceHolder.push(data.candidate);
+            iceHolder.push(cand);
           }
         });
 
-        this._p2pSafeStream.registerCallback("offer", async (data) => {
+        this._msgStream.on("session-assigned", (data: ProtoClientRequestRoomStream) => {
+          this._sessionId = data.sessionId;
+          localStorage.setItem("nestri-session-id", this._sessionId);
+          console.log("Session ID assigned:", this._sessionId, "for room:", data.roomName);
+        });
+
+        this._msgStream.on("offer", async (data: ProtoSDP) => {
           if (!this._pc) {
             // Setup peer connection now
             this._setupPeerConnection();
           }
-          await this._pc!.setRemoteDescription(data.sdp);
+          await this._pc!.setRemoteDescription({
+            sdp: data.sdp.sdp,
+            type: data.sdp.type as RTCSdpType,
+          });
           // Create our answer
           const answer = await this._pc!.createAnswer();
           // Force stereo in Chromium browsers
           answer.sdp = this.forceOpusStereo(answer.sdp!);
           await this._pc!.setLocalDescription(answer);
           // Send answer back
-          const answerMsg = NewMessageSDP("answer", answer);
+          const answerMsg = createMessage(
-          await this._p2pSafeStream?.writeMessage(answerMsg);
+            create(ProtoSDPSchema, {
+              sdp: answer,
+            }),
+            "answer",
+          );
+          await this._msgStream?.write(answerMsg);
         });
 
-        this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
+        this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => {
-          console.warn("Stream is offline for room:", data.roomName);
+          console.warn("Stream is offline for room:", msg.data);
           this._onConnected?.(null);
         });
 
+        const clientId = this.getSessionID();
+        if (clientId) {
+          console.debug("Using existing session ID:", clientId);
+        }
+
         // Send stream request
-        // marshal room name into json
+        const requestMsg = createMessage(
-        const request = NewMessageRaw(
+          create(ProtoClientRequestRoomStreamSchema, {
+            roomName: roomName,
+            sessionId: clientId,
+          }),
           "request-stream-room",
-          roomName,
         );
-        await this._p2pSafeStream.writeMessage(request);
+        await this._msgStream.write(requestMsg);
       }
     }
   }
 
+  public getSessionID(): string | null {
+    if (this._sessionId === null)
+      this._sessionId = localStorage.getItem("nestri-session-id");
+    return this._sessionId;
+  }
+
   // Forces opus to stereo in Chromium browsers, because of course
   private forceOpusStereo(SDP: string): string {
     // Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
@@ -200,11 +241,16 @@ export class WebRTCStream {
 
     this._pc.onicecandidate = (e) => {
       if (e.candidate) {
-        const iceMsg = NewMessageICE("ice-candidate", e.candidate);
+        const iceMsg = createMessage(
-        if (this._p2pSafeStream) {
+          create(ProtoICESchema, {
-          this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
+            candidate: e.candidate,
-            console.error("Error sending ICE candidate:", err),
+          }),
+          "ice-candidate",
         );
+        if (this._msgStream) {
+          this._msgStream
+            .write(iceMsg)
+            .catch((err) => console.error("Error sending ICE candidate:", err));
         } else {
           console.warn("P2P stream not established, cannot send ICE candidate");
         }
@@ -218,8 +264,7 @@ export class WebRTCStream {
   }
 
   private _checkConnectionState() {
-    if (!this._pc || !this._p2p || !this._p2pConn)
+    if (!this._pc || !this._p2p || !this._p2pConn) return;
-      return;
 
     console.debug("Checking connection state:", {
       connectionState: this._pc.connectionState,
@@ -256,7 +301,7 @@ export class WebRTCStream {
                 // @ts-ignore
                 receiver.jitterBufferTarget = receiver.jitterBufferDelayHint = receiver.playoutDelayHint = 0;
               }
-            }, 15);
+            }, 50);
           });
         }
       }
@@ -286,7 +331,9 @@ export class WebRTCStream {
 
     // Attempt to reconnect only if not already connected
     if (!this._isConnected && this._serverURL && this._roomName) {
-      this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
+      this._setup(this._serverURL, this._roomName).catch((err) =>
+        console.error("Reconnection failed:", err),
+      );
     }
   }
 
@@ -335,7 +382,9 @@ export class WebRTCStream {
   }
 
   public removeDataChannelCallback(callback: (data: any) => void) {
-    this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback);
+    this._dataChannelCallbacks = this._dataChannelCallbacks.filter(
+      (cb) => cb !== callback,
+    );
   }
 
   private _setupDataChannelEvents() {
@@ -343,7 +392,7 @@ export class WebRTCStream {
 
     this._dataChannel.onclose = () => console.log("sendChannel has closed");
     this._dataChannel.onopen = () => console.log("sendChannel has opened");
-    this._dataChannel.onmessage = (event => {
+    this._dataChannel.onmessage = (event) => {
       // Parse as ProtoBuf message
       const data = event.data;
       // Call registered callback if exists
@@ -354,7 +403,7 @@ export class WebRTCStream {
           console.error("Error in data channel callback:", err);
         }
       });
-    });
+    };
   }
 
   private _gatherFrameRate() {

packages/play-standalone/src/pages/[room].astro

@@ -90,11 +90,7 @@ if (envs_map.size > 0) {
   let nestriControllers: Controller[] = [];
 
   window.addEventListener("gamepadconnected", (e) => {
-    // Ignore gamepads with id including "nestri"
     console.log("Gamepad connected:", e.gamepad);
-    if (e.gamepad.id.toLowerCase().includes("nestri"))
-      return;
-
     const controller = new Controller({
       webrtc: stream,
       e: e,

packages/relay/go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/ipfs/go-cid v0.5.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/koron/go-ssdp v0.1.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect

packages/relay/go.sum

@@ -82,8 +82,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/koron/go-ssdp v0.1.0 h1:ckl5x5H6qSNFmi+wCuROvvGUu2FQnMbQrU95IHCcv3Y=

packages/relay/internal/common/common.go

@@ -26,7 +26,7 @@ func InitWebRTCAPI() error {
 	mediaEngine := &webrtc.MediaEngine{}
 
 	// Register our extensions
-	if err := RegisterExtensions(mediaEngine); err != nil {
+	if err = RegisterExtensions(mediaEngine); err != nil {
 		return fmt.Errorf("failed to register extensions: %w", err)
 	}
 

packages/relay/internal/common/safebufio.go

@@ -3,16 +3,28 @@ package common
 import (
 	"bufio"
 	"encoding/binary"
-	"encoding/json"
 	"errors"
 	"io"
+	gen "relay/internal/proto"
 	"sync"
 
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-// MaxSize is the maximum allowed data size (1MB)
+// readUvarint reads an unsigned varint from the reader
-const MaxSize = 1024 * 1024
+func readUvarint(r io.ByteReader) (uint64, error) {
+	return binary.ReadUvarint(r)
+}
+
+// writeUvarint writes an unsigned varint to the writer
+func writeUvarint(w io.Writer, x uint64) error {
+	buf := make([]byte, binary.MaxVarintLen64)
+	n := binary.PutUvarint(buf, x)
+	_, err := w.Write(buf[:n])
+	return err
+}
 
 // SafeBufioRW wraps a bufio.ReadWriter for sending and receiving JSON and protobufs safely
 type SafeBufioRW struct {
@@ -24,83 +36,6 @@ func NewSafeBufioRW(brw *bufio.ReadWriter) *SafeBufioRW {
 	return &SafeBufioRW{brw: brw}
 }
 
-// SendJSON serializes the given data as JSON and sends it with a 4-byte length prefix
-func (bu *SafeBufioRW) SendJSON(data interface{}) error {
-	bu.mutex.Lock()
-	defer bu.mutex.Unlock()
-
-	jsonData, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-
-	if len(jsonData) > MaxSize {
-		return errors.New("JSON data exceeds maximum size")
-	}
-
-	// Write the 4-byte length prefix
-	if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(jsonData))); err != nil {
-		return err
-	}
-
-	// Write the JSON data
-	if _, err = bu.brw.Write(jsonData); err != nil {
-		return err
-	}
-
-	// Flush the writer to ensure data is sent
-	return bu.brw.Flush()
-}
-
-// ReceiveJSON reads a 4-byte length prefix, then reads and unmarshals the JSON
-func (bu *SafeBufioRW) ReceiveJSON(dest interface{}) error {
-	bu.mutex.RLock()
-	defer bu.mutex.RUnlock()
-
-	// Read the 4-byte length prefix
-	var length uint32
-	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
-		return err
-	}
-
-	if length > MaxSize {
-		return errors.New("received JSON data exceeds maximum size")
-	}
-
-	// Read the JSON data
-	data := make([]byte, length)
-	if _, err := io.ReadFull(bu.brw, data); err != nil {
-		return err
-	}
-
-	return json.Unmarshal(data, dest)
-}
-
-// Receive reads a 4-byte length prefix, then reads the raw data
-func (bu *SafeBufioRW) Receive() ([]byte, error) {
-	bu.mutex.RLock()
-	defer bu.mutex.RUnlock()
-
-	// Read the 4-byte length prefix
-	var length uint32
-	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
-		return nil, err
-	}
-
-	if length > MaxSize {
-		return nil, errors.New("received data exceeds maximum size")
-	}
-
-	// Read the raw data
-	data := make([]byte, length)
-	if _, err := io.ReadFull(bu.brw, data); err != nil {
-		return nil, err
-	}
-
-	return data, nil
-}
-
-// SendProto serializes the given protobuf message and sends it with a 4-byte length prefix
 func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 	bu.mutex.Lock()
 	defer bu.mutex.Unlock()
@@ -110,12 +45,8 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 		return err
 	}
 
-	if len(protoData) > MaxSize {
+	// Write varint length prefix
-		return errors.New("protobuf data exceeds maximum size")
+	if err := writeUvarint(bu.brw, uint64(len(protoData))); err != nil {
-	}
-
-	// Write the 4-byte length prefix
-	if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(protoData))); err != nil {
 		return err
 	}
 
@@ -124,25 +55,19 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 		return err
 	}
 
-	// Flush the writer to ensure data is sent
 	return bu.brw.Flush()
 }
 
-// ReceiveProto reads a 4-byte length prefix, then reads and unmarshals the protobuf
 func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
 	bu.mutex.RLock()
 	defer bu.mutex.RUnlock()
 
-	// Read the 4-byte length prefix
+	// Read varint length prefix
-	var length uint32
+	length, err := readUvarint(bu.brw)
-	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
+	if err != nil {
 		return err
 	}
 
-	if length > MaxSize {
-		return errors.New("received Protobuf data exceeds maximum size")
-	}
-
 	// Read the Protobuf data
 	data := make([]byte, length)
 	if _, err := io.ReadFull(bu.brw, data); err != nil {
@@ -152,24 +77,51 @@ func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
 	return proto.Unmarshal(data, msg)
 }
 
-// Write writes raw data to the underlying buffer
+type CreateMessageOptions struct {
-func (bu *SafeBufioRW) Write(data []byte) (int, error) {
+	SequenceID string
-	bu.mutex.Lock()
+	Latency    *gen.ProtoLatencyTracker
-	defer bu.mutex.Unlock()
+}
-
+
-	if len(data) > MaxSize {
+func CreateMessage(payload proto.Message, payloadType string, opts *CreateMessageOptions) (*gen.ProtoMessage, error) {
-		return 0, errors.New("data exceeds maximum size")
+	msg := &gen.ProtoMessage{
-	}
+		MessageBase: &gen.ProtoMessageBase{
-
+			PayloadType: payloadType,
-	n, err := bu.brw.Write(data)
+		},
-	if err != nil {
+	}
-		return n, err
+
-	}
+	if opts != nil {
-
+		if opts.Latency != nil {
-	// Flush the writer to ensure data is sent
+			msg.MessageBase.Latency = opts.Latency
-	if err = bu.brw.Flush(); err != nil {
+		} else if opts.SequenceID != "" {
-		return n, err
+			msg.MessageBase.Latency = &gen.ProtoLatencyTracker{
-	}
+				SequenceId: opts.SequenceID,
-
+				Timestamps: []*gen.ProtoTimestampEntry{
-	return n, nil
+					{
+						Stage: "created",
+						Time:  timestamppb.Now(),
+					},
+				},
+			}
+		}
+	}
+
+	// Use reflection to set the oneof field automatically
+	msgReflect := msg.ProtoReflect()
+	payloadReflect := payload.ProtoReflect()
+
+	oneofDesc := msgReflect.Descriptor().Oneofs().ByName("payload")
+	if oneofDesc == nil {
+		return nil, errors.New("payload oneof not found")
+	}
+
+	fields := oneofDesc.Fields()
+	for i := 0; i < fields.Len(); i++ {
+		field := fields.Get(i)
+		if field.Message() != nil && field.Message().FullName() == payloadReflect.Descriptor().FullName() {
+			msgReflect.Set(field, protoreflect.ValueOfMessage(payloadReflect))
+			return msg, nil
+		}
+	}
+
+	return nil, errors.New("payload type not found in oneof")
 }

packages/relay/internal/connections/datachannel.go

@@ -31,16 +31,18 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
 		}
 
 		// Decode message
-		var base gen.ProtoMessageInput
+		var base gen.ProtoMessage
 		if err := proto.Unmarshal(msg.Data, &base); err != nil {
 			slog.Error("failed to decode binary DataChannel message", "err", err)
 			return
 		}
 
-		// Handle message type callback
+		// Route based on PayloadType
-		if callback, ok := ndc.callbacks["input"]; ok {
+		if base.MessageBase != nil && len(base.MessageBase.PayloadType) > 0 {
+			if callback, ok := ndc.callbacks[base.MessageBase.PayloadType]; ok {
 				go callback(msg.Data)
-		} // We don't care about unhandled messages
+			}
+		}
 	})
 
 	return ndc

packages/relay/internal/connections/messages.go

@@ -1,94 +0,0 @@
-package connections
-
-import (
-	"encoding/json"
-	"relay/internal/common"
-
-	"github.com/pion/webrtc/v4"
-)
-
-// MessageBase is the base type for any JSON message
-type MessageBase struct {
-	Type    string                 `json:"payload_type"`
-	Latency *common.LatencyTracker `json:"latency,omitempty"`
-}
-
-type MessageRaw struct {
-	MessageBase
-	Data json.RawMessage `json:"data"`
-}
-
-func NewMessageRaw(t string, data json.RawMessage) *MessageRaw {
-	return &MessageRaw{
-		MessageBase: MessageBase{
-			Type: t,
-		},
-		Data: data,
-	}
-}
-
-type MessageLog struct {
-	MessageBase
-	Level   string `json:"level"`
-	Message string `json:"message"`
-	Time    string `json:"time"`
-}
-
-func NewMessageLog(t string, level, message, time string) *MessageLog {
-	return &MessageLog{
-		MessageBase: MessageBase{
-			Type: t,
-		},
-		Level:   level,
-		Message: message,
-		Time:    time,
-	}
-}
-
-type MessageMetrics struct {
-	MessageBase
-	UsageCPU        float64 `json:"usage_cpu"`
-	UsageMemory     float64 `json:"usage_memory"`
-	Uptime          uint64  `json:"uptime"`
-	PipelineLatency float64 `json:"pipeline_latency"`
-}
-
-func NewMessageMetrics(t string, usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) *MessageMetrics {
-	return &MessageMetrics{
-		MessageBase: MessageBase{
-			Type: t,
-		},
-		UsageCPU:        usageCPU,
-		UsageMemory:     usageMemory,
-		Uptime:          uptime,
-		PipelineLatency: pipelineLatency,
-	}
-}
-
-type MessageICE struct {
-	MessageBase
-	Candidate webrtc.ICECandidateInit `json:"candidate"`
-}
-
-func NewMessageICE(t string, candidate webrtc.ICECandidateInit) *MessageICE {
-	return &MessageICE{
-		MessageBase: MessageBase{
-			Type: t,
-		},
-		Candidate: candidate,
-	}
-}
-
-type MessageSDP struct {
-	MessageBase
-	SDP webrtc.SessionDescription `json:"sdp"`
-}
-
-func NewMessageSDP(t string, sdp webrtc.SessionDescription) *MessageSDP {
-	return &MessageSDP{
-		MessageBase: MessageBase{
-			Type: t,
-		},
-		SDP: sdp,
-	}
-}

packages/relay/internal/core/core.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"relay/internal/common"
 	"relay/internal/shared"
+	"time"
 
 	"github.com/libp2p/go-libp2p"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
@@ -37,6 +38,16 @@ var globalRelay *Relay
 
 // -- Structs --
 
+// ClientSession tracks browser client connections
+type ClientSession struct {
+	PeerID          peer.ID
+	SessionID       string
+	RoomName        string
+	ConnectedAt     time.Time
+	LastActivity    time.Time
+	ControllerSlots []int32 // Track which controller slots this client owns
+}
+
 // Relay structure enhanced with metrics and state
 type Relay struct {
 	*PeerInfo
@@ -48,6 +59,7 @@ type Relay struct {
 	// Local
 	LocalRooms           *common.SafeMap[ulid.ULID, *shared.Room]         // room ID -> local Room struct (hosted by this relay)
 	LocalMeshConnections *common.SafeMap[peer.ID, *webrtc.PeerConnection] // peer ID -> PeerConnection (connected to this relay)
+	ClientSessions       *common.SafeMap[peer.ID, *ClientSession]         // peer ID -> ClientSession
 
 	// Protocols
 	ProtocolRegistry
@@ -144,6 +156,7 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay
 		PingService:          pingSvc,
 		LocalRooms:           common.NewSafeMap[ulid.ULID, *shared.Room](),
 		LocalMeshConnections: common.NewSafeMap[peer.ID, *webrtc.PeerConnection](),
+		ClientSessions:       common.NewSafeMap[peer.ID, *ClientSession](),
 	}
 
 	// Add network notifier after relay is initialized

packages/relay/internal/core/protocol_stream.go

@@ -3,14 +3,19 @@ package core
 import (
 	"bufio"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"math"
 	"relay/internal/common"
 	"relay/internal/connections"
 	"relay/internal/shared"
+	"time"
+
+	gen "relay/internal/proto"
+
+	"google.golang.org/protobuf/proto"
 
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -69,7 +74,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 	var currentRoomName string // Track the current room for this stream
 	iceHolder := make([]webrtc.ICECandidateInit, 0)
 	for {
-		data, err := safeBRW.Receive()
+		var msgWrapper gen.ProtoMessage
+		err := safeBRW.ReceiveProto(&msgWrapper)
 		if err != nil {
 			if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
 				slog.Debug("Stream request connection closed by peer", "peer", stream.Conn().RemotePeer())
@@ -82,79 +88,153 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 			return
 		}
 
-		var baseMsg connections.MessageBase
+		if msgWrapper.MessageBase == nil {
-		if err = json.Unmarshal(data, &baseMsg); err != nil {
+			slog.Error("No MessageBase in stream request")
-			slog.Error("Failed to unmarshal base message", "err", err)
+			_ = stream.Reset()
-			continue
+			return
 		}
 
-		switch baseMsg.Type {
+		switch msgWrapper.MessageBase.PayloadType {
 		case "request-stream-room":
-			var rawMsg connections.MessageRaw
+			reqMsg := msgWrapper.GetClientRequestRoomStream()
-			if err = json.Unmarshal(data, &rawMsg); err != nil {
+			if reqMsg != nil {
-				slog.Error("Failed to unmarshal raw message for room stream request", "err", err)
+				currentRoomName = reqMsg.RoomName
+
+				// Generate session ID if not provided (first connection)
+				sessionID := reqMsg.SessionId
+				if sessionID == "" {
+					ulid, err := common.NewULID()
+					if err != nil {
+						slog.Error("Failed to generate session ID", "err", err)
 						continue
 					}
-
+					sessionID = ulid.String()
-			var roomName string
-			if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
-				slog.Error("Failed to unmarshal room name from raw message", "err", err)
-				continue
 				}
 
-			currentRoomName = roomName // Store the room name
+				session := &ClientSession{
-			slog.Info("Received stream request for room", "room", roomName)
+					PeerID:       stream.Conn().RemotePeer(),
+					SessionID:    sessionID,
+					RoomName:     reqMsg.RoomName,
+					ConnectedAt:  time.Now(),
+					LastActivity: time.Now(),
+				}
+				sp.relay.ClientSessions.Set(stream.Conn().RemotePeer(), session)
 
-			room := sp.relay.GetRoomByName(roomName)
+				slog.Info("Client session established", "peer", session.PeerID, "session", sessionID, "room", reqMsg.RoomName)
+
+				// Send session ID back to client
+				sesMsg, err := common.CreateMessage(
+					&gen.ProtoClientRequestRoomStream{SessionId: sessionID, RoomName: reqMsg.RoomName},
+					"session-assigned", nil,
+				)
+				if err != nil {
+					slog.Error("Failed to create proto message", "err", err)
+					continue
+				}
+				if err = safeBRW.SendProto(sesMsg); err != nil {
+					slog.Error("Failed to send session assignment", "err", err)
+				}
+
+				slog.Info("Received stream request for room", "room", reqMsg.RoomName)
+
+				room := sp.relay.GetRoomByName(reqMsg.RoomName)
 				if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID {
 					// TODO: Allow forward requests to other relays from here?
-				slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", roomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
+					slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", reqMsg.RoomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
 					// Respond with "request-stream-offline" message with room name
 					// TODO: Store the peer and send "online" message when the room comes online
-				roomNameData, err := json.Marshal(roomName)
+					rawMsg, err := common.CreateMessage(
+						&gen.ProtoRaw{
+							Data: reqMsg.RoomName,
+						},
+						"request-stream-offline", nil,
+					)
 					if err != nil {
-					slog.Error("Failed to marshal room name for request stream offline", "room", roomName, "err", err)
+						slog.Error("Failed to create proto message", "err", err)
 						continue
-				} else {
-					if err = safeBRW.SendJSON(connections.NewMessageRaw(
-						"request-stream-offline",
-						roomNameData,
-					)); err != nil {
-						slog.Error("Failed to send request stream offline message", "room", roomName, "err", err)
 					}
+					if err = safeBRW.SendProto(rawMsg); err != nil {
+						slog.Error("Failed to send request stream offline message", "room", reqMsg.RoomName, "err", err)
 					}
 					continue
 				}
 
 				pc, err := common.CreatePeerConnection(func() {
-				slog.Info("PeerConnection closed for requested stream", "room", roomName)
+					slog.Info("PeerConnection closed for requested stream", "room", reqMsg.RoomName)
 					// Cleanup the stream connection
-				if roomMap, ok := sp.servedConns.Get(roomName); ok {
+					if roomMap, ok := sp.servedConns.Get(reqMsg.RoomName); ok {
 						roomMap.Delete(stream.Conn().RemotePeer())
 						// If the room map is empty, delete it
 						if roomMap.Len() == 0 {
-						sp.servedConns.Delete(roomName)
+							sp.servedConns.Delete(reqMsg.RoomName)
 						}
 					}
 				})
 				if err != nil {
-				slog.Error("Failed to create PeerConnection for requested stream", "room", roomName, "err", err)
+					slog.Error("Failed to create PeerConnection for requested stream", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
 
-			// Add tracks
+				// Create participant for this viewer
-			if room.AudioTrack != nil {
+				participant, err := shared.NewParticipant(
-				if _, err = pc.AddTrack(room.AudioTrack); err != nil {
+					"",
-					slog.Error("Failed to add audio track for requested stream", "room", roomName, "err", err)
+					stream.Conn().RemotePeer(),
+				)
+				if err != nil {
+					slog.Error("Failed to create participant", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
+
+				// If this is a client session, link it
+				if session, ok := sp.relay.ClientSessions.Get(stream.Conn().RemotePeer()); ok {
+					participant.SessionID = session.SessionID
 				}
-			if room.VideoTrack != nil {
+
-				if _, err = pc.AddTrack(room.VideoTrack); err != nil {
+				// Assign peer connection
-					slog.Error("Failed to add video track for requested stream", "room", roomName, "err", err)
+				participant.PeerConnection = pc
-					continue
+
+				// Add audio/video tracks
+				{
+					localTrack, err := webrtc.NewTrackLocalStaticRTP(
+						room.AudioCodec,
+						"participant-"+participant.ID.String(),
+						"participant-"+participant.ID.String()+"-audio",
+					)
+					if err != nil {
+						slog.Error("Failed to create track for stream request", "err", err)
+						return
 					}
+					participant.SetTrack(webrtc.RTPCodecTypeAudio, localTrack)
+					slog.Debug("Set audio track for requested stream", "room", room.Name)
 				}
+				{
+					localTrack, err := webrtc.NewTrackLocalStaticRTP(
+						room.VideoCodec,
+						"participant-"+participant.ID.String(),
+						"participant-"+participant.ID.String()+"-video",
+					)
+					if err != nil {
+						slog.Error("Failed to create track for stream request", "err", err)
+						return
+					}
+					participant.SetTrack(webrtc.RTPCodecTypeVideo, localTrack)
+					slog.Debug("Set video track for requested stream", "room", room.Name)
+				}
+
+				// Cleanup on disconnect
+				cleanupParticipantID := participant.ID
+				pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
+					if state == webrtc.PeerConnectionStateClosed ||
+						state == webrtc.PeerConnectionStateFailed ||
+						state == webrtc.PeerConnectionStateDisconnected {
+						slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID)
+						room.RemoveParticipantByID(cleanupParticipantID)
+						participant.Close()
+					} else if state == webrtc.PeerConnectionStateConnected {
+						// Add participant to room when connection is established
+						room.AddParticipant(participant)
+					}
+				})
 
 				// DataChannel setup
 				settingOrdered := true
@@ -164,21 +244,84 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 					MaxRetransmits: &settingMaxRetransmits,
 				})
 				if err != nil {
-				slog.Error("Failed to create DataChannel for requested stream", "room", roomName, "err", err)
+					slog.Error("Failed to create DataChannel for requested stream", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
 				ndc := connections.NewNestriDataChannel(dc)
 
 				ndc.RegisterOnOpen(func() {
-				slog.Debug("Relay DataChannel opened for requested stream", "room", roomName)
+					slog.Debug("Relay DataChannel opened for requested stream", "room", reqMsg.RoomName)
 				})
 				ndc.RegisterOnClose(func() {
-				slog.Debug("Relay DataChannel closed for requested stream", "room", roomName)
+					slog.Debug("Relay DataChannel closed for requested stream", "room", reqMsg.RoomName)
 				})
 				ndc.RegisterMessageCallback("input", func(data []byte) {
 					if room.DataChannel != nil {
 						if err = room.DataChannel.SendBinary(data); err != nil {
-						slog.Error("Failed to forward input message from mesh to upstream room", "room", roomName, "err", err)
+							slog.Error("Failed to forward input message from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
+						}
+					}
+				})
+				// Track controller input separately
+				ndc.RegisterMessageCallback("controllerInput", func(data []byte) {
+					// Parse the message to track controller slots for client sessions
+					var msgWrapper gen.ProtoMessage
+					if err = proto.Unmarshal(data, &msgWrapper); err != nil {
+						slog.Error("Failed to unmarshal controller input", "err", err)
+					} else if msgWrapper.Payload != nil {
+						// Get the peer ID for this connection
+						peerID := stream.Conn().RemotePeer()
+
+						// Check if it's a controller attach with assigned slot
+						if attach := msgWrapper.GetControllerAttach(); attach != nil && attach.SessionSlot >= 0 {
+							if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
+								// Check if slot already tracked
+								hasSlot := false
+								for _, slot := range session.ControllerSlots {
+									if slot == attach.SessionSlot {
+										hasSlot = true
+										break
+									}
+								}
+								if !hasSlot {
+									session.ControllerSlots = append(session.ControllerSlots, attach.SessionSlot)
+									session.LastActivity = time.Now()
+									slog.Info("Controller slot assigned to client session",
+										"session", session.SessionID,
+										"slot", attach.SessionSlot,
+										"total_slots", len(session.ControllerSlots))
+								}
+							}
+						}
+
+						// Check if it's a controller detach
+						if detach := msgWrapper.GetControllerDetach(); detach != nil && detach.SessionSlot >= 0 {
+							if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
+								newSlots := make([]int32, 0, len(session.ControllerSlots))
+								for _, slot := range session.ControllerSlots {
+									if slot != detach.SessionSlot {
+										newSlots = append(newSlots, slot)
+									}
+								}
+								session.ControllerSlots = newSlots
+								session.LastActivity = time.Now()
+								slog.Info("Controller slot removed from client session",
+									"session", session.SessionID,
+									"slot", detach.SessionSlot,
+									"remaining_slots", len(session.ControllerSlots))
+							}
+						}
+
+						// Update last activity on any controller input
+						if session, ok := sp.relay.ClientSessions.Get(peerID); ok {
+							session.LastActivity = time.Now()
+						}
+					}
+
+					// Forward to upstream room
+					if room.DataChannel != nil {
+						if err = room.DataChannel.SendBinary(data); err != nil {
+							slog.Error("Failed to forward controller input from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
 						}
 					}
 				})
@@ -189,8 +332,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 						return
 					}
 
-				if err = safeBRW.SendJSON(connections.NewMessageICE("ice-candidate", candidate.ToJSON())); err != nil {
+					candInit := candidate.ToJSON()
-					slog.Error("Failed to send ICE candidate message for requested stream", "room", roomName, "err", err)
+					biggified := uint32(*candInit.SDPMLineIndex)
+					iceMsg, err := common.CreateMessage(
+						&gen.ProtoICE{
+							Candidate: &gen.RTCIceCandidateInit{
+								Candidate:     candInit.Candidate,
+								SdpMLineIndex: &biggified,
+								SdpMid:        candInit.SDPMid,
+							},
+						},
+						"ice-candidate", nil,
+					)
+					if err != nil {
+						slog.Error("Failed to create proto message", "err", err)
+						return
+					}
+					if err = safeBRW.SendProto(iceMsg); err != nil {
+						slog.Error("Failed to send ICE candidate message for requested stream", "room", reqMsg.RoomName, "err", err)
 						return
 					}
 				})
@@ -198,23 +357,36 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 				// Create offer
 				offer, err := pc.CreateOffer(nil)
 				if err != nil {
-				slog.Error("Failed to create offer for requested stream", "room", roomName, "err", err)
+					slog.Error("Failed to create offer for requested stream", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
 				if err = pc.SetLocalDescription(offer); err != nil {
-				slog.Error("Failed to set local description for requested stream", "room", roomName, "err", err)
+					slog.Error("Failed to set local description for requested stream", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
-			if err = safeBRW.SendJSON(connections.NewMessageSDP("offer", offer)); err != nil {
+				offerMsg, err := common.CreateMessage(
-				slog.Error("Failed to send offer for requested stream", "room", roomName, "err", err)
+					&gen.ProtoSDP{
+						Sdp: &gen.RTCSessionDescriptionInit{
+							Sdp:  offer.SDP,
+							Type: offer.Type.String(),
+						},
+					},
+					"offer", nil,
+				)
+				if err != nil {
+					slog.Error("Failed to create proto message", "err", err)
+					continue
+				}
+				if err = safeBRW.SendProto(offerMsg); err != nil {
+					slog.Error("Failed to send offer for requested stream", "room", reqMsg.RoomName, "err", err)
 					continue
 				}
 
 				// Store the connection
-			roomMap, ok := sp.servedConns.Get(roomName)
+				roomMap, ok := sp.servedConns.Get(reqMsg.RoomName)
 				if !ok {
 					roomMap = common.NewSafeMap[peer.ID, *StreamConnection]()
-				sp.servedConns.Set(roomName, roomMap)
+					sp.servedConns.Set(reqMsg.RoomName, roomMap)
 				}
 				roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{
 					pc:  pc,
@@ -222,17 +394,24 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 				})
 
 				slog.Debug("Sent offer for requested stream")
+			} else {
+				slog.Error("Could not get ClientRequestRoomStream for stream request")
+			}
 		case "ice-candidate":
-			var iceMsg connections.MessageICE
+			iceMsg := msgWrapper.GetIce()
-			if err := json.Unmarshal(data, &iceMsg); err != nil {
+			if iceMsg != nil {
-				slog.Error("Failed to unmarshal ICE message", "err", err)
+				smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
-				continue
+				cand := webrtc.ICECandidateInit{
+					Candidate:        iceMsg.Candidate.Candidate,
+					SDPMid:           iceMsg.Candidate.SdpMid,
+					SDPMLineIndex:    &smollified,
+					UsernameFragment: iceMsg.Candidate.UsernameFragment,
 				}
 				// Use currentRoomName to get the connection from nested map
 				if len(currentRoomName) > 0 {
 					if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
 						if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil {
-						if err := conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
+							if err = conn.pc.AddICECandidate(cand); err != nil {
 								slog.Error("Failed to add ICE candidate", "err", err)
 							}
 							for _, heldIce := range iceHolder {
@@ -244,24 +423,28 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 							iceHolder = make([]webrtc.ICECandidateInit, 0)
 						} else {
 							// Hold the candidate until remote description is set
-						iceHolder = append(iceHolder, iceMsg.Candidate)
+							iceHolder = append(iceHolder, cand)
 						}
 					}
 				} else {
 					// Hold the candidate until remote description is set
-				iceHolder = append(iceHolder, iceMsg.Candidate)
+					iceHolder = append(iceHolder, cand)
+				}
+			} else {
+				slog.Error("Could not GetIce from ice-candidate")
 			}
 		case "answer":
-			var answerMsg connections.MessageSDP
+			answerMsg := msgWrapper.GetSdp()
-			if err := json.Unmarshal(data, &answerMsg); err != nil {
+			if answerMsg != nil {
-				slog.Error("Failed to unmarshal answer from signaling message", "err", err)
+				ansSdp := webrtc.SessionDescription{
-				continue
+					SDP:  answerMsg.Sdp.Sdp,
+					Type: webrtc.NewSDPType(answerMsg.Sdp.Type),
 				}
 				// Use currentRoomName to get the connection from nested map
 				if len(currentRoomName) > 0 {
 					if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
 						if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok {
-						if err := conn.pc.SetRemoteDescription(answerMsg.SDP); err != nil {
+							if err = conn.pc.SetRemoteDescription(ansSdp); err != nil {
 								slog.Error("Failed to set remote description for answer", "err", err)
 								continue
 							}
@@ -273,199 +456,11 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 				} else {
 					slog.Warn("Received answer without active PeerConnection")
 				}
-		}
-	}
-}
-
-// requestStream manages the internals of the stream request
-func (sp *StreamProtocol) requestStream(stream network.Stream, room *shared.Room) error {
-	brw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream))
-	safeBRW := common.NewSafeBufioRW(brw)
-
-	slog.Debug("Requesting room stream from peer", "room", room.Name, "peer", stream.Conn().RemotePeer())
-
-	// Send room name to the remote peer
-	roomData, err := json.Marshal(room.Name)
-	if err != nil {
-		_ = stream.Close()
-		return fmt.Errorf("failed to marshal room name: %w", err)
-	}
-	if err = safeBRW.SendJSON(connections.NewMessageRaw(
-		"request-stream-room",
-		roomData,
-	)); err != nil {
-		_ = stream.Close()
-		return fmt.Errorf("failed to send room request: %w", err)
-	}
-
-	pc, err := common.CreatePeerConnection(func() {
-		slog.Info("Relay PeerConnection closed for requested stream", "room", room.Name)
-		_ = stream.Close() // ignore error as may be closed already
-		// Cleanup the stream connection
-		if ok := sp.requestedConns.Has(room.Name); ok {
-			sp.requestedConns.Delete(room.Name)
-		}
-	})
-	if err != nil {
-		_ = stream.Close()
-		return fmt.Errorf("failed to create PeerConnection: %w", err)
-	}
-
-	pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
-		localTrack, _ := webrtc.NewTrackLocalStaticRTP(track.Codec().RTPCodecCapability, track.ID(), "relay-"+room.Name+"-"+track.Kind().String())
-		slog.Debug("Received track for requested stream", "room", room.Name, "track_kind", track.Kind().String())
-
-		room.SetTrack(track.Kind(), localTrack)
-
-		go func() {
-			for {
-				rtpPacket, _, err := track.ReadRTP()
-				if err != nil {
-					if !errors.Is(err, io.EOF) {
-						slog.Error("Failed to read RTP packet for requested stream room", "room", room.Name, "err", err)
-					}
-					break
-				}
-
-				err = localTrack.WriteRTP(rtpPacket)
-				if err != nil && !errors.Is(err, io.ErrClosedPipe) {
-					slog.Error("Failed to write RTP to local track for requested stream room", "room", room.Name, "err", err)
-					break
-				}
-			}
-		}()
-	})
-
-	pc.OnDataChannel(func(dc *webrtc.DataChannel) {
-		ndc := connections.NewNestriDataChannel(dc)
-		ndc.RegisterOnOpen(func() {
-			slog.Debug("Relay DataChannel opened for requested stream", "room", room.Name)
-		})
-		ndc.RegisterOnClose(func() {
-			slog.Debug("Relay DataChannel closed for requested stream", "room", room.Name)
-		})
-
-		// Set the DataChannel in the requestedConns map
-		if conn, ok := sp.requestedConns.Get(room.Name); ok {
-			conn.ndc = ndc
 			} else {
-			sp.requestedConns.Set(room.Name, &StreamConnection{
+				slog.Warn("Could not GetSdp from answer")
-				pc:  pc,
-				ndc: ndc,
-			})
-		}
-
-		// We do not handle any messages from upstream here
-	})
-
-	pc.OnICECandidate(func(candidate *webrtc.ICECandidate) {
-		if candidate == nil {
-			return
-		}
-
-		if err = safeBRW.SendJSON(connections.NewMessageICE(
-			"ice-candidate",
-			candidate.ToJSON(),
-		)); err != nil {
-			slog.Error("Failed to send ICE candidate message for requested stream", "room", room.Name, "err", err)
-			return
-		}
-	})
-
-	// Handle incoming messages (offer and candidates)
-	go func() {
-		iceHolder := make([]webrtc.ICECandidateInit, 0)
-
-		for {
-			data, err := safeBRW.Receive()
-			if err != nil {
-				if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
-					slog.Debug("Connection for requested stream closed by peer", "room", room.Name)
-					return
-				}
-
-				slog.Error("Failed to receive data for requested stream", "room", room.Name, "err", err)
-				_ = stream.Reset()
-
-				return
-			}
-
-			var baseMsg connections.MessageBase
-			if err = json.Unmarshal(data, &baseMsg); err != nil {
-				slog.Error("Failed to unmarshal base message for requested stream", "room", room.Name, "err", err)
-				return
-			}
-
-			switch baseMsg.Type {
-			case "ice-candidate":
-				var iceMsg connections.MessageICE
-				if err = json.Unmarshal(data, &iceMsg); err != nil {
-					slog.Error("Failed to unmarshal ICE candidate for requested stream", "room", room.Name, "err", err)
-					continue
-				}
-				if conn, ok := sp.requestedConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
-					if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
-						slog.Error("Failed to add ICE candidate for requested stream", "room", room.Name, "err", err)
-					}
-					// Add held candidates
-					for _, heldCandidate := range iceHolder {
-						if err = conn.pc.AddICECandidate(heldCandidate); err != nil {
-							slog.Error("Failed to add held ICE candidate for requested stream", "room", room.Name, "err", err)
 			}
 		}
-					// Clear the held candidates
-					iceHolder = make([]webrtc.ICECandidateInit, 0)
-				} else {
-					// Hold the candidate until remote description is set
-					iceHolder = append(iceHolder, iceMsg.Candidate)
 	}
-			case "offer":
-				var offerMsg connections.MessageSDP
-				if err = json.Unmarshal(data, &offerMsg); err != nil {
-					slog.Error("Failed to unmarshal offer for requested stream", "room", room.Name, "err", err)
-					continue
-				}
-				if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
-					slog.Error("Failed to set remote description for requested stream", "room", room.Name, "err", err)
-					continue
-				}
-				answer, err := pc.CreateAnswer(nil)
-				if err != nil {
-					slog.Error("Failed to create answer for requested stream", "room", room.Name, "err", err)
-					if err = stream.Reset(); err != nil {
-						slog.Error("Failed to reset stream for requested stream", "err", err)
-					}
-					return
-				}
-				if err = pc.SetLocalDescription(answer); err != nil {
-					slog.Error("Failed to set local description for requested stream", "room", room.Name, "err", err)
-					if err = stream.Reset(); err != nil {
-						slog.Error("Failed to reset stream for requested stream", "err", err)
-					}
-					return
-				}
-				if err = safeBRW.SendJSON(connections.NewMessageSDP(
-					"answer",
-					answer,
-				)); err != nil {
-					slog.Error("Failed to send answer for requested stream", "room", room.Name, "err", err)
-					continue
-				}
-
-				// Store the connection
-				sp.requestedConns.Set(room.Name, &StreamConnection{
-					pc:  pc,
-					ndc: nil,
-				})
-
-				slog.Debug("Sent answer for requested stream", "room", room.Name)
-			default:
-				slog.Warn("Unknown signaling message type", "room", room.Name, "type", baseMsg.Type)
-			}
-		}
-	}()
-
-	return nil
 }
 
 // handleStreamPush manages a stream push from a node (nestri-server)
@@ -476,42 +471,39 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 	var room *shared.Room
 	iceHolder := make([]webrtc.ICECandidateInit, 0)
 	for {
-		data, err := safeBRW.Receive()
+		var msgWrapper gen.ProtoMessage
+		err := safeBRW.ReceiveProto(&msgWrapper)
 		if err != nil {
 			if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
 				slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err)
+				if room != nil {
+					room.Close()
+					sp.incomingConns.Set(room.Name, nil)
+				}
 				return
 			}
 
 			slog.Error("Failed to receive data for stream push", "err", err)
 			_ = stream.Reset()
-
+			if room != nil {
+				room.Close()
+				sp.incomingConns.Set(room.Name, nil)
+			}
 			return
 		}
 
-		var baseMsg connections.MessageBase
+		if msgWrapper.MessageBase == nil {
-		if err = json.Unmarshal(data, &baseMsg); err != nil {
+			slog.Error("No MessageBase in stream push")
-			slog.Error("Failed to unmarshal base message from base message", "err", err)
 			continue
 		}
 
-		switch baseMsg.Type {
+		switch msgWrapper.MessageBase.PayloadType {
 		case "push-stream-room":
-			var rawMsg connections.MessageRaw
+			pushMsg := msgWrapper.GetServerPushStream()
-			if err = json.Unmarshal(data, &rawMsg); err != nil {
+			if pushMsg != nil {
-				slog.Error("Failed to unmarshal room name from data", "err", err)
+				slog.Info("Received stream push request for room", "room", pushMsg.RoomName)
-				continue
-			}
 
-			var roomName string
+				room = sp.relay.GetRoomByName(pushMsg.RoomName)
-			if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
-				slog.Error("Failed to unmarshal room name from raw message", "err", err)
-				continue
-			}
-
-			slog.Info("Received stream push request for room", "room", roomName)
-
-			room = sp.relay.GetRoomByName(roomName)
 				if room != nil {
 					if room.OwnerID != sp.relay.ID {
 						slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID)
@@ -523,34 +515,43 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					}
 				} else {
 					// Create a new room if it doesn't exist
-				room = sp.relay.CreateRoom(roomName)
+					room = sp.relay.CreateRoom(pushMsg.RoomName)
 				}
 
 				// Respond with an OK with the room name
-			roomData, err := json.Marshal(room.Name)
+				resMsg, err := common.CreateMessage(
+					&gen.ProtoServerPushStream{
+						RoomName: pushMsg.RoomName,
+					},
+					"push-stream-ok", nil,
+				)
 				if err != nil {
-				slog.Error("Failed to marshal room name for push stream response", "err", err)
+					slog.Error("Failed to create proto message", "err", err)
 					continue
 				}
-			if err = safeBRW.SendJSON(connections.NewMessageRaw(
+				if err = safeBRW.SendProto(resMsg); err != nil {
-				"push-stream-ok",
-				roomData,
-			)); err != nil {
 					slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err)
 					continue
 				}
+			} else {
+				slog.Error("Failed to GetServerPushStream in push-stream-room")
+			}
 		case "ice-candidate":
-			var iceMsg connections.MessageICE
+			iceMsg := msgWrapper.GetIce()
-			if err = json.Unmarshal(data, &iceMsg); err != nil {
+			if iceMsg != nil {
-				slog.Error("Failed to unmarshal ICE candidate from data", "err", err)
+				smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
-				continue
+				cand := webrtc.ICECandidateInit{
+					Candidate:        iceMsg.Candidate.Candidate,
+					SDPMid:           iceMsg.Candidate.SdpMid,
+					SDPMLineIndex:    &smollified,
+					UsernameFragment: iceMsg.Candidate.UsernameFragment,
 				}
 				if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
-				if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
+					if err = conn.pc.AddICECandidate(cand); err != nil {
 						slog.Error("Failed to add ICE candidate for pushed stream", "err", err)
 					}
 					for _, heldIce := range iceHolder {
-					if err := conn.pc.AddICECandidate(heldIce); err != nil {
+						if err = conn.pc.AddICECandidate(heldIce); err != nil {
 							slog.Error("Failed to add held ICE candidate for pushed stream", "err", err)
 						}
 					}
@@ -558,7 +559,10 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					iceHolder = make([]webrtc.ICECandidateInit, 0)
 				} else {
 					// Hold the candidate until remote description is set
-				iceHolder = append(iceHolder, iceMsg.Candidate)
+					iceHolder = append(iceHolder, cand)
+				}
+			} else {
+				slog.Error("Failed to GetIce in pushed stream ice-candidate")
 			}
 		case "offer":
 			// Make sure we have room set to push to (set by "push-stream-room")
@@ -567,12 +571,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				continue
 			}
 
-			var offerMsg connections.MessageSDP
+			offerMsg := msgWrapper.GetSdp()
-			if err = json.Unmarshal(data, &offerMsg); err != nil {
+			if offerMsg != nil {
-				slog.Error("Failed to unmarshal offer from data", "err", err)
+				offSdp := webrtc.SessionDescription{
-				continue
+					SDP:  offerMsg.Sdp.Sdp,
+					Type: webrtc.NewSDPType(offerMsg.Sdp.Type),
 				}
-
 				// Create PeerConnection for the incoming stream
 				pc, err := common.CreatePeerConnection(func() {
 					slog.Info("PeerConnection closed for pushed stream", "room", room.Name)
@@ -586,6 +590,9 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					continue
 				}
 
+				// Assign room peer connection
+				room.PeerConnection = pc
+
 				pc.OnDataChannel(func(dc *webrtc.DataChannel) {
 					// TODO: Is this the best way to handle DataChannel? Should we just use the map directly?
 					room.DataChannel = connections.NewNestriDataChannel(dc)
@@ -595,20 +602,19 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					room.DataChannel.RegisterOnClose(func() {
 						slog.Debug("DataChannel closed for pushed stream", "room", room.Name)
 					})
-				room.DataChannel.RegisterMessageCallback("input", func(data []byte) {
+					// Handle controller feedback reverse-flow (like rumble events coming from game to client)
-					if room.DataChannel != nil {
+					room.DataChannel.RegisterMessageCallback("controllerInput", func(data []byte) {
-						// Pass to servedConns DataChannels for this specific room
+						// Forward controller input to all viewers
 						if roomMap, ok := sp.servedConns.Get(room.Name); ok {
 							roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool {
 								if conn.ndc != nil {
 									if err = conn.ndc.SendBinary(data); err != nil {
-										slog.Error("Failed to forward input message from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
+										slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
 									}
 								}
-								return true // Continue iteration
+								return true
 							})
 						}
-					}
 					})
 
 					// Set the DataChannel in the incomingConns map
@@ -627,27 +633,29 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 						return
 					}
 
-				if err = safeBRW.SendJSON(connections.NewMessageICE(
+					candInit := candidate.ToJSON()
-					"ice-candidate",
+					biggified := uint32(*candInit.SDPMLineIndex)
-					candidate.ToJSON(),
+					iceMsg, err := common.CreateMessage(
-				)); err != nil {
+						&gen.ProtoICE{
+							Candidate: &gen.RTCIceCandidateInit{
+								Candidate:     candInit.Candidate,
+								SdpMLineIndex: &biggified,
+								SdpMid:        candInit.SDPMid,
+							},
+						},
+						"ice-candidate", nil,
+					)
+					if err != nil {
+						slog.Error("Failed to create proto message", "err", err)
+						return
+					}
+					if err = safeBRW.SendProto(iceMsg); err != nil {
 						slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err)
 						return
 					}
 				})
 
 				pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
-				localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String()))
-				if err != nil {
-					slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err)
-					return
-				}
-
-				slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String())
-
-				// Set track for Room
-				room.SetTrack(remoteTrack.Kind(), localTrack)
-
 					// Prepare PlayoutDelayExtension so we don't need to recreate it for each packet
 					playoutExt := &rtp.PlayoutDelayExtension{
 						MinDelay: 0,
@@ -659,6 +667,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 						return
 					}
 
+					if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio {
+						room.AudioCodec = remoteTrack.Codec().RTPCodecCapability
+					} else if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
+						room.VideoCodec = remoteTrack.Codec().RTPCodecCapability
+					}
+
 					for {
 						rtpPacket, _, err := remoteTrack.ReadRTP()
 						if err != nil {
@@ -670,27 +684,65 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 
 						// Use PlayoutDelayExtension for low latency, if set for this track kind
 						if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok {
-						if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil {
+							if err = rtpPacket.SetExtension(extID, playoutPayload); err != nil {
 								slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err)
 								continue
 							}
 						}
 
-					err = localTrack.WriteRTP(rtpPacket)
+						// Calculate differences
-					if err != nil && !errors.Is(err, io.ErrClosedPipe) {
+						var timeDiff int64
-						slog.Error("Failed to write RTP to local track for room", "room", room.Name, "err", err)
+						var sequenceDiff int
-						break
+
+						if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
+							timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastVideoTimestamp)
+							if !room.VideoTimestampSet {
+								timeDiff = 0
+								room.VideoTimestampSet = true
+							} else if timeDiff < -(math.MaxUint32 / 10) {
+								timeDiff += math.MaxUint32 + 1
 							}
+
+							sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastVideoSequenceNumber)
+							if !room.VideoSequenceSet {
+								sequenceDiff = 0
+								room.VideoSequenceSet = true
+							} else if sequenceDiff < -(math.MaxUint16 / 10) {
+								sequenceDiff += math.MaxUint16 + 1
+							}
+
+							room.LastVideoTimestamp = rtpPacket.Timestamp
+							room.LastVideoSequenceNumber = rtpPacket.SequenceNumber
+						} else { // Audio
+							timeDiff = int64(rtpPacket.Timestamp) - int64(room.LastAudioTimestamp)
+							if !room.AudioTimestampSet {
+								timeDiff = 0
+								room.AudioTimestampSet = true
+							} else if timeDiff < -(math.MaxUint32 / 10) {
+								timeDiff += math.MaxUint32 + 1
+							}
+
+							sequenceDiff = int(rtpPacket.SequenceNumber) - int(room.LastAudioSequenceNumber)
+							if !room.AudioSequenceSet {
+								sequenceDiff = 0
+								room.AudioSequenceSet = true
+							} else if sequenceDiff < -(math.MaxUint16 / 10) {
+								sequenceDiff += math.MaxUint16 + 1
+							}
+
+							room.LastAudioTimestamp = rtpPacket.Timestamp
+							room.LastAudioSequenceNumber = rtpPacket.SequenceNumber
+						}
+
+						// Broadcast with differences
+						room.BroadcastPacketRetimed(remoteTrack.Kind(), rtpPacket, timeDiff, sequenceDiff)
 					}
 
 					slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())
-
-				// Cleanup the track from the room
-				room.SetTrack(remoteTrack.Kind(), nil)
 				})
 
 				// Set the remote description
-			if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
+				if err = pc.SetRemoteDescription(offSdp); err != nil {
 					slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err)
 					continue
 				}
@@ -706,10 +758,20 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err)
 					continue
 				}
-			if err = safeBRW.SendJSON(connections.NewMessageSDP(
+				answerMsg, err := common.CreateMessage(
-				"answer",
+					&gen.ProtoSDP{
-				answer,
+						Sdp: &gen.RTCSessionDescriptionInit{
-			)); err != nil {
+							Sdp:  answer.SDP,
+							Type: answer.Type.String(),
+						},
+					},
+					"answer", nil,
+				)
+				if err != nil {
+					slog.Error("Failed to create proto message", "err", err)
+					continue
+				}
+				if err = safeBRW.SendProto(answerMsg); err != nil {
 					slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err)
 				}
 
@@ -721,16 +783,17 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				slog.Debug("Sent answer for pushed stream", "room", room.Name)
 			}
 		}
+	}
 }
 
 // --- Public Usable Methods ---
 
 // RequestStream sends a request to get room stream from another relay
 func (sp *StreamProtocol) RequestStream(ctx context.Context, room *shared.Room, peerID peer.ID) error {
-	stream, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
+	_, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
 	if err != nil {
 		return fmt.Errorf("failed to create stream: %w", err)
 	}
 
-	return sp.requestStream(stream, room)
+	return nil /* TODO: This? */
 }

packages/relay/internal/core/room.go

@@ -45,7 +45,7 @@ func (r *Relay) DeleteRoomIfEmpty(room *shared.Room) {
 	if room == nil {
 		return
 	}
-	if room.Participants.Len() == 0 && r.LocalRooms.Has(room.ID) {
+	if len(room.Participants) <= 0 && r.LocalRooms.Has(room.ID) {
 		slog.Debug("Deleting empty room without participants", "room", room.Name)
 		r.LocalRooms.Delete(room.ID)
 		err := room.PeerConnection.Close()

packages/relay/internal/core/state.go

@@ -5,9 +5,14 @@ import (
 	"encoding/json"
 	"errors"
 	"log/slog"
+	"relay/internal/common"
 	"relay/internal/shared"
 	"time"
 
+	gen "relay/internal/proto"
+
+	"google.golang.org/protobuf/proto"
+
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -129,12 +134,51 @@ func (r *Relay) onPeerConnected(peerID peer.ID) {
 
 // onPeerDisconnected marks a peer as disconnected in our status view and removes latency info
 func (r *Relay) onPeerDisconnected(peerID peer.ID) {
+	// Check if this was a client session disconnect
+	if session, ok := r.ClientSessions.Get(peerID); ok {
+		slog.Info("Client session disconnected",
+			"peer", peerID,
+			"session", session.SessionID,
+			"room", session.RoomName,
+			"controller_slots", session.ControllerSlots)
+
+		// Send cleanup message to nestri-server if client had active controllers
+		if len(session.ControllerSlots) > 0 {
+			room := r.GetRoomByName(session.RoomName)
+			if room != nil && room.DataChannel != nil {
+				// Create disconnect notification
+				disconnectMsg, err := common.CreateMessage(&gen.ProtoClientDisconnected{
+					SessionId:       session.SessionID,
+					ControllerSlots: session.ControllerSlots,
+				}, "client-disconnected", nil)
+				if err != nil {
+					slog.Error("Failed to create client disconnect message", "err", err)
+				}
+
+				disMarshal, err := proto.Marshal(disconnectMsg)
+				if err != nil {
+					slog.Error("Failed to marshal client disconnect message", "err", err)
+				} else {
+					if err = room.DataChannel.SendBinary(disMarshal); err != nil {
+						slog.Error("Failed to send client disconnect notification", "err", err)
+					} else {
+						slog.Info("Sent controller cleanup notification to nestri-server",
+							"session", session.SessionID,
+							"slots", session.ControllerSlots)
+					}
+				}
+			}
+		}
+
+		r.ClientSessions.Delete(peerID)
+		return
+	}
+
+	// Relay peer disconnect handling
 	slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID)
-	// Remove peer from local mesh peers
 	if r.Peers.Has(peerID) {
 		r.Peers.Delete(peerID)
 	}
-	// Remove any rooms associated with this peer
 	if r.Rooms.Has(peerID.String()) {
 		r.Rooms.Delete(peerID.String())
 	}
@@ -151,18 +195,18 @@ func (r *Relay) updateMeshRoomStates(peerID peer.ID, states []shared.RoomInfo) {
 		}
 
 		// If previously did not exist, but does now, request a connection if participants exist for our room
-		existed := r.Rooms.Has(state.ID.String())
+		/*existed := r.Rooms.Has(state.ID.String())
 		if !existed {
 			// Request connection to this peer if we have participants in our local room
 			if room, ok := r.LocalRooms.Get(state.ID); ok {
-				if room.Participants.Len() > 0 {
+				if len(room.Participants) > 0 {
 					slog.Debug("Got new remote room state, we locally have participants for, requesting stream", "room_name", room.Name, "peer", peerID)
 					if err := r.StreamProtocol.RequestStream(context.Background(), room, peerID); err != nil {
 						slog.Error("Failed to request stream for new remote room state", "room_name", room.Name, "peer", peerID, "err", err)
 					}
 				}
 			}
-		}
+		}*/
 
 		r.Rooms.Set(state.ID.String(), state)
 	}

packages/relay/internal/proto/messages.pb.go

@@ -73,28 +73,50 @@ func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker {
 	return nil
 }
 
-type ProtoMessageInput struct {
+type ProtoMessage struct {
 	state       protoimpl.MessageState `protogen:"open.v1"`
 	MessageBase *ProtoMessageBase      `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"`
-	Data          *ProtoInput            `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+	// Types that are valid to be assigned to Payload:
+	//
+	//	*ProtoMessage_MouseMove
+	//	*ProtoMessage_MouseMoveAbs
+	//	*ProtoMessage_MouseWheel
+	//	*ProtoMessage_MouseKeyDown
+	//	*ProtoMessage_MouseKeyUp
+	//	*ProtoMessage_KeyDown
+	//	*ProtoMessage_KeyUp
+	//	*ProtoMessage_ControllerAttach
+	//	*ProtoMessage_ControllerDetach
+	//	*ProtoMessage_ControllerButton
+	//	*ProtoMessage_ControllerTrigger
+	//	*ProtoMessage_ControllerStick
+	//	*ProtoMessage_ControllerAxis
+	//	*ProtoMessage_ControllerRumble
+	//	*ProtoMessage_Ice
+	//	*ProtoMessage_Sdp
+	//	*ProtoMessage_Raw
+	//	*ProtoMessage_ClientRequestRoomStream
+	//	*ProtoMessage_ClientDisconnected
+	//	*ProtoMessage_ServerPushStream
+	Payload       isProtoMessage_Payload `protobuf_oneof:"payload"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
-func (x *ProtoMessageInput) Reset() {
+func (x *ProtoMessage) Reset() {
-	*x = ProtoMessageInput{}
+	*x = ProtoMessage{}
 	mi := &file_messages_proto_msgTypes[1]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
 
-func (x *ProtoMessageInput) String() string {
+func (x *ProtoMessage) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*ProtoMessageInput) ProtoMessage() {}
+func (*ProtoMessage) ProtoMessage() {}
 
-func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
+func (x *ProtoMessage) ProtoReflect() protoreflect.Message {
 	mi := &file_messages_proto_msgTypes[1]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -106,25 +128,331 @@ func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead.
+// Deprecated: Use ProtoMessage.ProtoReflect.Descriptor instead.
-func (*ProtoMessageInput) Descriptor() ([]byte, []int) {
+func (*ProtoMessage) Descriptor() ([]byte, []int) {
 	return file_messages_proto_rawDescGZIP(), []int{1}
 }
 
-func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase {
+func (x *ProtoMessage) GetMessageBase() *ProtoMessageBase {
 	if x != nil {
 		return x.MessageBase
 	}
 	return nil
 }
 
-func (x *ProtoMessageInput) GetData() *ProtoInput {
+func (x *ProtoMessage) GetPayload() isProtoMessage_Payload {
 	if x != nil {
-		return x.Data
+		return x.Payload
 	}
 	return nil
 }
 
+func (x *ProtoMessage) GetMouseMove() *ProtoMouseMove {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_MouseMove); ok {
+			return x.MouseMove
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetMouseMoveAbs() *ProtoMouseMoveAbs {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_MouseMoveAbs); ok {
+			return x.MouseMoveAbs
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetMouseWheel() *ProtoMouseWheel {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_MouseWheel); ok {
+			return x.MouseWheel
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetMouseKeyDown() *ProtoMouseKeyDown {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_MouseKeyDown); ok {
+			return x.MouseKeyDown
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetMouseKeyUp() *ProtoMouseKeyUp {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_MouseKeyUp); ok {
+			return x.MouseKeyUp
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetKeyDown() *ProtoKeyDown {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_KeyDown); ok {
+			return x.KeyDown
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetKeyUp() *ProtoKeyUp {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_KeyUp); ok {
+			return x.KeyUp
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerAttach() *ProtoControllerAttach {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerAttach); ok {
+			return x.ControllerAttach
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerDetach); ok {
+			return x.ControllerDetach
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerButton() *ProtoControllerButton {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerButton); ok {
+			return x.ControllerButton
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerTrigger() *ProtoControllerTrigger {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerTrigger); ok {
+			return x.ControllerTrigger
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerStick() *ProtoControllerStick {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerStick); ok {
+			return x.ControllerStick
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerAxis() *ProtoControllerAxis {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerAxis); ok {
+			return x.ControllerAxis
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok {
+			return x.ControllerRumble
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetIce() *ProtoICE {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_Ice); ok {
+			return x.Ice
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetSdp() *ProtoSDP {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_Sdp); ok {
+			return x.Sdp
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetRaw() *ProtoRaw {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_Raw); ok {
+			return x.Raw
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetClientRequestRoomStream() *ProtoClientRequestRoomStream {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ClientRequestRoomStream); ok {
+			return x.ClientRequestRoomStream
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetClientDisconnected() *ProtoClientDisconnected {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ClientDisconnected); ok {
+			return x.ClientDisconnected
+		}
+	}
+	return nil
+}
+
+func (x *ProtoMessage) GetServerPushStream() *ProtoServerPushStream {
+	if x != nil {
+		if x, ok := x.Payload.(*ProtoMessage_ServerPushStream); ok {
+			return x.ServerPushStream
+		}
+	}
+	return nil
+}
+
+type isProtoMessage_Payload interface {
+	isProtoMessage_Payload()
+}
+
+type ProtoMessage_MouseMove struct {
+	// Input types
+	MouseMove *ProtoMouseMove `protobuf:"bytes,2,opt,name=mouse_move,json=mouseMove,proto3,oneof"`
+}
+
+type ProtoMessage_MouseMoveAbs struct {
+	MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,3,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"`
+}
+
+type ProtoMessage_MouseWheel struct {
+	MouseWheel *ProtoMouseWheel `protobuf:"bytes,4,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"`
+}
+
+type ProtoMessage_MouseKeyDown struct {
+	MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,5,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"`
+}
+
+type ProtoMessage_MouseKeyUp struct {
+	MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,6,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"`
+}
+
+type ProtoMessage_KeyDown struct {
+	KeyDown *ProtoKeyDown `protobuf:"bytes,7,opt,name=key_down,json=keyDown,proto3,oneof"`
+}
+
+type ProtoMessage_KeyUp struct {
+	KeyUp *ProtoKeyUp `protobuf:"bytes,8,opt,name=key_up,json=keyUp,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerAttach struct {
+	ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerDetach struct {
+	ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerButton struct {
+	ControllerButton *ProtoControllerButton `protobuf:"bytes,11,opt,name=controller_button,json=controllerButton,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerTrigger struct {
+	ControllerTrigger *ProtoControllerTrigger `protobuf:"bytes,12,opt,name=controller_trigger,json=controllerTrigger,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerStick struct {
+	ControllerStick *ProtoControllerStick `protobuf:"bytes,13,opt,name=controller_stick,json=controllerStick,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerAxis struct {
+	ControllerAxis *ProtoControllerAxis `protobuf:"bytes,14,opt,name=controller_axis,json=controllerAxis,proto3,oneof"`
+}
+
+type ProtoMessage_ControllerRumble struct {
+	ControllerRumble *ProtoControllerRumble `protobuf:"bytes,15,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"`
+}
+
+type ProtoMessage_Ice struct {
+	// Signaling types
+	Ice *ProtoICE `protobuf:"bytes,20,opt,name=ice,proto3,oneof"`
+}
+
+type ProtoMessage_Sdp struct {
+	Sdp *ProtoSDP `protobuf:"bytes,21,opt,name=sdp,proto3,oneof"`
+}
+
+type ProtoMessage_Raw struct {
+	Raw *ProtoRaw `protobuf:"bytes,22,opt,name=raw,proto3,oneof"`
+}
+
+type ProtoMessage_ClientRequestRoomStream struct {
+	ClientRequestRoomStream *ProtoClientRequestRoomStream `protobuf:"bytes,23,opt,name=client_request_room_stream,json=clientRequestRoomStream,proto3,oneof"`
+}
+
+type ProtoMessage_ClientDisconnected struct {
+	ClientDisconnected *ProtoClientDisconnected `protobuf:"bytes,24,opt,name=client_disconnected,json=clientDisconnected,proto3,oneof"`
+}
+
+type ProtoMessage_ServerPushStream struct {
+	ServerPushStream *ProtoServerPushStream `protobuf:"bytes,25,opt,name=server_push_stream,json=serverPushStream,proto3,oneof"`
+}
+
+func (*ProtoMessage_MouseMove) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_MouseMoveAbs) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_MouseWheel) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_MouseKeyDown) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_MouseKeyUp) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_KeyDown) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_KeyUp) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerButton) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerTrigger) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerStick) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerAxis) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_Ice) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_Sdp) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_Raw) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ClientRequestRoomStream) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ClientDisconnected) isProtoMessage_Payload() {}
+
+func (*ProtoMessage_ServerPushStream) isProtoMessage_Payload() {}
+
 var File_messages_proto protoreflect.FileDescriptor
 
 const file_messages_proto_rawDesc = "" +
@@ -132,10 +460,35 @@ const file_messages_proto_rawDesc = "" +
 	"\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" +
 	"\x10ProtoMessageBase\x12!\n" +
 	"\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" +
-	"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"v\n" +
+	"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\xef\n" +
-	"\x11ProtoMessageInput\x12:\n" +
+	"\n" +
-	"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x12%\n" +
+	"\fProtoMessage\x12:\n" +
-	"\x04data\x18\x02 \x01(\v2\x11.proto.ProtoInputR\x04dataB\x16Z\x14relay/internal/protob\x06proto3"
+	"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" +
+	"\n" +
+	"mouse_move\x18\x02 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" +
+	"\x0emouse_move_abs\x18\x03 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" +
+	"\vmouse_wheel\x18\x04 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" +
+	"mouseWheel\x12@\n" +
+	"\x0emouse_key_down\x18\x05 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" +
+	"\fmouse_key_up\x18\x06 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" +
+	"mouseKeyUp\x120\n" +
+	"\bkey_down\x18\a \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" +
+	"\x06key_up\x18\b \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" +
+	"\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" +
+	"\x11controller_detach\x18\n" +
+	" \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" +
+	"\x11controller_button\x18\v \x01(\v2\x1c.proto.ProtoControllerButtonH\x00R\x10controllerButton\x12N\n" +
+	"\x12controller_trigger\x18\f \x01(\v2\x1d.proto.ProtoControllerTriggerH\x00R\x11controllerTrigger\x12H\n" +
+	"\x10controller_stick\x18\r \x01(\v2\x1b.proto.ProtoControllerStickH\x00R\x0fcontrollerStick\x12E\n" +
+	"\x0fcontroller_axis\x18\x0e \x01(\v2\x1a.proto.ProtoControllerAxisH\x00R\x0econtrollerAxis\x12K\n" +
+	"\x11controller_rumble\x18\x0f \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12#\n" +
+	"\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" +
+	"\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" +
+	"\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" +
+	"\x1aclient_request_room_stream\x18\x17 \x01(\v2#.proto.ProtoClientRequestRoomStreamH\x00R\x17clientRequestRoomStream\x12Q\n" +
+	"\x13client_disconnected\x18\x18 \x01(\v2\x1e.proto.ProtoClientDisconnectedH\x00R\x12clientDisconnected\x12L\n" +
+	"\x12server_push_stream\x18\x19 \x01(\v2\x1c.proto.ProtoServerPushStreamH\x00R\x10serverPushStreamB\t\n" +
+	"\apayloadB\x16Z\x14relay/internal/protob\x06proto3"
 
 var (
 	file_messages_proto_rawDescOnce sync.Once
@@ -152,19 +505,57 @@ func file_messages_proto_rawDescGZIP() []byte {
 var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
 var file_messages_proto_goTypes = []any{
 	(*ProtoMessageBase)(nil),             // 0: proto.ProtoMessageBase
-	(*ProtoMessageInput)(nil),   // 1: proto.ProtoMessageInput
+	(*ProtoMessage)(nil),                 // 1: proto.ProtoMessage
 	(*ProtoLatencyTracker)(nil),          // 2: proto.ProtoLatencyTracker
-	(*ProtoInput)(nil),          // 3: proto.ProtoInput
+	(*ProtoMouseMove)(nil),               // 3: proto.ProtoMouseMove
+	(*ProtoMouseMoveAbs)(nil),            // 4: proto.ProtoMouseMoveAbs
+	(*ProtoMouseWheel)(nil),              // 5: proto.ProtoMouseWheel
+	(*ProtoMouseKeyDown)(nil),            // 6: proto.ProtoMouseKeyDown
+	(*ProtoMouseKeyUp)(nil),              // 7: proto.ProtoMouseKeyUp
+	(*ProtoKeyDown)(nil),                 // 8: proto.ProtoKeyDown
+	(*ProtoKeyUp)(nil),                   // 9: proto.ProtoKeyUp
+	(*ProtoControllerAttach)(nil),        // 10: proto.ProtoControllerAttach
+	(*ProtoControllerDetach)(nil),        // 11: proto.ProtoControllerDetach
+	(*ProtoControllerButton)(nil),        // 12: proto.ProtoControllerButton
+	(*ProtoControllerTrigger)(nil),       // 13: proto.ProtoControllerTrigger
+	(*ProtoControllerStick)(nil),         // 14: proto.ProtoControllerStick
+	(*ProtoControllerAxis)(nil),          // 15: proto.ProtoControllerAxis
+	(*ProtoControllerRumble)(nil),        // 16: proto.ProtoControllerRumble
+	(*ProtoICE)(nil),                     // 17: proto.ProtoICE
+	(*ProtoSDP)(nil),                     // 18: proto.ProtoSDP
+	(*ProtoRaw)(nil),                     // 19: proto.ProtoRaw
+	(*ProtoClientRequestRoomStream)(nil), // 20: proto.ProtoClientRequestRoomStream
+	(*ProtoClientDisconnected)(nil),      // 21: proto.ProtoClientDisconnected
+	(*ProtoServerPushStream)(nil),        // 22: proto.ProtoServerPushStream
 }
 var file_messages_proto_depIdxs = []int32{
 	2,  // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker
-	0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase
+	0,  // 1: proto.ProtoMessage.message_base:type_name -> proto.ProtoMessageBase
-	3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput
+	3,  // 2: proto.ProtoMessage.mouse_move:type_name -> proto.ProtoMouseMove
-	3, // [3:3] is the sub-list for method output_type
+	4,  // 3: proto.ProtoMessage.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs
-	3, // [3:3] is the sub-list for method input_type
+	5,  // 4: proto.ProtoMessage.mouse_wheel:type_name -> proto.ProtoMouseWheel
-	3, // [3:3] is the sub-list for extension type_name
+	6,  // 5: proto.ProtoMessage.mouse_key_down:type_name -> proto.ProtoMouseKeyDown
-	3, // [3:3] is the sub-list for extension extendee
+	7,  // 6: proto.ProtoMessage.mouse_key_up:type_name -> proto.ProtoMouseKeyUp
-	0, // [0:3] is the sub-list for field type_name
+	8,  // 7: proto.ProtoMessage.key_down:type_name -> proto.ProtoKeyDown
+	9,  // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp
+	10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach
+	11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach
+	12, // 11: proto.ProtoMessage.controller_button:type_name -> proto.ProtoControllerButton
+	13, // 12: proto.ProtoMessage.controller_trigger:type_name -> proto.ProtoControllerTrigger
+	14, // 13: proto.ProtoMessage.controller_stick:type_name -> proto.ProtoControllerStick
+	15, // 14: proto.ProtoMessage.controller_axis:type_name -> proto.ProtoControllerAxis
+	16, // 15: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble
+	17, // 16: proto.ProtoMessage.ice:type_name -> proto.ProtoICE
+	18, // 17: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP
+	19, // 18: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw
+	20, // 19: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream
+	21, // 20: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected
+	22, // 21: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream
+	22, // [22:22] is the sub-list for method output_type
+	22, // [22:22] is the sub-list for method input_type
+	22, // [22:22] is the sub-list for extension type_name
+	22, // [22:22] is the sub-list for extension extendee
+	0,  // [0:22] is the sub-list for field type_name
 }
 
 func init() { file_messages_proto_init() }
@@ -174,6 +565,28 @@ func file_messages_proto_init() {
 	}
 	file_types_proto_init()
 	file_latency_tracker_proto_init()
+	file_messages_proto_msgTypes[1].OneofWrappers = []any{
+		(*ProtoMessage_MouseMove)(nil),
+		(*ProtoMessage_MouseMoveAbs)(nil),
+		(*ProtoMessage_MouseWheel)(nil),
+		(*ProtoMessage_MouseKeyDown)(nil),
+		(*ProtoMessage_MouseKeyUp)(nil),
+		(*ProtoMessage_KeyDown)(nil),
+		(*ProtoMessage_KeyUp)(nil),
+		(*ProtoMessage_ControllerAttach)(nil),
+		(*ProtoMessage_ControllerDetach)(nil),
+		(*ProtoMessage_ControllerButton)(nil),
+		(*ProtoMessage_ControllerTrigger)(nil),
+		(*ProtoMessage_ControllerStick)(nil),
+		(*ProtoMessage_ControllerAxis)(nil),
+		(*ProtoMessage_ControllerRumble)(nil),
+		(*ProtoMessage_Ice)(nil),
+		(*ProtoMessage_Sdp)(nil),
+		(*ProtoMessage_Raw)(nil),
+		(*ProtoMessage_ClientRequestRoomStream)(nil),
+		(*ProtoMessage_ClientDisconnected)(nil),
+		(*ProtoMessage_ServerPushStream)(nil),
+	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{

packages/relay/internal/shared/participant.go

@@ -1,44 +1,136 @@
 package shared
 
 import (
+	"errors"
 	"fmt"
+	"io"
+	"log/slog"
 	"relay/internal/common"
 	"relay/internal/connections"
+	"sync"
 
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/oklog/ulid/v2"
 	"github.com/pion/webrtc/v4"
 )
 
 type Participant struct {
 	ID             ulid.ULID
+	SessionID      string  // Track session for reconnection
+	PeerID         peer.ID // libp2p peer ID
 	PeerConnection *webrtc.PeerConnection
 	DataChannel    *connections.NestriDataChannel
+
+	// Per-viewer tracks and channels
+	VideoTrack *webrtc.TrackLocalStaticRTP
+	AudioTrack *webrtc.TrackLocalStaticRTP
+
+	// Per-viewer RTP state for retiming
+	VideoSequenceNumber uint16
+	VideoTimestamp      uint32
+	AudioSequenceNumber uint16
+	AudioTimestamp      uint32
+
+	packetQueue chan *participantPacket
+	closeOnce   sync.Once
 }
 
-func NewParticipant() (*Participant, error) {
+func NewParticipant(sessionID string, peerID peer.ID) (*Participant, error) {
 	id, err := common.NewULID()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ULID for Participant: %w", err)
 	}
-	return &Participant{
+	p := &Participant{
 		ID:                  id,
-	}, nil
+		SessionID:           sessionID,
+		PeerID:              peerID,
+		VideoSequenceNumber: 0,
+		VideoTimestamp:      0,
+		AudioSequenceNumber: 0,
+		AudioTimestamp:      0,
+		packetQueue:         make(chan *participantPacket, 1000),
+	}
+
+	go p.packetWriter()
+
+	return p, nil
 }
 
-func (p *Participant) addTrack(trackLocal *webrtc.TrackLocalStaticRTP) error {
+// SetTrack sets audio/video track for Participant
-	rtpSender, err := p.PeerConnection.AddTrack(trackLocal)
+func (p *Participant) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) {
+	switch trackType {
+	case webrtc.RTPCodecTypeAudio:
+		p.AudioTrack = track
+		_, err := p.PeerConnection.AddTrack(track)
 		if err != nil {
-		return err
+			slog.Error("Failed to add Participant audio track", err)
 		}
-
+	case webrtc.RTPCodecTypeVideo:
-	go func() {
+		p.VideoTrack = track
-		rtcpBuffer := make([]byte, 1400)
+		_, err := p.PeerConnection.AddTrack(track)
-		for {
+		if err != nil {
-			if _, _, rtcpErr := rtpSender.Read(rtcpBuffer); rtcpErr != nil {
+			slog.Error("Failed to add Participant video track", err)
-				break
 		}
+	default:
+		slog.Warn("Unknown track type", "participant", p.ID, "trackType", trackType)
+	}
+}
+
+// Close cleans up participant resources
+func (p *Participant) Close() {
+	if p.DataChannel != nil {
+		err := p.DataChannel.Close()
+		if err != nil {
+			slog.Error("Failed to close Participant DataChannel", err)
+		}
+		p.DataChannel = nil
+	}
+	if p.PeerConnection != nil {
+		err := p.PeerConnection.Close()
+		if err != nil {
+			slog.Error("Failed to close Participant PeerConnection", err)
+		}
+		p.PeerConnection = nil
+	}
+	if p.VideoTrack != nil {
+		p.VideoTrack = nil
+	}
+	if p.AudioTrack != nil {
+		p.AudioTrack = nil
+	}
+}
+
+func (p *Participant) packetWriter() {
+	for pkt := range p.packetQueue {
+		var track *webrtc.TrackLocalStaticRTP
+		var sequenceNumber uint16
+		var timestamp uint32
+
+		// No mutex needed - only this goroutine modifies these
+		if pkt.kind == webrtc.RTPCodecTypeAudio {
+			track = p.AudioTrack
+			p.AudioSequenceNumber = uint16(int(p.AudioSequenceNumber) + pkt.sequenceDiff)
+			p.AudioTimestamp = uint32(int64(p.AudioTimestamp) + pkt.timeDiff)
+			sequenceNumber = p.AudioSequenceNumber
+			timestamp = p.AudioTimestamp
+		} else {
+			track = p.VideoTrack
+			p.VideoSequenceNumber = uint16(int(p.VideoSequenceNumber) + pkt.sequenceDiff)
+			p.VideoTimestamp = uint32(int64(p.VideoTimestamp) + pkt.timeDiff)
+			sequenceNumber = p.VideoSequenceNumber
+			timestamp = p.VideoTimestamp
+		}
+
+		if track != nil {
+			pkt.packet.SequenceNumber = sequenceNumber
+			pkt.packet.Timestamp = timestamp
+
+			if err := track.WriteRTP(pkt.packet); err != nil && !errors.Is(err, io.ErrClosedPipe) {
+				slog.Error("WriteRTP failed", "participant", p.ID, "kind", pkt.kind, "err", err)
+			}
+		}
+
+		// Return packet struct to pool
+		participantPacketPool.Put(pkt)
 	}
-	}()
-
-	return nil
 }

packages/relay/internal/shared/room.go

@@ -2,14 +2,29 @@ package shared
 
 import (
 	"log/slog"
-	"relay/internal/common"
 	"relay/internal/connections"
+	"sync"
+	"sync/atomic"
 
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/oklog/ulid/v2"
+	"github.com/pion/rtp"
 	"github.com/pion/webrtc/v4"
 )
 
+var participantPacketPool = sync.Pool{
+	New: func() interface{} {
+		return &participantPacket{}
+	},
+}
+
+type participantPacket struct {
+	kind         webrtc.RTPCodecType
+	packet       *rtp.Packet
+	timeDiff     int64
+	sequenceDiff int
+}
+
 type RoomInfo struct {
 	ID      ulid.ULID `json:"id"`
 	Name    string    `json:"name"`
@@ -18,49 +33,139 @@ type RoomInfo struct {
 
 type Room struct {
 	RoomInfo
+	AudioCodec     webrtc.RTPCodecCapability
+	VideoCodec     webrtc.RTPCodecCapability
 	PeerConnection *webrtc.PeerConnection
-	AudioTrack     *webrtc.TrackLocalStaticRTP
-	VideoTrack     *webrtc.TrackLocalStaticRTP
 	DataChannel    *connections.NestriDataChannel
-	Participants   *common.SafeMap[ulid.ULID, *Participant]
+
+	// Atomic pointer to slice of participant channels
+	participantChannels atomic.Pointer[[]chan<- *participantPacket]
+	participantsMtx     sync.Mutex // Use only for add/remove
+
+	Participants map[ulid.ULID]*Participant // Keep general track of Participant(s)
+
+	// Track last seen values to calculate diffs
+	LastVideoTimestamp      uint32
+	LastVideoSequenceNumber uint16
+	LastAudioTimestamp      uint32
+	LastAudioSequenceNumber uint16
+
+	VideoTimestampSet bool
+	VideoSequenceSet  bool
+	AudioTimestampSet bool
+	AudioSequenceSet  bool
 }
 
 func NewRoom(name string, roomID ulid.ULID, ownerID peer.ID) *Room {
-	return &Room{
+	r := &Room{
 		RoomInfo: RoomInfo{
 			ID:      roomID,
 			Name:    name,
 			OwnerID: ownerID,
 		},
-		Participants: common.NewSafeMap[ulid.ULID, *Participant](),
+		PeerConnection: nil,
+		DataChannel:    nil,
+		Participants:   make(map[ulid.ULID]*Participant),
+	}
+
+	emptyChannels := make([]chan<- *participantPacket, 0)
+	r.participantChannels.Store(&emptyChannels)
+
+	return r
+}
+
+// Close closes up Room (stream ended)
+func (r *Room) Close() {
+	if r.DataChannel != nil {
+		err := r.DataChannel.Close()
+		if err != nil {
+			slog.Error("Failed to close Room DataChannel", err)
+		}
+		r.DataChannel = nil
+	}
+	if r.PeerConnection != nil {
+		err := r.PeerConnection.Close()
+		if err != nil {
+			slog.Error("Failed to close Room PeerConnection", err)
+		}
+		r.PeerConnection = nil
 	}
 }
 
 // AddParticipant adds a Participant to a Room
 func (r *Room) AddParticipant(participant *Participant) {
-	slog.Debug("Adding participant to room", "participant", participant.ID, "room", r.Name)
+	r.participantsMtx.Lock()
-	r.Participants.Set(participant.ID, participant)
+	defer r.participantsMtx.Unlock()
+
+	r.Participants[participant.ID] = participant
+
+	// Update channel slice atomically
+	current := r.participantChannels.Load()
+	newChannels := make([]chan<- *participantPacket, len(*current)+1)
+	copy(newChannels, *current)
+	newChannels[len(*current)] = participant.packetQueue
+
+	r.participantChannels.Store(&newChannels)
+
+	slog.Debug("Added participant", "participant", participant.ID, "room", r.Name)
 }
 
-// Removes a Participant from a Room by participant's ID
+// RemoveParticipantByID removes a Participant from a Room by participant's ID
-func (r *Room) removeParticipantByID(pID ulid.ULID) {
+func (r *Room) RemoveParticipantByID(pID ulid.ULID) {
-	if _, ok := r.Participants.Get(pID); ok {
+	r.participantsMtx.Lock()
-		r.Participants.Delete(pID)
+	defer r.participantsMtx.Unlock()
+
+	participant, ok := r.Participants[pID]
+	if !ok {
+		return
 	}
+
+	delete(r.Participants, pID)
+
+	// Update channel slice
+	current := r.participantChannels.Load()
+	newChannels := make([]chan<- *participantPacket, 0, len(*current)-1)
+	for _, ch := range *current {
+		if ch != participant.packetQueue {
+			newChannels = append(newChannels, ch)
+		}
+	}
+
+	r.participantChannels.Store(&newChannels)
+
+	slog.Debug("Removed participant", "participant", pID, "room", r.Name)
 }
 
-// IsOnline checks if the room is online (has both audio and video tracks)
+// IsOnline checks if the room is online
 func (r *Room) IsOnline() bool {
-	return r.AudioTrack != nil && r.VideoTrack != nil
+	return r.PeerConnection != nil
 }
 
-func (r *Room) SetTrack(trackType webrtc.RTPCodecType, track *webrtc.TrackLocalStaticRTP) {
+func (r *Room) BroadcastPacketRetimed(kind webrtc.RTPCodecType, pkt *rtp.Packet, timeDiff int64, sequenceDiff int) {
-	switch trackType {
+	// Lock-free load of channel slice
-	case webrtc.RTPCodecTypeAudio:
+	channels := r.participantChannels.Load()
-		r.AudioTrack = track
+
-	case webrtc.RTPCodecTypeVideo:
+	// no participants..
-		r.VideoTrack = track
+	if len(*channels) == 0 {
+		return
+	}
+
+	// Send to each participant channel (non-blocking)
+	for i, ch := range *channels {
+		// Get packet struct from pool
+		pp := participantPacketPool.Get().(*participantPacket)
+		pp.kind = kind
+		pp.packet = pkt.Clone()
+		pp.timeDiff = timeDiff
+		pp.sequenceDiff = sequenceDiff
+
+		select {
+		case ch <- pp:
+			// Sent successfully
 		default:
-		slog.Warn("Unknown track type", "room", r.Name, "trackType", trackType)
+			// Channel full, drop packet, log?
+			slog.Warn("Channel full, dropping packet", "channel_index", i)
+			participantPacketPool.Put(pp)
+		}
 	}
 }

packages/scripts/entrypoint.sh

@@ -15,13 +15,13 @@ NVIDIA_INSTALLER_DIR="/tmp"
 TIMEOUT_SECONDS=10
 ENTCMD_PREFIX=""
 
-# Ensures user directory ownership
+# Ensures user ownership across directories
-chown_user_directory() {
+handle_user_permissions() {
     if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}" 2>/dev/null; then
         echo "Error: Failed to change ownership of ${NESTRI_HOME} to ${NESTRI_USER}:${NESTRI_USER}" >&2
         return 1
     fi
-    # Also apply to .cache separately
+    # Also apply to .cache
     if [[ -d "${NESTRI_HOME}/.cache" ]]; then
         if ! $ENTCMD_PREFIX chown "${NESTRI_USER}:${NESTRI_USER}" "${NESTRI_HOME}/.cache" 2>/dev/null; then
             echo "Error: Failed to change ownership of ${NESTRI_HOME}/.cache to ${NESTRI_USER}:${NESTRI_USER}" >&2
@@ -324,9 +324,23 @@ main() {
         log "Skipping CAP_SYS_NICE for gamescope, capability not available"
     fi
 
-    # Handle user directory permissions
+    # Make sure /tmp/.X11-unix exists..
-    log "Ensuring user directory permissions..."
+    if [[ ! -d "/tmp/.X11-unix" ]]; then
-    chown_user_directory || exit 1
+        log "Creating /tmp/.X11-unix directory.."
+        $ENTCMD_PREFIX mkdir -p /tmp/.X11-unix || {
+            log "Error: Failed to create /tmp/.X11-unix directory"
+            exit 1
+        }
+        # Set required perms..
+        $ENTCMD_PREFIX chmod 1777 /tmp/.X11-unix || {
+            log "Error: Failed to chmod /tmp/.X11-unix to 1777"
+            exit 1
+        }
+    fi
+
+    # Handle user permissions
+    log "Ensuring user permissions..."
+    handle_user_permissions || exit 1
 
     # Setup namespaceless env if needed for container runtime
     if [[ "$container_runtime" != "podman" ]]; then
@@ -336,7 +350,7 @@ main() {
 
     # Make sure /run/udev/ directory exists with /run/udev/control, needed for virtual controller support
     if [[ ! -d "/run/udev" || ! -e "/run/udev/control" ]]; then
-        log "Creating /run/udev directory and control file..."
+        log "Creating /run/udev directory and control file.."
         $ENTCMD_PREFIX mkdir -p /run/udev || {
             log "Error: Failed to create /run/udev directory"
             exit 1

packages/scripts/entrypoint_nestri.sh

@@ -187,7 +187,7 @@ start_compositor() {
 
                     if [[ -n "${NESTRI_LAUNCH_CMD}" ]]; then
                         log "Starting application: $NESTRI_LAUNCH_CMD"
-                        WAYLAND_DISPLAY=wayland-0 /bin/bash -c "$NESTRI_LAUNCH_CMD" &
+                        WAYLAND_DISPLAY="$COMPOSITOR_SOCKET" /bin/bash -c "$NESTRI_LAUNCH_CMD" &
                         APP_PID=$!
                     fi
                 else

packages/server/Cargo.lock

@@ -181,7 +181,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "synstructure",
 ]
 
@@ -193,7 +193,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "synstructure",
 ]
 
@@ -205,7 +205,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -246,7 +246,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -257,7 +257,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -395,7 +395,7 @@ version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "cexpr",
  "clang-sys",
  "itertools 0.13.0",
@@ -406,7 +406,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -417,9 +417,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.9.4"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"
+checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
 
 [[package]]
 name = "blake2"
@@ -603,9 +603,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.49"
+version = "4.5.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4512b90fa68d3a9932cea5184017c5d200f5921df706d45e853537dea51508f"
+checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -613,9 +613,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.49"
+version = "4.5.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0025e98baa12e766c67ba13ff4695a887a1eba19569aad00a472546795bd6730"
+checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0"
 dependencies = [
  "anstream",
  "anstyle",
@@ -632,7 +632,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -839,7 +839,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -879,7 +879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -956,7 +956,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -1090,7 +1090,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -1262,7 +1262,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -1396,7 +1396,7 @@ version = "0.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1f2cbc4577536c849335878552f42086bfd25a8dcd6f54a18655cf818b20c8f"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "futures-channel",
  "futures-core",
  "futures-executor",
@@ -1421,7 +1421,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -2299,9 +2299,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.11.4"
+version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
+checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
 dependencies = [
  "equivalent",
  "hashbrown 0.16.0",
@@ -2368,9 +2368,9 @@ dependencies = [
 
 [[package]]
 name = "is_terminal_polyfill"
-version = "1.70.1"
+version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
 
 [[package]]
 name = "itertools"
@@ -2828,7 +2828,7 @@ checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c"
 dependencies = [
  "heck",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -3151,6 +3151,7 @@ dependencies = [
  "tokio-stream",
  "tracing",
  "tracing-subscriber",
+ "unsigned-varint 0.8.0",
  "vimputti",
  "webrtc",
 ]
@@ -3238,7 +3239,7 @@ version = "0.30.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "cfg-if",
  "cfg_aliases",
  "libc",
@@ -3354,9 +3355,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell_polyfill"
-version = "1.70.1"
+version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
 
 [[package]]
 name = "opaque-debug"
@@ -3498,7 +3499,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -3603,7 +3604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -3626,9 +3627,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.101"
+version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
+checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7"
 dependencies = [
  "unicode-ident",
 ]
@@ -3653,7 +3654,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -3676,7 +3677,7 @@ dependencies = [
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -3860,7 +3861,7 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
 ]
 
 [[package]]
@@ -4037,7 +4038,7 @@ version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -4046,9 +4047,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.33"
+version = "0.23.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c"
+checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -4179,7 +4180,7 @@ version = "3.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -4239,7 +4240,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4480,9 +4481,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.106"
+version = "2.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
+checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4506,7 +4507,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4515,7 +4516,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -4573,7 +4574,7 @@ checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4602,7 +4603,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4613,7 +4614,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4706,7 +4707,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4816,7 +4817,7 @@ version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
 dependencies = [
- "bitflags 2.9.4",
+ "bitflags 2.10.0",
  "bytes",
  "futures-util",
  "http",
@@ -4860,7 +4861,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -4985,9 +4986,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.19"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
+checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
 
 [[package]]
 name = "universal-hash"
@@ -5078,9 +5079,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "vimputti"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a5839a89185ccec572f746ccc02e37702cc6c0b62a6aa0d9bcd6e5921edba12"
+checksum = "ffb370ee43e3ee4ca5329886e64dc5b27c83dc8cced5a63c2418777dac9a41a8"
 dependencies = [
  "anyhow",
  "libc",
@@ -5176,7 +5177,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "wasm-bindgen-shared",
 ]
 
@@ -5211,7 +5212,7 @@ checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5501,7 +5502,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -5512,7 +5513,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -5958,7 +5959,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "synstructure",
 ]
 
@@ -5979,7 +5980,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -5999,7 +6000,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
  "synstructure",
 ]
 
@@ -6020,7 +6021,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]
 
 [[package]]
@@ -6053,5 +6054,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.106",
+ "syn 2.0.108",
 ]

packages/server/Cargo.toml

@@ -22,7 +22,7 @@ rand = "0.9"
 rustls = { version = "0.23", features = ["ring"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-vimputti = "0.1.3"
+vimputti = "0.1.4"
 chrono = "0.4"
 prost = "0.14"
 prost-types = "0.14"
@@ -40,3 +40,4 @@ libp2p-tcp = { version = "0.44", features = ["tokio"] }
 libp2p-websocket = "0.45"
 dashmap = "6.1"
 anyhow = "1.0"
+unsigned-varint = "0.8"

packages/server/src/args.rs

@@ -211,6 +211,14 @@ impl Args {
                     .value_parser(value_parser!(u32).range(1..))
                     .default_value("192"),
             )
+            .arg(
+                Arg::new("software-render")
+                    .long("software-render")
+                    .env("SOFTWARE_RENDER")
+                    .help("Use software rendering for wayland")
+                    .value_parser(BoolishValueParser::new())
+                    .default_value("false"),
+            )
             .arg(
                 Arg::new("zero-copy")
                     .long("zero-copy")

packages/server/src/args/app_args.rs

@@ -15,6 +15,9 @@ pub struct AppArgs {
     /// vimputti socket path
     pub vimputti_path: Option<String>,
 
+    /// Use software rendering for wayland display
+    pub software_render: bool,
+
     /// Experimental zero-copy pipeline support
     /// TODO: Move to video encoding flags
     pub zero_copy: bool,
@@ -51,6 +54,10 @@ impl AppArgs {
             vimputti_path: matches
                 .get_one::<String>("vimputti-path")
                 .map(|s| s.clone()),
+            software_render: matches
+                .get_one::<bool>("software-render")
+                .unwrap_or(&false)
+                .clone(),
             zero_copy: matches
                 .get_one::<bool>("zero-copy")
                 .unwrap_or(&false)
@@ -73,6 +80,7 @@ impl AppArgs {
             "> vimputti_path: '{}'",
             self.vimputti_path.as_ref().map_or("None", |s| s.as_str())
         );
+        tracing::info!("> software_render: {}", self.software_render);
         tracing::info!("> zero_copy: {}", self.zero_copy);
     }
 }

packages/server/src/enc_helper.rs

@@ -585,7 +585,6 @@ pub fn get_best_working_encoder(
     encoders: &Vec<VideoEncoderInfo>,
     codec: &Codec,
     encoder_type: &EncoderType,
-    zero_copy: bool,
 ) -> Result<VideoEncoderInfo, Box<dyn Error>> {
     let mut candidates = get_encoders_by_videocodec(
         encoders,
@@ -601,7 +600,7 @@ pub fn get_best_working_encoder(
     while !candidates.is_empty() {
         let best = get_best_compatible_encoder(&candidates, codec, encoder_type)?;
         tracing::info!("Testing encoder: {}", best.name,);
-        if test_encoder(&best, zero_copy).is_ok() {
+        if test_encoder(&best).is_ok() {
             return Ok(best);
         } else {
             // Remove this encoder and try next best
@@ -613,25 +612,10 @@ pub fn get_best_working_encoder(
 }
 
 /// Test if a pipeline with the given encoder can be created and set to Playing
-pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), Box<dyn Error>> {
+pub fn test_encoder(encoder: &VideoEncoderInfo) -> Result<(), Box<dyn Error>> {
-    let src = gstreamer::ElementFactory::make("waylanddisplaysrc").build()?;
+    let src = gstreamer::ElementFactory::make("videotestsrc").build()?;
-    if let Some(gpu_info) = &encoder.gpu_info {
-        src.set_property_from_str("render-node", gpu_info.render_path());
-    }
     let caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?;
-    let caps = gstreamer::Caps::from_str(&format!(
+    let caps = gstreamer::Caps::from_str("video/x-raw,width=1280,height=720,framerate=30/1")?;
-        "{},width=1280,height=720,framerate=30/1{}",
-        if zero_copy {
-            if encoder.encoder_api == EncoderAPI::NVENC {
-                "video/x-raw(memory:CUDAMemory)"
-            } else {
-                "video/x-raw(memory:DMABuf)"
-            }
-        } else {
-            "video/x-raw"
-        },
-        if zero_copy { "" } else { ",format=RGBx" }
-    ))?;
     caps_filter.set_property("caps", &caps);
 
     let enc = gstreamer::ElementFactory::make(&encoder.name).build()?;
@@ -642,41 +626,9 @@ pub fn test_encoder(encoder: &VideoEncoderInfo, zero_copy: bool) -> Result<(), B
     // Create pipeline and link elements
     let pipeline = gstreamer::Pipeline::new();
 
-    if zero_copy {
-        if encoder.encoder_api == EncoderAPI::NVENC {
-            // NVENC zero-copy path
-            pipeline.add_many(&[&src, &caps_filter, &enc, &sink])?;
-            gstreamer::Element::link_many(&[&src, &caps_filter, &enc, &sink])?;
-        } else {
-            // VA-API/QSV zero-copy path
-            let vapostproc = gstreamer::ElementFactory::make("vapostproc").build()?;
-            let va_caps_filter = gstreamer::ElementFactory::make("capsfilter").build()?;
-            let va_caps = gstreamer::Caps::from_str("video/x-raw(memory:VAMemory),format=NV12")?;
-            va_caps_filter.set_property("caps", &va_caps);
-
-            pipeline.add_many(&[
-                &src,
-                &caps_filter,
-                &vapostproc,
-                &va_caps_filter,
-                &enc,
-                &sink,
-            ])?;
-            gstreamer::Element::link_many(&[
-                &src,
-                &caps_filter,
-                &vapostproc,
-                &va_caps_filter,
-                &enc,
-                &sink,
-            ])?;
-        }
-    } else {
-        // Non-zero-copy path for all encoders - needs videoconvert
     let videoconvert = gstreamer::ElementFactory::make("videoconvert").build()?;
     pipeline.add_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?;
     gstreamer::Element::link_many(&[&src, &caps_filter, &videoconvert, &enc, &sink])?;
-    }
 
     let bus = pipeline.bus().ok_or("Pipeline has no bus")?;
     pipeline.set_state(gstreamer::State::Playing)?;

packages/server/src/input/controller.rs

@@ -1,7 +1,5 @@
-use crate::proto::proto::proto_input::InputType::{
+use crate::proto::proto::ProtoControllerAttach;
-    ControllerAttach, ControllerAxis, ControllerButton, ControllerDetach, ControllerRumble,
+use crate::proto::proto::proto_message::Payload;
-    ControllerStick, ControllerTrigger,
-};
 use anyhow::Result;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -48,63 +46,94 @@ impl ControllerInput {
 
 pub struct ControllerManager {
     vimputti_client: Arc<vimputti::client::VimputtiClient>,
-    cmd_tx: mpsc::Sender<crate::proto::proto::ProtoInput>,
+    cmd_tx: mpsc::Sender<Payload>,
-    rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>, // (slot, strong, weak, duration_ms)
+    rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>, // (slot, strong, weak, duration_ms, session_id)
+    attach_tx: mpsc::Sender<ProtoControllerAttach>,
 }
 impl ControllerManager {
     pub fn new(
         vimputti_client: Arc<vimputti::client::VimputtiClient>,
-    ) -> Result<(Self, mpsc::Receiver<(u32, u16, u16, u16)>)> {
+    ) -> Result<(
-        let (cmd_tx, cmd_rx) = mpsc::channel(100);
+        Self,
-        let (rumble_tx, rumble_rx) = mpsc::channel(100);
+        mpsc::Receiver<(u32, u16, u16, u16, String)>,
+        mpsc::Receiver<ProtoControllerAttach>,
+    )> {
+        let (cmd_tx, cmd_rx) = mpsc::channel(512);
+        let (rumble_tx, rumble_rx) = mpsc::channel(256);
+        let (attach_tx, attach_rx) = mpsc::channel(64);
         tokio::spawn(command_loop(
             cmd_rx,
             vimputti_client.clone(),
             rumble_tx.clone(),
+            attach_tx.clone(),
         ));
         Ok((
             Self {
                 vimputti_client,
                 cmd_tx,
                 rumble_tx,
+                attach_tx,
             },
             rumble_rx,
+            attach_rx,
         ))
     }
 
-    pub async fn send_command(&self, input: crate::proto::proto::ProtoInput) -> Result<()> {
+    pub async fn send_command(&self, payload: Payload) -> Result<()> {
-        self.cmd_tx.send(input).await?;
+        self.cmd_tx.send(payload).await?;
         Ok(())
     }
 }
 
+struct ControllerSlot {
+    controller: ControllerInput,
+    session_id: String,
+    session_slot: u32,
+}
+
+// Returns first free controller slot from 0-16
+fn get_free_slot(controllers: &HashMap<u32, ControllerSlot>) -> Option<u32> {
+    for slot in 0..17 {
+        if !controllers.contains_key(&slot) {
+            return Some(slot);
+        }
+    }
+    None
+}
+
 async fn command_loop(
-    mut cmd_rx: mpsc::Receiver<crate::proto::proto::ProtoInput>,
+    mut cmd_rx: mpsc::Receiver<Payload>,
     vimputti_client: Arc<vimputti::client::VimputtiClient>,
-    rumble_tx: mpsc::Sender<(u32, u16, u16, u16)>,
+    rumble_tx: mpsc::Sender<(u32, u16, u16, u16, String)>,
+    attach_tx: mpsc::Sender<ProtoControllerAttach>,
 ) {
-    let mut controllers: HashMap<u32, ControllerInput> = HashMap::new();
+    let mut controllers: HashMap<u32, ControllerSlot> = HashMap::new();
-    while let Some(input) = cmd_rx.recv().await {
+    while let Some(payload) = cmd_rx.recv().await {
-        if let Some(input_type) = input.input_type {
+        match payload {
-            match input_type {
+            Payload::ControllerAttach(data) => {
-                ControllerAttach(data) => {
+                let session_id = data.session_id.clone();
-                    // Check if controller already exists in the slot, if so, ignore
+                let session_slot = data.session_slot.clone();
-                    if controllers.contains_key(&(data.slot as u32)) {
+
-                        tracing::warn!(
+                // Check if this session already has a slot (reconnection)
-                            "Controller slot {} already occupied, ignoring attach",
+                let existing_slot = controllers
-                            data.slot
+                    .iter()
-                        );
+                    .find(|(_, slot)| {
-                    } else {
+                        slot.session_id == session_id && slot.session_slot == session_slot as u32
+                    })
+                    .map(|(slot_num, _)| *slot_num);
+                let slot = existing_slot.or_else(|| get_free_slot(&controllers));
+
+                if let Some(slot) = slot {
                     if let Ok(mut controller) =
                         ControllerInput::new(data.id.clone(), &vimputti_client).await
                     {
-                            let slot = data.slot as u32;
                         let rumble_tx = rumble_tx.clone();
+                        let attach_tx = attach_tx.clone();
 
                         controller
                             .device_mut()
                             .on_rumble(move |strong, weak, duration_ms| {
-                                    let _ = rumble_tx.try_send((slot, strong, weak, duration_ms));
+                                let _ = rumble_tx.try_send((slot, strong, weak, duration_ms, data.session_id.clone()));
                             })
                             .await
                             .map_err(|e| {
@@ -116,38 +145,68 @@ async fn command_loop(
                             })
                             .ok();
 
-                            controllers.insert(data.slot as u32, controller);
+                        // Return to attach_tx what slot was assigned
-                            tracing::info!("Controller {} attached to slot {}", data.id, data.slot);
+                        let attach_info = ProtoControllerAttach {
+                            id: data.id.clone(),
+                            session_slot: slot as i32,
+                            session_id: session_id.clone(),
+                        };
+
+                        match attach_tx.send(attach_info).await {
+                            Ok(_) => {
+                                controllers.insert(
+                                    slot,
+                                    ControllerSlot {
+                                        controller,
+                                        session_id: session_id.clone(),
+                                        session_slot: session_slot.clone() as u32,
+                                    },
+                                );
+                                tracing::info!(
+                                    "Controller {} attached to slot {} (session: {})",
+                                    data.id,
+                                    slot,
+                                    session_id
+                                );
+                            }
+                            Err(e) => {
+                                tracing::error!(
+                                    "Failed to send attach info for slot {}: {}",
+                                    slot,
+                                    e
+                                );
+                            }
+                        }
                     } else {
                         tracing::error!(
                             "Failed to create controller of type {} for slot {}",
                             data.id,
-                                data.slot
+                            slot
                         );
                     }
                 }
             }
-                ControllerDetach(data) => {
+            Payload::ControllerDetach(data) => {
-                    if controllers.remove(&(data.slot as u32)).is_some() {
+                if controllers.remove(&(data.session_slot as u32)).is_some() {
-                        tracing::info!("Controller detached from slot {}", data.slot);
+                    tracing::info!("Controller detached from slot {}", data.session_slot);
                 } else {
-                        tracing::warn!("No controller found in slot {} to detach", data.slot);
+                    tracing::warn!("No controller found in slot {} to detach", data.session_slot);
                 }
             }
-                ControllerButton(data) => {
+            Payload::ControllerButton(data) => {
-                    if let Some(controller) = controllers.get(&(data.slot as u32)) {
+                if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
                     if let Some(button) = vimputti::Button::from_ev_code(data.button as u16) {
-                            let device = controller.device();
+                        let device = controller.controller.device();
                         device.button(button, data.pressed);
                         device.sync();
                     }
                 } else {
-                        tracing::warn!("Controller slot {} not found for button event", data.slot);
+                    tracing::warn!("Controller slot {} not found for button event", data.session_slot);
                 }
             }
-                ControllerStick(data) => {
+            Payload::ControllerStick(data) => {
-                    if let Some(controller) = controllers.get(&(data.slot as u32)) {
+                if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
-                        let device = controller.device();
+                    let device = controller.controller.device();
                     if data.stick == 0 {
                         // Left stick
                         device.axis(vimputti::Axis::LeftStickX, data.x);
@@ -161,12 +220,12 @@ async fn command_loop(
                     }
                     device.sync();
                 } else {
-                        tracing::warn!("Controller slot {} not found for stick event", data.slot);
+                    tracing::warn!("Controller slot {} not found for stick event", data.session_slot);
                 }
             }
-                ControllerTrigger(data) => {
+            Payload::ControllerTrigger(data) => {
-                    if let Some(controller) = controllers.get(&(data.slot as u32)) {
+                if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
-                        let device = controller.device();
+                    let device = controller.controller.device();
                     if data.trigger == 0 {
                         // Left trigger
                         device.axis(vimputti::Axis::LowerLeftTrigger, data.value);
@@ -176,12 +235,12 @@ async fn command_loop(
                     }
                     device.sync();
                 } else {
-                        tracing::warn!("Controller slot {} not found for trigger event", data.slot);
+                    tracing::warn!("Controller slot {} not found for trigger event", data.session_slot);
                 }
             }
-                ControllerAxis(data) => {
+            Payload::ControllerAxis(data) => {
-                    if let Some(controller) = controllers.get(&(data.slot as u32)) {
+                if let Some(controller) = controllers.get(&(data.session_slot as u32)) {
-                        let device = controller.device();
+                    let device = controller.controller.device();
                     if data.axis == 0 {
                         // dpad x
                         device.axis(vimputti::Axis::DPadX, data.value);
@@ -192,14 +251,32 @@ async fn command_loop(
                     device.sync();
                 }
             }
-                // Rumble will be outgoing event..
+            Payload::ClientDisconnected(data) => {
-                ControllerRumble(_) => {
+                tracing::info!(
-                    //no-op
+                    "Client disconnected, cleaning up controller slots: {:?} (client session: {})",
+                    data.controller_slots,
+                    data.session_id
+                );
+                // Remove all controllers for the disconnected slots
+                for slot in &data.controller_slots {
+                    if controllers.remove(&(*slot as u32)).is_some() {
+                        tracing::info!(
+                            "Removed controller from slot {} (client session: {})",
+                            slot,
+                            data.session_id
+                        );
+                    } else {
+                        tracing::warn!(
+                            "No controller found in slot {} to cleanup (client session: {})",
+                            slot,
+                            data.session_id
+                        );
+                    }
+                }
             }
             _ => {
                 //no-op
             }
         }
     }
-    }
 }

packages/server/src/main.rs

@@ -3,7 +3,6 @@ mod enc_helper;
 mod gpu;
 mod input;
 mod latency;
-mod messages;
 mod nestrisink;
 mod p2p;
 mod proto;
@@ -25,7 +24,7 @@ use tracing_subscriber::EnvFilter;
 use tracing_subscriber::filter::LevelFilter;
 
 // Handles gathering GPU information and selecting the most suitable GPU
-fn handle_gpus(args: &args::Args) -> Result<Vec<gpu::GPUInfo>, Box<dyn Error>> {
+fn handle_gpus(args: &args::Args) -> Result<Vec<GPUInfo>, Box<dyn Error>> {
     tracing::info!("Gathering GPU information..");
     let mut gpus = gpu::get_gpus()?;
     if gpus.is_empty() {
@@ -120,7 +119,6 @@ fn handle_encoder_video(
             &video_encoders,
             &args.encoding.video.codec,
             &args.encoding.video.encoder_type,
-            args.app.zero_copy,
         )?;
     }
     tracing::info!("Selected video encoder: '{}'", video_encoder.name);
@@ -257,11 +255,15 @@ async fn main() -> Result<(), Box<dyn Error>> {
             None
         }
     };
-    let (controller_manager, rumble_rx) = if let Some(vclient) = vimputti_client {
+    let (controller_manager, rumble_rx, attach_rx) = if let Some(vclient) = vimputti_client {
-        let (controller_manager, rumble_rx) = ControllerManager::new(vclient)?;
+        let (controller_manager, rumble_rx, attach_rx) = ControllerManager::new(vclient)?;
-        (Some(Arc::new(controller_manager)), Some(rumble_rx))
+        (
+            Some(Arc::new(controller_manager)),
+            Some(rumble_rx),
+            Some(attach_rx),
+        )
     } else {
-        (None, None)
+        (None, None, None)
     };
 
     /*** PIPELINE CREATION ***/
@@ -320,7 +322,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
     /* Video */
     // Video Source Element
     let video_source = Arc::new(gstreamer::ElementFactory::make("waylanddisplaysrc").build()?);
-    if let Some(gpu_info) = &video_encoder_info.gpu_info {
+    if args.app.software_render {
+        video_source.set_property_from_str("render-node", "software");
+    } else if let Some(gpu_info) = &video_encoder_info.gpu_info {
         video_source.set_property_from_str("render-node", gpu_info.render_path());
     }
 
@@ -416,6 +420,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
         video_source.clone(),
         controller_manager,
         rumble_rx,
+        attach_rx,
     )
     .await?;
     let webrtcsink = BaseWebRTCSink::with_signaller(Signallable::from(signaller.clone()));
@@ -424,20 +429,16 @@ async fn main() -> Result<(), Box<dyn Error>> {
     webrtcsink.set_property("do-retransmission", false);
 
     /* Queues */
-    let video_source_queue = gstreamer::ElementFactory::make("queue")
-        .property("max-size-buffers", 5u32)
-        .build()?;
-
-    let audio_source_queue = gstreamer::ElementFactory::make("queue")
-        .property("max-size-buffers", 5u32)
-        .build()?;
-
     let video_queue = gstreamer::ElementFactory::make("queue")
-        .property("max-size-buffers", 5u32)
+        .property("max-size-buffers", 2u32)
+        .property("max-size-time", 0u64)
+        .property("max-size-bytes", 0u32)
         .build()?;
 
     let audio_queue = gstreamer::ElementFactory::make("queue")
-        .property("max-size-buffers", 5u32)
+        .property("max-size-buffers", 2u32)
+        .property("max-size-time", 0u64)
+        .property("max-size-bytes", 0u32)
         .build()?;
 
     /* Clock Sync */
@@ -456,7 +457,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
         &caps_filter,
         &video_queue,
         &video_clocksync,
-        &video_source_queue,
         &video_source,
         &audio_encoder,
         &audio_capsfilter,
@@ -464,7 +464,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
         &audio_clocksync,
         &audio_rate,
         &audio_converter,
-        &audio_source_queue,
         &audio_source,
     ])?;
 
@@ -491,7 +490,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
     // Link main audio branch
     gstreamer::Element::link_many(&[
         &audio_source,
-        &audio_source_queue,
         &audio_converter,
         &audio_rate,
         &audio_capsfilter,
@@ -513,7 +511,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
         if let (Some(vapostproc), Some(va_caps_filter)) = (&vapostproc, &va_caps_filter) {
             gstreamer::Element::link_many(&[
                 &video_source,
-                &video_source_queue,
                 &caps_filter,
                 &video_queue,
                 &video_clocksync,
@@ -525,7 +522,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
             // NVENC pipeline
             gstreamer::Element::link_many(&[
                 &video_source,
-                &video_source_queue,
                 &caps_filter,
                 &video_encoder,
             ])?;
@@ -533,7 +529,6 @@ async fn main() -> Result<(), Box<dyn Error>> {
     } else {
         gstreamer::Element::link_many(&[
             &video_source,
-            &video_source_queue,
             &caps_filter,
             &video_queue,
             &video_clocksync,
@@ -550,7 +545,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     }
 
     // Make sure QOS is disabled to avoid latency
-    video_encoder.set_property("qos", false);
+    video_encoder.set_property("qos", true);
 
     // Optimize latency of pipeline
     video_source

packages/server/src/messages.rs

@@ -1,50 +0,0 @@
-use crate::latency::LatencyTracker;
-use serde::{Deserialize, Serialize};
-use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
-use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageBase {
-    pub payload_type: String,
-    pub latency: Option<LatencyTracker>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageRaw {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub data: serde_json::Value,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageLog {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub level: String,
-    pub message: String,
-    pub time: String,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageMetrics {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub usage_cpu: f64,
-    pub usage_memory: f64,
-    pub uptime: u64,
-    pub pipeline_latency: f64,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageICE {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub candidate: RTCIceCandidateInit,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageSDP {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub sdp: RTCSessionDescription,
-}

packages/server/src/nestrisink/imp.rs

@@ -1,11 +1,11 @@
 use crate::input::controller::ControllerManager;
-use crate::messages::{MessageBase, MessageICE, MessageRaw, MessageSDP};
 use crate::p2p::p2p::NestriConnection;
 use crate::p2p::p2p_protocol_stream::NestriStreamProtocol;
-use crate::proto::proto::proto_input::InputType::{
+use crate::proto::proto::proto_message::Payload;
-    KeyDown, KeyUp, MouseKeyDown, MouseKeyUp, MouseMove, MouseMoveAbs, MouseWheel,
+use crate::proto::proto::{
+    ProtoControllerAttach, ProtoControllerRumble, ProtoIce, ProtoMessage, ProtoSdp,
+    ProtoServerPushStream, RtcIceCandidateInit, RtcSessionDescriptionInit,
 };
-use crate::proto::proto::{ProtoInput, ProtoMessageInput};
 use anyhow::Result;
 use glib::subclass::prelude::*;
 use gstreamer::glib;
@@ -16,8 +16,6 @@ use parking_lot::RwLock as PLRwLock;
 use prost::Message;
 use std::sync::{Arc, LazyLock};
 use tokio::sync::{Mutex, mpsc};
-use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
-use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
 
 pub struct Signaller {
     stream_room: PLRwLock<Option<String>>,
@@ -25,7 +23,8 @@ pub struct Signaller {
     wayland_src: PLRwLock<Option<Arc<gstreamer::Element>>>,
     data_channel: PLRwLock<Option<Arc<gstreamer_webrtc::WebRTCDataChannel>>>,
     controller_manager: PLRwLock<Option<Arc<ControllerManager>>>,
-    rumble_rx: Mutex<Option<mpsc::Receiver<(u32, u16, u16, u16)>>>,
+    rumble_rx: Mutex<Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>>,
+    attach_rx: Mutex<Option<mpsc::Receiver<ProtoControllerAttach>>>,
 }
 impl Default for Signaller {
     fn default() -> Self {
@@ -36,6 +35,7 @@ impl Default for Signaller {
             data_channel: PLRwLock::new(None),
             controller_manager: PLRwLock::new(None),
             rumble_rx: Mutex::new(None),
+            attach_rx: Mutex::new(None),
         }
     }
 }
@@ -70,15 +70,27 @@ impl Signaller {
         self.controller_manager.read().clone()
     }
 
-    pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16)>) {
+    pub async fn set_rumble_rx(&self, rumble_rx: mpsc::Receiver<(u32, u16, u16, u16, String)>) {
         *self.rumble_rx.lock().await = Some(rumble_rx);
     }
 
-    // Change getter to take ownership:
+    pub async fn take_rumble_rx(&self) -> Option<mpsc::Receiver<(u32, u16, u16, u16, String)>> {
-    pub async fn take_rumble_rx(&self) -> Option<mpsc::Receiver<(u32, u16, u16, u16)>> {
         self.rumble_rx.lock().await.take()
     }
 
+    pub async fn set_attach_rx(
+        &self,
+        attach_rx: mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>,
+    ) {
+        *self.attach_rx.lock().await = Some(attach_rx);
+    }
+
+    pub async fn take_attach_rx(
+        &self,
+    ) -> Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>> {
+        self.attach_rx.lock().await.take()
+    }
+
     pub fn set_data_channel(&self, data_channel: gstreamer_webrtc::WebRTCDataChannel) {
         *self.data_channel.write() = Some(Arc::new(data_channel));
     }
@@ -95,59 +107,70 @@ impl Signaller {
         };
         {
             let self_obj = self.obj().clone();
-            stream_protocol.register_callback("answer", move |data| {
+            stream_protocol.register_callback("answer", move |msg| {
-                if let Ok(message) = serde_json::from_slice::<MessageSDP>(&data) {
+                if let Some(payload) = msg.payload {
-                    let sdp = gst_sdp::SDPMessage::parse_buffer(message.sdp.sdp.as_bytes())
+                    match payload {
-                        .map_err(|e| anyhow::anyhow!("Invalid SDP in 'answer': {e:?}"))?;
+                        Payload::Sdp(sdp) => {
-                    let answer = WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
+                            if let Some(sdp) = sdp.sdp {
-                    Ok(self_obj.emit_by_name::<()>(
+                                let sdp = gst_sdp::SDPMessage::parse_buffer(sdp.sdp.as_bytes())
+                                    .map_err(|e| {
+                                        anyhow::anyhow!("Invalid SDP in 'answer': {e:?}")
+                                    })?;
+                                let answer =
+                                    WebRTCSessionDescription::new(WebRTCSDPType::Answer, sdp);
+                                return Ok(self_obj.emit_by_name::<()>(
                                     "session-description",
                                     &[&"unique-session-id", &answer],
-                    ))
+                                ));
-                } else {
-                    anyhow::bail!("Failed to decode SDP message");
                             }
+                        }
+                        _ => {
+                            tracing::warn!("Unexpected payload type for answer");
+                            return Ok(());
+                        }
+                    }
+                } else {
+                    anyhow::bail!("Failed to decode answer message");
+                }
+                Ok(())
             });
         }
         {
             let self_obj = self.obj().clone();
-            stream_protocol.register_callback("ice-candidate", move |data| {
+            stream_protocol.register_callback("ice-candidate", move |msg| {
-                if let Ok(message) = serde_json::from_slice::<MessageICE>(&data) {
+                if let Some(payload) = msg.payload {
-                    let candidate = message.candidate;
+                    match payload {
-                    let sdp_m_line_index = candidate.sdp_mline_index.unwrap_or(0) as u32;
+                        Payload::Ice(ice) => {
-                    let sdp_mid = candidate.sdp_mid;
+                            if let Some(candidate) = ice.candidate {
-                    Ok(self_obj.emit_by_name::<()>(
+                                let sdp_m_line_index = candidate.sdp_m_line_index.unwrap_or(0);
+                                return Ok(self_obj.emit_by_name::<()>(
                                     "handle-ice",
                                     &[
                                         &"unique-session-id",
                                         &sdp_m_line_index,
-                            &sdp_mid,
+                                        &candidate.sdp_mid,
                                         &candidate.candidate,
                                     ],
-                    ))
+                                ));
+                            }
+                        }
+                        _ => {
+                            tracing::warn!("Unexpected payload type for ice-candidate");
+                            return Ok(());
+                        }
+                    }
                 } else {
                     anyhow::bail!("Failed to decode ICE message");
                 }
+                Ok(())
             });
         }
         {
             let self_obj = self.obj().clone();
-            stream_protocol.register_callback("push-stream-ok", move |data| {
+            stream_protocol.register_callback("push-stream-ok", move |msg| {
-                if let Ok(answer) = serde_json::from_slice::<MessageRaw>(&data) {
+                if let Some(payload) = msg.payload {
-                    // Decode room name string
+                    return match payload {
-                    if let Some(room_name) = answer.data.as_str() {
+                        Payload::ServerPushStream(_res) => {
-                        gstreamer::info!(
-                            gstreamer::CAT_DEFAULT,
-                            "Received OK answer for room: {}",
-                            room_name
-                        );
-                    } else {
-                        gstreamer::error!(
-                            gstreamer::CAT_DEFAULT,
-                            "Failed to decode room name from answer"
-                        );
-                    }
-
                             // Send our SDP offer
                             Ok(self_obj.emit_by_name::<()>(
                                 "session-requested",
@@ -157,6 +180,12 @@ impl Signaller {
                                     &None::<WebRTCSessionDescription>,
                                 ],
                             ))
+                        }
+                        _ => {
+                            tracing::warn!("Unexpected payload type for push-stream-ok");
+                            Ok(())
+                        }
+                    };
                 } else {
                     anyhow::bail!("Failed to decode answer");
                 }
@@ -200,12 +229,14 @@ impl Signaller {
                                 // Spawn async task to take the receiver and set up
                                 tokio::spawn(async move {
                                     let rumble_rx = signaller.imp().take_rumble_rx().await;
+                                    let attach_rx = signaller.imp().take_attach_rx().await;
                                     let controller_manager =
                                         signaller.imp().get_controller_manager();
 
                                     setup_data_channel(
                                         controller_manager,
                                         rumble_rx,
+                                        attach_rx,
                                         data_channel,
                                         &wayland_src,
                                     );
@@ -243,19 +274,18 @@ impl SignallableImpl for Signaller {
             return;
         };
 
-        let push_msg = MessageRaw {
-            base: MessageBase {
-                payload_type: "push-stream-room".to_string(),
-                latency: None,
-            },
-            data: serde_json::Value::from(stream_room),
-        };
-
         let Some(stream_protocol) = self.get_stream_protocol() else {
             gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
             return;
         };
 
+        let push_msg = crate::proto::create_message(
+            Payload::ServerPushStream(ProtoServerPushStream {
+                room_name: stream_room,
+            }),
+            "push-stream-room",
+            None,
+        );
         if let Err(e) = stream_protocol.send_message(&push_msg) {
             tracing::error!("Failed to send push stream room message: {:?}", e);
         }
@@ -266,20 +296,22 @@ impl SignallableImpl for Signaller {
     }
 
     fn send_sdp(&self, _session_id: &str, sdp: &WebRTCSessionDescription) {
-        let sdp_message = MessageSDP {
-            base: MessageBase {
-                payload_type: "offer".to_string(),
-                latency: None,
-            },
-            sdp: RTCSessionDescription::offer(sdp.sdp().as_text().unwrap()).unwrap(),
-        };
-
         let Some(stream_protocol) = self.get_stream_protocol() else {
             gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
             return;
         };
 
-        if let Err(e) = stream_protocol.send_message(&sdp_message) {
+        let sdp_msg = crate::proto::create_message(
+            Payload::Sdp(ProtoSdp {
+                sdp: Some(RtcSessionDescriptionInit {
+                    sdp: sdp.sdp().as_text().unwrap(),
+                    r#type: "offer".to_string(),
+                }),
+            }),
+            "offer",
+            None,
+        );
+        if let Err(e) = stream_protocol.send_message(&sdp_msg) {
             tracing::error!("Failed to send SDP message: {:?}", e);
         }
     }
@@ -291,26 +323,25 @@ impl SignallableImpl for Signaller {
         sdp_m_line_index: u32,
         sdp_mid: Option<String>,
     ) {
-        let candidate_init = RTCIceCandidateInit {
-            candidate: candidate.to_string(),
-            sdp_mid,
-            sdp_mline_index: Some(sdp_m_line_index as u16),
-            ..Default::default()
-        };
-        let ice_message = MessageICE {
-            base: MessageBase {
-                payload_type: "ice-candidate".to_string(),
-                latency: None,
-            },
-            candidate: candidate_init,
-        };
-
         let Some(stream_protocol) = self.get_stream_protocol() else {
             gstreamer::error!(gstreamer::CAT_DEFAULT, "Stream protocol not set");
             return;
         };
 
-        if let Err(e) = stream_protocol.send_message(&ice_message) {
+        let candidate_init = RtcIceCandidateInit {
+            candidate: candidate.to_string(),
+            sdp_mid,
+            sdp_m_line_index: Some(sdp_m_line_index),
+            ..Default::default() //username_fragment: Some(session_id.to_string()), TODO: required?
+        };
+        let ice_msg = crate::proto::create_message(
+            Payload::Ice(ProtoIce {
+                candidate: Some(candidate_init),
+            }),
+            "ice-candidate",
+            None,
+        );
+        if let Err(e) = stream_protocol.send_message(&ice_msg) {
             tracing::error!("Failed to send ICE candidate message: {:?}", e);
         }
     }
@@ -351,7 +382,8 @@ impl ObjectImpl for Signaller {
 
 fn setup_data_channel(
     controller_manager: Option<Arc<ControllerManager>>,
-    rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>, // (slot, strong, weak, duration_ms)
+    rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>, // (slot, strong, weak, duration_ms, session_id)
+    attach_rx: Option<mpsc::Receiver<ProtoControllerAttach>>,
     data_channel: Arc<gstreamer_webrtc::WebRTCDataChannel>,
     wayland_src: &gstreamer::Element,
 ) {
@@ -361,11 +393,11 @@ fn setup_data_channel(
     // Spawn async processor
     tokio::spawn(async move {
         while let Some(data) = rx.recv().await {
-            match ProtoMessageInput::decode(data.as_slice()) {
+            match ProtoMessage::decode(data.as_slice()) {
-                Ok(message_input) => {
+                Ok(msg_wrapper) => {
-                    if let Some(message_base) = message_input.message_base {
+                    if let Some(message_base) = msg_wrapper.message_base {
                         if message_base.payload_type == "input" {
-                            if let Some(input_data) = message_input.data {
+                            if let Some(input_data) = msg_wrapper.payload {
                                 if let Some(event) = handle_input_message(input_data) {
                                     // Send the event to wayland source, result bool is ignored
                                     let _ = wayland_src.send_event(event);
@@ -373,7 +405,7 @@ fn setup_data_channel(
                             }
                         } else if message_base.payload_type == "controllerInput" {
                             if let Some(controller_manager) = &controller_manager {
-                                if let Some(input_data) = message_input.data {
+                                if let Some(input_data) = msg_wrapper.payload {
                                     let _ = controller_manager.send_command(input_data).await;
                                 }
                             }
@@ -391,26 +423,18 @@ fn setup_data_channel(
     if let Some(mut rumble_rx) = rumble_rx {
         let data_channel_clone = data_channel.clone();
         tokio::spawn(async move {
-            while let Some((slot, strong, weak, duration_ms)) = rumble_rx.recv().await {
+            while let Some((slot, strong, weak, duration_ms, session_id)) = rumble_rx.recv().await {
-                let rumble_msg = ProtoMessageInput {
+                let rumble_msg = crate::proto::create_message(
-                    message_base: Some(crate::proto::proto::ProtoMessageBase {
+                    Payload::ControllerRumble(ProtoControllerRumble {
-                        payload_type: "controllerInput".to_string(),
+                        session_slot: slot as i32,
-                        latency: None,
+                        session_id: session_id,
-                    }),
-                    data: Some(ProtoInput {
-                        input_type: Some(
-                            crate::proto::proto::proto_input::InputType::ControllerRumble(
-                                crate::proto::proto::ProtoControllerRumble {
-                                    r#type: "ControllerRumble".to_string(),
-                                    slot: slot as i32,
                         low_frequency: weak as i32,
                         high_frequency: strong as i32,
                         duration: duration_ms as i32,
-                                },
-                            ),
-                        ),
                     }),
-                };
+                    "controllerInput",
+                    None,
+                );
 
                 let data = rumble_msg.encode_to_vec();
                 let bytes = glib::Bytes::from_owned(data);
@@ -422,6 +446,27 @@ fn setup_data_channel(
         });
     }
 
+    // Spawn attach sender
+    if let Some(mut attach_rx) = attach_rx {
+        let data_channel_clone = data_channel.clone();
+        tokio::spawn(async move {
+            while let Some(attach_msg) = attach_rx.recv().await {
+                let proto_msg = crate::proto::create_message(
+                    Payload::ControllerAttach(attach_msg),
+                    "controllerInput",
+                    None,
+                );
+
+                let data = proto_msg.encode_to_vec();
+                let bytes = glib::Bytes::from_owned(data);
+
+                if let Err(e) = data_channel_clone.send_data_full(Some(&bytes)) {
+                    tracing::warn!("Failed to send controller attach data: {}", e);
+                }
+            }
+        });
+    }
+
     data_channel.connect_on_message_data(move |_data_channel, data| {
         if let Some(data) = data {
             let _ = tx.send(data.to_vec());
@@ -429,10 +474,9 @@ fn setup_data_channel(
     });
 }
 
-fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
+fn handle_input_message(payload: Payload) -> Option<gstreamer::Event> {
-    if let Some(input_type) = input_msg.input_type {
+    match payload {
-        match input_type {
+        Payload::MouseMove(data) => {
-            MouseMove(data) => {
             let structure = gstreamer::Structure::builder("MouseMoveRelative")
                 .field("pointer_x", data.x as f64)
                 .field("pointer_y", data.y as f64)
@@ -440,7 +484,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            MouseMoveAbs(data) => {
+        Payload::MouseMoveAbs(data) => {
             let structure = gstreamer::Structure::builder("MouseMoveAbsolute")
                 .field("pointer_x", data.x as f64)
                 .field("pointer_y", data.y as f64)
@@ -448,7 +492,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            KeyDown(data) => {
+        Payload::KeyDown(data) => {
             let structure = gstreamer::Structure::builder("KeyboardKey")
                 .field("key", data.key as u32)
                 .field("pressed", true)
@@ -456,7 +500,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            KeyUp(data) => {
+        Payload::KeyUp(data) => {
             let structure = gstreamer::Structure::builder("KeyboardKey")
                 .field("key", data.key as u32)
                 .field("pressed", false)
@@ -464,7 +508,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            MouseWheel(data) => {
+        Payload::MouseWheel(data) => {
             let structure = gstreamer::Structure::builder("MouseAxis")
                 .field("x", data.x as f64)
                 .field("y", data.y as f64)
@@ -472,7 +516,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            MouseKeyDown(data) => {
+        Payload::MouseKeyDown(data) => {
             let structure = gstreamer::Structure::builder("MouseButton")
                 .field("button", data.key as u32)
                 .field("pressed", true)
@@ -480,7 +524,7 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
 
             Some(gstreamer::event::CustomUpstream::new(structure))
         }
-            MouseKeyUp(data) => {
+        Payload::MouseKeyUp(data) => {
             let structure = gstreamer::Structure::builder("MouseButton")
                 .field("button", data.key as u32)
                 .field("pressed", false)
@@ -490,7 +534,4 @@ fn handle_input_message(input_msg: ProtoInput) -> Option<gstreamer::Event> {
         }
         _ => None,
     }
-    } else {
-        None
-    }
 }

packages/server/src/nestrisink/mod.rs

@@ -18,7 +18,8 @@ impl NestriSignaller {
         nestri_conn: NestriConnection,
         wayland_src: Arc<gstreamer::Element>,
         controller_manager: Option<Arc<ControllerManager>>,
-        rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16)>>,
+        rumble_rx: Option<mpsc::Receiver<(u32, u16, u16, u16, String)>>,
+        attach_rx: Option<mpsc::Receiver<crate::proto::proto::ProtoControllerAttach>>,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let obj: Self = glib::Object::new();
         obj.imp().set_stream_room(room);
@@ -30,6 +31,9 @@ impl NestriSignaller {
         if let Some(rumble_rx) = rumble_rx {
             obj.imp().set_rumble_rx(rumble_rx).await;
         }
+        if let Some(attach_rx) = attach_rx {
+            obj.imp().set_attach_rx(attach_rx).await;
+        }
         Ok(obj)
     }
 }

packages/server/src/p2p/p2p_protocol_stream.rs

@@ -3,21 +3,22 @@ use crate::p2p::p2p_safestream::SafeStream;
 use anyhow::Result;
 use dashmap::DashMap;
 use libp2p::StreamProtocol;
+use prost::Message;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 
 // Cloneable callback type
-pub type CallbackInner = dyn Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static;
+pub type CallbackInner = dyn Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static;
 pub struct Callback(Arc<CallbackInner>);
 impl Callback {
     pub fn new<F>(f: F) -> Self
     where
-        F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
+        F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
     {
         Callback(Arc::new(f))
     }
 
-    pub fn call(&self, data: Vec<u8>) -> Result<()> {
+    pub fn call(&self, data: crate::proto::proto::ProtoMessage) -> Result<()> {
         self.0(data)
     }
 }
@@ -104,15 +105,17 @@ impl NestriStreamProtocol {
                     }
                 };
 
-                match serde_json::from_slice::<crate::messages::MessageBase>(&data) {
+                match crate::proto::proto::ProtoMessage::decode(data.as_slice()) {
-                    Ok(base_message) => {
+                    Ok(message) => {
-                        let response_type = base_message.payload_type;
+                        if let Some(base_message) = &message.message_base {
+                            let response_type = &base_message.payload_type;
+                            let response_type = response_type.clone();
 
                             // With DashMap, we don't need explicit locking
                             // we just get the callback directly if it exists
                             if let Some(callback) = callbacks.get(&response_type) {
                                 // Execute the callback
-                            if let Err(e) = callback.call(data.clone()) {
+                                if let Err(e) = callback.call(message) {
                                     tracing::error!(
                                         "Callback for response type '{}' errored: {:?}",
                                         response_type,
@@ -125,6 +128,9 @@ impl NestriStreamProtocol {
                                     response_type
                                 );
                             }
+                        } else {
+                            tracing::error!("No base message in decoded protobuf message",);
+                        }
                     }
                     Err(e) => {
                         tracing::error!("Failed to decode message: {}", e);
@@ -154,8 +160,9 @@ impl NestriStreamProtocol {
         })
     }
 
-    pub fn send_message<M: serde::Serialize>(&self, message: &M) -> Result<()> {
+    pub fn send_message(&self, message: &crate::proto::proto::ProtoMessage) -> Result<()> {
-        let json_data = serde_json::to_vec(message)?;
+        let mut buf = Vec::new();
+        message.encode(&mut buf)?;
         let Some(tx) = &self.tx else {
             return Err(anyhow::Error::msg(
                 if self.read_handle.is_none() && self.write_handle.is_none() {
@@ -165,13 +172,13 @@ impl NestriStreamProtocol {
                 },
             ));
         };
-        tx.try_send(json_data)?;
+        tx.try_send(buf)?;
         Ok(())
     }
 
     pub fn register_callback<F>(&self, response_type: &str, callback: F)
     where
-        F: Fn(Vec<u8>) -> Result<()> + Send + Sync + 'static,
+        F: Fn(crate::proto::proto::ProtoMessage) -> Result<()> + Send + Sync + 'static,
     {
         self.callbacks
             .insert(response_type.to_string(), Callback::new(callback));

packages/server/src/p2p/p2p_safestream.rs

@@ -1,11 +1,9 @@
 use anyhow::Result;
-use byteorder::{BigEndian, ByteOrder};
 use libp2p::futures::io::{ReadHalf, WriteHalf};
 use libp2p::futures::{AsyncReadExt, AsyncWriteExt};
 use std::sync::Arc;
 use tokio::sync::Mutex;
-
+use unsigned_varint::{decode, encode};
-const MAX_SIZE: usize = 1024 * 1024; // 1MB
 
 pub struct SafeStream {
     stream_read: Arc<Mutex<ReadHalf<libp2p::Stream>>>,
@@ -29,34 +27,52 @@ impl SafeStream {
     }
 
     async fn send_with_length_prefix(&self, data: &[u8]) -> Result<()> {
-        if data.len() > MAX_SIZE {
-            anyhow::bail!("Data exceeds maximum size");
-        }
-
-        let mut buffer = Vec::with_capacity(4 + data.len());
-        buffer.extend_from_slice(&(data.len() as u32).to_be_bytes()); // Length prefix
-        buffer.extend_from_slice(data); // Payload
-
         let mut stream_write = self.stream_write.lock().await;
-        stream_write.write_all(&buffer).await?; // Single write
+
+        // Encode length as varint
+        let mut length_buf = encode::usize_buffer();
+        let length_bytes = encode::usize(data.len(), &mut length_buf);
+
+        // Write varint length prefix
+        stream_write.write_all(length_bytes).await?;
+
+        // Write payload
+        stream_write.write_all(data).await?;
         stream_write.flush().await?;
+
         Ok(())
     }
 
     async fn receive_with_length_prefix(&self) -> Result<Vec<u8>> {
         let mut stream_read = self.stream_read.lock().await;
 
-        // Read length prefix + data in one syscall
+        // Read varint length prefix (up to 10 bytes for u64)
-        let mut length_prefix = [0u8; 4];
+        let mut length_buf = Vec::new();
-        stream_read.read_exact(&mut length_prefix).await?;
+        let mut temp_byte = [0u8; 1];
-        let length = BigEndian::read_u32(&length_prefix) as usize;
 
-        if length > MAX_SIZE {
+        loop {
-            anyhow::bail!("Received data exceeds maximum size");
+            stream_read.read_exact(&mut temp_byte).await?;
+            length_buf.push(temp_byte[0]);
+
+            // Check if this is the last byte (MSB = 0)
+            if temp_byte[0] & 0x80 == 0 {
+                break;
             }
 
+            // Protect against malicious infinite varints
+            if length_buf.len() > 10 {
+                anyhow::bail!("Invalid varint encoding");
+            }
+        }
+
+        // Decode the varint
+        let (length, _) = decode::usize(&length_buf)
+            .map_err(|e| anyhow::anyhow!("Failed to decode varint: {}", e))?;
+
+        // Read payload
         let mut buffer = vec![0u8; length];
         stream_read.read_exact(&mut buffer).await?;
+
         Ok(buffer)
     }
 }

packages/server/src/proto.rs

@@ -1 +1,35 @@
 pub mod proto;
+
+pub struct CreateMessageOptions {
+    pub sequence_id: Option<String>,
+    pub latency: Option<proto::ProtoLatencyTracker>,
+}
+
+pub fn create_message(
+    payload: proto::proto_message::Payload,
+    payload_type: impl Into<String>,
+    options: Option<CreateMessageOptions>,
+) -> proto::ProtoMessage {
+    let opts = options.unwrap_or(CreateMessageOptions {
+        sequence_id: None,
+        latency: None,
+    });
+
+    let latency = opts.latency.or_else(|| {
+        opts.sequence_id.map(|seq_id| proto::ProtoLatencyTracker {
+            sequence_id: seq_id,
+            timestamps: vec![proto::ProtoTimestampEntry {
+                stage: "created".to_string(),
+                time: Some(prost_types::Timestamp::from(std::time::SystemTime::now())),
+            }],
+        })
+    });
+
+    proto::ProtoMessage {
+        message_base: Some(proto::ProtoMessageBase {
+            payload_type: payload_type.into(),
+            latency,
+        }),
+        payload: Some(payload),
+    }
+}

packages/server/src/proto/gen.rs

@@ -1,202 +0,0 @@
-// @generated
-// This file is @generated by prost-build.
-/// EntityState represents the state of an entity in the mesh (e.g., a room).
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct EntityState {
-    /// Type of entity (e.g., "room")
-    #[prost(string, tag="1")]
-    pub entity_type: ::prost::alloc::string::String,
-    /// Unique identifier (e.g., room name)
-    #[prost(string, tag="2")]
-    pub entity_id: ::prost::alloc::string::String,
-    /// Whether the entity is active
-    #[prost(bool, tag="3")]
-    pub active: bool,
-    /// Relay ID that owns this entity
-    #[prost(string, tag="4")]
-    pub owner_relay_id: ::prost::alloc::string::String,
-}
-/// MeshMessage is the top-level message for all relay-to-relay communication.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct MeshMessage {
-    #[prost(oneof="mesh_message::Type", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13")]
-    pub r#type: ::core::option::Option<mesh_message::Type>,
-}
-/// Nested message and enum types in `MeshMessage`.
-pub mod mesh_message {
-    #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Oneof)]
-    pub enum Type {
-        /// Level 0
-        #[prost(message, tag="1")]
-        StateUpdate(super::StateUpdate),
-        #[prost(message, tag="2")]
-        Ack(super::Ack),
-        #[prost(message, tag="3")]
-        RetransmissionRequest(super::RetransmissionRequest),
-        #[prost(message, tag="4")]
-        Retransmission(super::Retransmission),
-        #[prost(message, tag="5")]
-        Heartbeat(super::Heartbeat),
-        #[prost(message, tag="6")]
-        SuspectRelay(super::SuspectRelay),
-        #[prost(message, tag="7")]
-        Disconnect(super::Disconnect),
-        /// Level 1
-        #[prost(message, tag="8")]
-        ForwardSdp(super::ForwardSdp),
-        #[prost(message, tag="9")]
-        ForwardIce(super::ForwardIce),
-        #[prost(message, tag="10")]
-        ForwardIngest(super::ForwardIngest),
-        #[prost(message, tag="11")]
-        StreamRequest(super::StreamRequest),
-        /// Level 2
-        #[prost(message, tag="12")]
-        Handshake(super::Handshake),
-        #[prost(message, tag="13")]
-        HandshakeResponse(super::HandshakeResponse),
-    }
-}
-/// Handshake to inititiate new connection to mesh.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Handshake {
-    /// UUID of the relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// base64 encoded Diffie-Hellman public key
-    #[prost(string, tag="2")]
-    pub dh_public_key: ::prost::alloc::string::String,
-}
-/// HandshakeResponse to respond to a mesh joiner.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct HandshakeResponse {
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
-    pub dh_public_key: ::prost::alloc::string::String,
-    /// relay id to signature
-    #[prost(map="string, string", tag="3")]
-    pub approvals: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
-}
-/// Forwarded SDP from another relay.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ForwardSdp {
-    #[prost(string, tag="1")]
-    pub room_name: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
-    pub participant_id: ::prost::alloc::string::String,
-    #[prost(string, tag="3")]
-    pub sdp: ::prost::alloc::string::String,
-    /// "offer" or "answer"
-    #[prost(string, tag="4")]
-    pub r#type: ::prost::alloc::string::String,
-}
-/// Forwarded ICE candidate from another relay.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ForwardIce {
-    #[prost(string, tag="1")]
-    pub room_name: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
-    pub participant_id: ::prost::alloc::string::String,
-    #[prost(string, tag="3")]
-    pub candidate: ::prost::alloc::string::String,
-}
-/// Forwarded ingest room from another relay.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ForwardIngest {
-    #[prost(string, tag="1")]
-    pub room_name: ::prost::alloc::string::String,
-}
-/// Stream request from mesh.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct StreamRequest {
-    #[prost(string, tag="1")]
-    pub room_name: ::prost::alloc::string::String,
-}
-/// StateUpdate propagates entity state changes across the mesh.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct StateUpdate {
-    /// Unique sequence number for this update
-    #[prost(uint64, tag="1")]
-    pub sequence_number: u64,
-    /// Key: entity_id (e.g., room name), Value: EntityState
-    #[prost(map="string, message", tag="2")]
-    pub entities: ::std::collections::HashMap<::prost::alloc::string::String, EntityState>,
-}
-/// Ack acknowledges receipt of a StateUpdate.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Ack {
-    /// UUID of the acknowledging relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// Sequence number being acknowledged
-    #[prost(uint64, tag="2")]
-    pub sequence_number: u64,
-}
-/// RetransmissionRequest requests a missed StateUpdate.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct RetransmissionRequest {
-    /// UUID of the requesting relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// Sequence number of the missed update
-    #[prost(uint64, tag="2")]
-    pub sequence_number: u64,
-}
-/// Retransmission resends a StateUpdate.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Retransmission {
-    /// UUID of the sending relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// The retransmitted update
-    #[prost(message, optional, tag="2")]
-    pub state_update: ::core::option::Option<StateUpdate>,
-}
-/// Heartbeat signals relay liveness.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Heartbeat {
-    /// UUID of the sending relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// Time of the heartbeat
-    #[prost(message, optional, tag="2")]
-    pub timestamp: ::core::option::Option<::prost_types::Timestamp>,
-}
-/// SuspectRelay marks a relay as potentially unresponsive.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct SuspectRelay {
-    /// UUID of the suspected relay
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// Reason for suspicion (e.g., "no heartbeat")
-    #[prost(string, tag="2")]
-    pub reason: ::prost::alloc::string::String,
-}
-/// Disconnect signals to remove a relay from the mesh.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Disconnect {
-    /// UUID of the relay to disconnect
-    #[prost(string, tag="1")]
-    pub relay_id: ::prost::alloc::string::String,
-    /// Reason for disconnection (e.g., "unresponsive")
-    #[prost(string, tag="2")]
-    pub reason: ::prost::alloc::string::String,
-}
-// @@protoc_insertion_point(module)

packages/server/src/proto/proto.rs

@@ -20,80 +20,59 @@ pub struct ProtoLatencyTracker {
 
 /// MouseMove message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoMouseMove {
-    /// Fixed value "MouseMove"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub x: i32,
-    #[prost(int32, tag="3")]
+    #[prost(int32, tag="2")]
     pub y: i32,
 }
 /// MouseMoveAbs message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoMouseMoveAbs {
-    /// Fixed value "MouseMoveAbs"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub x: i32,
-    #[prost(int32, tag="3")]
+    #[prost(int32, tag="2")]
     pub y: i32,
 }
 /// MouseWheel message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoMouseWheel {
-    /// Fixed value "MouseWheel"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub x: i32,
-    #[prost(int32, tag="3")]
+    #[prost(int32, tag="2")]
     pub y: i32,
 }
 /// MouseKeyDown message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoMouseKeyDown {
-    /// Fixed value "MouseKeyDown"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub key: i32,
 }
 /// MouseKeyUp message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoMouseKeyUp {
-    /// Fixed value "MouseKeyUp"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub key: i32,
 }
 // Keyboard messages 
 
 /// KeyDown message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoKeyDown {
-    /// Fixed value "KeyDown"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub key: i32,
 }
 /// KeyUp message
 #[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
 pub struct ProtoKeyUp {
-    /// Fixed value "KeyUp"
+    #[prost(int32, tag="1")]
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
-    #[prost(int32, tag="2")]
     pub key: i32,
 }
 // Controller messages 
@@ -102,37 +81,37 @@ pub struct ProtoKeyUp {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerAttach {
-    /// Fixed value "ControllerAttach"
-    #[prost(string, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
     /// One of the following enums: "ps", "xbox" or "switch"
-    #[prost(string, tag="2")]
+    #[prost(string, tag="1")]
     pub id: ::prost::alloc::string::String,
-    /// Slot number (0-3)
+    /// Session specific slot number (0-3)
-    #[prost(int32, tag="3")]
+    #[prost(int32, tag="2")]
-    pub slot: i32,
+    pub session_slot: i32,
+    /// Session ID of the client
+    #[prost(string, tag="3")]
+    pub session_id: ::prost::alloc::string::String,
 }
 /// ControllerDetach message
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerDetach {
-    /// Fixed value "ControllerDetach"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
 }
 /// ControllerButton message
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerButton {
-    /// Fixed value "ControllerButtons"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
     /// Button code (linux input event code)
     #[prost(int32, tag="3")]
     pub button: i32,
@@ -144,12 +123,12 @@ pub struct ProtoControllerButton {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerTrigger {
-    /// Fixed value "ControllerTriggers"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
     /// Trigger number (0 for left, 1 for right)
     #[prost(int32, tag="3")]
     pub trigger: i32,
@@ -161,12 +140,12 @@ pub struct ProtoControllerTrigger {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerStick {
-    /// Fixed value "ControllerStick"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
     /// Stick number (0 for left, 1 for right)
     #[prost(int32, tag="3")]
     pub stick: i32,
@@ -181,12 +160,12 @@ pub struct ProtoControllerStick {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerAxis {
-    /// Fixed value "ControllerAxis"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
     /// Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
     #[prost(int32, tag="3")]
     pub axis: i32,
@@ -198,12 +177,12 @@ pub struct ProtoControllerAxis {
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ProtoControllerRumble {
-    /// Fixed value "ControllerRumble"
+    /// Session specific slot number (0-3)
-    #[prost(string, tag="1")]
+    #[prost(int32, tag="1")]
-    pub r#type: ::prost::alloc::string::String,
+    pub session_slot: i32,
-    /// Slot number (0-3)
+    /// Session ID of the client
-    #[prost(int32, tag="2")]
+    #[prost(string, tag="2")]
-    pub slot: i32,
+    pub session_id: ::prost::alloc::string::String,
     /// Low frequency rumble (0-65535)
     #[prost(int32, tag="3")]
     pub low_frequency: i32,
@@ -214,47 +193,73 @@ pub struct ProtoControllerRumble {
     #[prost(int32, tag="5")]
     pub duration: i32,
 }
-/// Union of all Input types
+// WebRTC + signaling 
+
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ProtoInput {
+pub struct RtcIceCandidateInit {
-    #[prost(oneof="proto_input::InputType", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14")]
+    #[prost(string, tag="1")]
-    pub input_type: ::core::option::Option<proto_input::InputType>,
+    pub candidate: ::prost::alloc::string::String,
+    #[prost(uint32, optional, tag="2")]
+    pub sdp_m_line_index: ::core::option::Option<u32>,
+    #[prost(string, optional, tag="3")]
+    pub sdp_mid: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(string, optional, tag="4")]
+    pub username_fragment: ::core::option::Option<::prost::alloc::string::String>,
 }
-/// Nested message and enum types in `ProtoInput`.
+#[allow(clippy::derive_partial_eq_without_eq)]
-pub mod proto_input {
+#[derive(Clone, PartialEq, ::prost::Message)]
-    #[allow(clippy::derive_partial_eq_without_eq)]
+pub struct RtcSessionDescriptionInit {
-#[derive(Clone, PartialEq, ::prost::Oneof)]
+    #[prost(string, tag="1")]
-    pub enum InputType {
+    pub sdp: ::prost::alloc::string::String,
-        #[prost(message, tag="1")]
+    #[prost(string, tag="2")]
-        MouseMove(super::ProtoMouseMove),
+    pub r#type: ::prost::alloc::string::String,
-        #[prost(message, tag="2")]
+}
-        MouseMoveAbs(super::ProtoMouseMoveAbs),
+/// ProtoICE message
-        #[prost(message, tag="3")]
+#[allow(clippy::derive_partial_eq_without_eq)]
-        MouseWheel(super::ProtoMouseWheel),
+#[derive(Clone, PartialEq, ::prost::Message)]
-        #[prost(message, tag="4")]
+pub struct ProtoIce {
-        MouseKeyDown(super::ProtoMouseKeyDown),
+    #[prost(message, optional, tag="1")]
-        #[prost(message, tag="5")]
+    pub candidate: ::core::option::Option<RtcIceCandidateInit>,
-        MouseKeyUp(super::ProtoMouseKeyUp),
+}
-        #[prost(message, tag="6")]
+/// ProtoSDP message
-        KeyDown(super::ProtoKeyDown),
+#[allow(clippy::derive_partial_eq_without_eq)]
-        #[prost(message, tag="7")]
+#[derive(Clone, PartialEq, ::prost::Message)]
-        KeyUp(super::ProtoKeyUp),
+pub struct ProtoSdp {
-        #[prost(message, tag="8")]
+    #[prost(message, optional, tag="1")]
-        ControllerAttach(super::ProtoControllerAttach),
+    pub sdp: ::core::option::Option<RtcSessionDescriptionInit>,
-        #[prost(message, tag="9")]
+}
-        ControllerDetach(super::ProtoControllerDetach),
+/// ProtoRaw message
-        #[prost(message, tag="10")]
+#[allow(clippy::derive_partial_eq_without_eq)]
-        ControllerButton(super::ProtoControllerButton),
+#[derive(Clone, PartialEq, ::prost::Message)]
-        #[prost(message, tag="11")]
+pub struct ProtoRaw {
-        ControllerTrigger(super::ProtoControllerTrigger),
+    #[prost(string, tag="1")]
-        #[prost(message, tag="12")]
+    pub data: ::prost::alloc::string::String,
-        ControllerStick(super::ProtoControllerStick),
+}
-        #[prost(message, tag="13")]
+/// ProtoClientRequestRoomStream message
-        ControllerAxis(super::ProtoControllerAxis),
+#[allow(clippy::derive_partial_eq_without_eq)]
-        #[prost(message, tag="14")]
+#[derive(Clone, PartialEq, ::prost::Message)]
-        ControllerRumble(super::ProtoControllerRumble),
+pub struct ProtoClientRequestRoomStream {
-    }
+    #[prost(string, tag="1")]
+    pub room_name: ::prost::alloc::string::String,
+    #[prost(string, tag="2")]
+    pub session_id: ::prost::alloc::string::String,
+}
+/// ProtoClientDisconnected message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoClientDisconnected {
+    #[prost(string, tag="1")]
+    pub session_id: ::prost::alloc::string::String,
+    #[prost(int32, repeated, tag="2")]
+    pub controller_slots: ::prost::alloc::vec::Vec<i32>,
+}
+/// ProtoServerPushStream message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoServerPushStream {
+    #[prost(string, tag="1")]
+    pub room_name: ::prost::alloc::string::String,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -266,10 +271,59 @@ pub struct ProtoMessageBase {
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ProtoMessageInput {
+pub struct ProtoMessage {
     #[prost(message, optional, tag="1")]
     pub message_base: ::core::option::Option<ProtoMessageBase>,
-    #[prost(message, optional, tag="2")]
+    #[prost(oneof="proto_message::Payload", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25")]
-    pub data: ::core::option::Option<ProtoInput>,
+    pub payload: ::core::option::Option<proto_message::Payload>,
+}
+/// Nested message and enum types in `ProtoMessage`.
+pub mod proto_message {
+    #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum Payload {
+        /// Input types
+        #[prost(message, tag="2")]
+        MouseMove(super::ProtoMouseMove),
+        #[prost(message, tag="3")]
+        MouseMoveAbs(super::ProtoMouseMoveAbs),
+        #[prost(message, tag="4")]
+        MouseWheel(super::ProtoMouseWheel),
+        #[prost(message, tag="5")]
+        MouseKeyDown(super::ProtoMouseKeyDown),
+        #[prost(message, tag="6")]
+        MouseKeyUp(super::ProtoMouseKeyUp),
+        #[prost(message, tag="7")]
+        KeyDown(super::ProtoKeyDown),
+        #[prost(message, tag="8")]
+        KeyUp(super::ProtoKeyUp),
+        #[prost(message, tag="9")]
+        ControllerAttach(super::ProtoControllerAttach),
+        #[prost(message, tag="10")]
+        ControllerDetach(super::ProtoControllerDetach),
+        #[prost(message, tag="11")]
+        ControllerButton(super::ProtoControllerButton),
+        #[prost(message, tag="12")]
+        ControllerTrigger(super::ProtoControllerTrigger),
+        #[prost(message, tag="13")]
+        ControllerStick(super::ProtoControllerStick),
+        #[prost(message, tag="14")]
+        ControllerAxis(super::ProtoControllerAxis),
+        #[prost(message, tag="15")]
+        ControllerRumble(super::ProtoControllerRumble),
+        /// Signaling types
+        #[prost(message, tag="20")]
+        Ice(super::ProtoIce),
+        #[prost(message, tag="21")]
+        Sdp(super::ProtoSdp),
+        #[prost(message, tag="22")]
+        Raw(super::ProtoRaw),
+        #[prost(message, tag="23")]
+        ClientRequestRoomStream(super::ProtoClientRequestRoomStream),
+        #[prost(message, tag="24")]
+        ClientDisconnected(super::ProtoClientDisconnected),
+        #[prost(message, tag="25")]
+        ServerPushStream(super::ProtoServerPushStream),
+    }
 }
 // @@protoc_insertion_point(module)

protobufs/messages.proto

@@ -12,7 +12,31 @@ message ProtoMessageBase {
   ProtoLatencyTracker latency = 2;
 }
 
-message ProtoMessageInput {
+message ProtoMessage {
   ProtoMessageBase message_base = 1;
-    ProtoInput data = 2;
+  oneof payload {
+    // Input types
+    ProtoMouseMove mouse_move = 2;
+    ProtoMouseMoveAbs mouse_move_abs = 3;
+    ProtoMouseWheel mouse_wheel = 4;
+    ProtoMouseKeyDown mouse_key_down = 5;
+    ProtoMouseKeyUp mouse_key_up = 6;
+    ProtoKeyDown key_down = 7;
+    ProtoKeyUp key_up = 8;
+    ProtoControllerAttach controller_attach = 9;
+    ProtoControllerDetach controller_detach = 10;
+    ProtoControllerButton controller_button = 11;
+    ProtoControllerTrigger controller_trigger = 12;
+    ProtoControllerStick controller_stick = 13;
+    ProtoControllerAxis controller_axis = 14;
+    ProtoControllerRumble controller_rumble = 15;
+
+    // Signaling types
+    ProtoICE ice = 20;
+    ProtoSDP sdp = 21;
+    ProtoRaw raw = 22;
+    ProtoClientRequestRoomStream client_request_room_stream = 23;
+    ProtoClientDisconnected client_disconnected = 24;
+    ProtoServerPushStream server_push_stream = 25;
+  }
 }

protobufs/types.proto

@@ -8,86 +8,79 @@ package proto;
 
 // MouseMove message
 message ProtoMouseMove {
-  string type = 1; // Fixed value "MouseMove"
+  int32 x = 1;
-  int32 x = 2;
+  int32 y = 2;
-  int32 y = 3;
 }
 
 // MouseMoveAbs message
 message ProtoMouseMoveAbs {
-  string type = 1; // Fixed value "MouseMoveAbs"
+  int32 x = 1;
-  int32 x = 2;
+  int32 y = 2;
-  int32 y = 3;
 }
 
 // MouseWheel message
 message ProtoMouseWheel {
-  string type = 1; // Fixed value "MouseWheel"
+  int32 x = 1;
-  int32 x = 2;
+  int32 y = 2;
-  int32 y = 3;
 }
 
 // MouseKeyDown message
 message ProtoMouseKeyDown {
-  string type = 1; // Fixed value "MouseKeyDown"
+  int32 key = 1;
-  int32 key = 2;
 }
 
 // MouseKeyUp message
 message ProtoMouseKeyUp {
-  string type = 1; // Fixed value "MouseKeyUp"
+  int32 key = 1;
-  int32 key = 2;
 }
 
 /* Keyboard messages */
 
 // KeyDown message
 message ProtoKeyDown {
-  string type = 1; // Fixed value "KeyDown"
+  int32 key = 1;
-  int32 key = 2;
 }
 
 // KeyUp message
 message ProtoKeyUp {
-  string type = 1; // Fixed value "KeyUp"
+  int32 key = 1;
-  int32 key = 2;
 }
 
 /* Controller messages */
 
 // ControllerAttach message
 message ProtoControllerAttach {
-  string type = 1; // Fixed value "ControllerAttach"
+  string id = 1; // One of the following enums: "ps", "xbox" or "switch"
-  string id = 2; // One of the following enums: "ps", "xbox" or "switch"
+  int32 session_slot = 2; // Session specific slot number (0-3)
-  int32 slot = 3; // Slot number (0-3)
+  string session_id = 3; // Session ID of the client
 }
 
 // ControllerDetach message
 message ProtoControllerDetach {
-  string type = 1; // Fixed value "ControllerDetach"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
 }
 
 // ControllerButton message
 message ProtoControllerButton {
-  string type = 1; // Fixed value "ControllerButtons"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
   int32 button = 3; // Button code (linux input event code)
   bool pressed = 4; // true if pressed, false if released
 }
 
 // ControllerTriggers message
 message ProtoControllerTrigger {
-  string type = 1; // Fixed value "ControllerTriggers"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
   int32 trigger = 3; // Trigger number (0 for left, 1 for right)
   int32 value = 4; // trigger value (-32768 to 32767)
 }
 
 // ControllerSticks message
 message ProtoControllerStick {
-  string type = 1; // Fixed value "ControllerStick"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
   int32 stick = 3; // Stick number (0 for left, 1 for right)
   int32 x = 4; // X axis value (-32768 to 32767)
   int32 y = 5; // Y axis value (-32768 to 32767)
@@ -95,37 +88,63 @@ message ProtoControllerStick {
 
 // ControllerAxis message
 message ProtoControllerAxis {
-  string type = 1; // Fixed value "ControllerAxis"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
   int32 axis = 3; // Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
   int32 value = 4; // axis value (-1 to 1)
 }
 
 // ControllerRumble message
 message ProtoControllerRumble {
-  string type = 1; // Fixed value "ControllerRumble"
+  int32 session_slot = 1; // Session specific slot number (0-3)
-  int32 slot = 2; // Slot number (0-3)
+  string session_id = 2; // Session ID of the client
   int32 low_frequency = 3; // Low frequency rumble (0-65535)
   int32 high_frequency = 4; // High frequency rumble (0-65535)
   int32 duration = 5; // Duration in milliseconds
 }
 
-// Union of all Input types
+/* WebRTC + signaling */
-message ProtoInput {
+
-  oneof input_type {
+message RTCIceCandidateInit {
-    ProtoMouseMove mouse_move = 1;
+  string candidate = 1;
-    ProtoMouseMoveAbs mouse_move_abs = 2;
+  optional uint32 sdpMLineIndex = 2;
-    ProtoMouseWheel mouse_wheel = 3;
+  optional string sdpMid = 3;
-    ProtoMouseKeyDown mouse_key_down = 4;
+  optional string usernameFragment = 4;
-    ProtoMouseKeyUp mouse_key_up = 5;
+}
-    ProtoKeyDown key_down = 6;
+
-    ProtoKeyUp key_up = 7;
+message RTCSessionDescriptionInit {
-    ProtoControllerAttach controller_attach = 8;
+  string sdp = 1;
-    ProtoControllerDetach controller_detach = 9;
+  string type = 2;
-    ProtoControllerButton controller_button = 10;
+}
-    ProtoControllerTrigger controller_trigger = 11;
+
-    ProtoControllerStick controller_stick = 12;
+// ProtoICE message
-    ProtoControllerAxis controller_axis = 13;
+message ProtoICE {
-    ProtoControllerRumble controller_rumble = 14;
+  RTCIceCandidateInit candidate = 1;
-  }
+}
+
+// ProtoSDP message
+message ProtoSDP {
+  RTCSessionDescriptionInit sdp = 1;
+}
+
+// ProtoRaw message
+message ProtoRaw {
+  string data = 1;
+}
+
+// ProtoClientRequestRoomStream message
+message ProtoClientRequestRoomStream {
+  string room_name = 1;
+  string session_id = 2;
+}
+
+// ProtoClientDisconnected message
+message ProtoClientDisconnected {
+  string session_id = 1;
+  repeated int32 controller_slots = 2;
+}
+
+// ProtoServerPushStream message
+message ProtoServerPushStream {
+  string room_name = 1;
 }