⭐ feat: Add protobuf (#171)

This is a second attempt to add protobuf to Nestri, after the first one
failed

---------

Co-authored-by: Philipp Neumann <3daquawolf@gmail.com>
Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>

apps/www/src/routes/(play)/play/[id]/index.tsx

@@ -18,7 +18,7 @@ export default component$(() => {
       video = document.createElement("video");
       video.id = "stream-video-player";
       video.style.visibility = "hidden";
-      const webrtc = new WebRTCStream("http://localhost:8088", id, (mediaStream) => {
+      const webrtc = new WebRTCStream("https://relay.dathorse.com", id, (mediaStream) => {
         if (video && mediaStream && (video as HTMLVideoElement).srcObject === null) {
           console.log("Setting mediastream");
           (video as HTMLVideoElement).srcObject = mediaStream;

apps/www/src/routes/watch/[id]/index.tsx

@@ -0,0 +1,303 @@
+import { useLocation } from "@builder.io/qwik-city";
+import {WebRTCStream } from "@nestri/input"
+import { component$, useSignal, useVisibleTask$ } from "@builder.io/qwik";
+
+export default component$(() => {
+    const id = useLocation().params.id;
+    const canvas = useSignal<HTMLCanvasElement>();
+
+    // eslint-disable-next-line qwik/no-use-visible-task
+    useVisibleTask$(({ track }) => {
+        track(() => canvas.value);
+
+        if (!canvas.value) return; // Ensure canvas is available
+
+        // Create video element and make it output to canvas (TODO: improve this)
+        let video = document.getElementById("webrtc-video-player");
+        if (!video) {
+            video = document.createElement("video");
+            video.id = "stream-video-player";
+            video.style.visibility = "hidden";
+            new WebRTCStream("https://relay.dathorse.com", id, (mediaStream) => {
+                if (video && mediaStream && (video as HTMLVideoElement).srcObject === null) {
+                    console.log("Setting mediastream");
+                    (video as HTMLVideoElement).srcObject = mediaStream;
+
+                    // @ts-ignore
+                    window.hasstream = true;
+                    // @ts-ignore
+                    window.roomOfflineElement?.remove();
+                    // @ts-ignore
+                    window.playbtnelement?.remove();
+
+                    const playbtn = document.createElement("button");
+                    playbtn.style.position = "absolute";
+                    playbtn.style.left = "50%";
+                    playbtn.style.top = "50%";
+                    playbtn.style.transform = "translateX(-50%) translateY(-50%)";
+                    playbtn.style.width = "12rem";
+                    playbtn.style.height = "6rem";
+                    playbtn.style.borderRadius = "1rem";
+                    playbtn.style.backgroundColor = "rgb(175, 50, 50)";
+                    playbtn.style.color = "black";
+                    playbtn.style.fontSize = "1.5em";
+                    playbtn.textContent = "< Start >";
+
+                    playbtn.onclick = () => {
+                        playbtn.remove();
+                        (video as HTMLVideoElement).play().then(() => {
+                            if (canvas.value) {
+                                canvas.value.width = (video as HTMLVideoElement).videoWidth;
+                                canvas.value.height = (video as HTMLVideoElement).videoHeight;
+
+                                const ctx = canvas.value.getContext("2d");
+                                const renderer = () => {
+                                    // @ts-ignore
+                                    if (ctx && window.hasstream) {
+                                        ctx.drawImage((video as HTMLVideoElement), 0, 0);
+                                        (video as HTMLVideoElement).requestVideoFrameCallback(renderer);
+                                    }
+                                }
+                                (video as HTMLVideoElement).requestVideoFrameCallback(renderer);
+                            }
+                        });
+                    };
+                    document.body.append(playbtn);
+                    // @ts-ignore
+                    window.playbtnelement = playbtn;
+                } else if (mediaStream === null) {
+                    console.log("MediaStream is null, Room is offline");
+                    // @ts-ignore
+                    window.playbtnelement?.remove();
+                    // @ts-ignore
+                    window.roomOfflineElement?.remove();
+                    // Add a message to the screen
+                    const offline = document.createElement("div");
+                    offline.style.position = "absolute";
+                    offline.style.left = "50%";
+                    offline.style.top = "50%";
+                    offline.style.transform = "translateX(-50%) translateY(-50%)";
+                    offline.style.width = "auto";
+                    offline.style.height = "auto";
+                    offline.style.color = "lightgray";
+                    offline.style.fontSize = "2em";
+                    offline.textContent = "Offline";
+                    document.body.append(offline);
+                    // @ts-ignore
+                    window.roomOfflineElement = offline;
+                    // @ts-ignore
+                    window.hasstream = false;
+                    // Clear canvas if it has been set
+                    if (canvas.value) {
+                        const ctx = canvas.value.getContext("2d");
+                        if (ctx) ctx.clearRect(0, 0, canvas.value.width, canvas.value.height);
+                    }
+                } else if ((video as HTMLVideoElement).srcObject !== null) {
+                    console.log("Setting new mediastream");
+                    (video as HTMLVideoElement).srcObject = mediaStream;
+                    // @ts-ignore
+                    window.hasstream = true;
+                    // Start video rendering
+                    (video as HTMLVideoElement).play().then(() => {
+                        // @ts-ignore
+                        window.roomOfflineElement?.remove();
+                        if (canvas.value) {
+                            canvas.value.width = (video as HTMLVideoElement).videoWidth;
+                            canvas.value.height = (video as HTMLVideoElement).videoHeight;
+
+                            const ctx = canvas.value.getContext("2d");
+                            const renderer = () => {
+                                // @ts-ignore
+                                if (ctx && window.hasstream) {
+                                    ctx.drawImage((video as HTMLVideoElement), 0, 0);
+                                    (video as HTMLVideoElement).requestVideoFrameCallback(renderer);
+                                }
+                            }
+                            (video as HTMLVideoElement).requestVideoFrameCallback(renderer);
+                        }
+                    });
+                }
+            });
+        }
+    })
+
+    return (
+        <canvas
+            ref={canvas}
+            onClick$={async () => {
+                // @ts-ignore
+                if (canvas.value && window.hasstream && !window.nestriLock) {
+                    await canvas.value.requestFullscreen()
+                    if (document.fullscreenElement !== null) {
+                        // @ts-ignore
+                        if ('keyboard' in window.navigator && 'lock' in window.navigator.keyboard) {
+                            const keys = [
+                                "AltLeft",
+                                "AltRight",
+                                "Tab",
+                                "Escape",
+                                "ContextMenu",
+                                "MetaLeft",
+                                "MetaRight"
+                            ];
+                            console.log("requesting keyboard lock");
+                            // @ts-ignore
+                            window.navigator.keyboard.lock(keys).then(
+                                () => {
+                                    console.log("keyboard lock success");
+                                    // @ts-ignore
+                                    window.nestriLock = true;
+                                }
+                            ).catch(
+                                (e: any) => {
+                                    console.log("keyboard lock failed: ", e);
+                                    // @ts-ignore
+                                    window.nestriLock = false;
+                                }
+                            )
+                        } else {
+                            console.log("keyboard lock not supported, navigator is: ", window.navigator, navigator);
+                            // @ts-ignore
+                            window.nestriLock = undefined;
+                        }
+                    }
+                }
+            }}
+            //TODO: go full screen, then lock on "landscape" screen-orientation on mobile
+            class="aspect-video h-full w-full object-contain max-h-screen" />
+    )
+})
+
+{/**
+ .spinningCircleInner_b6db20 {
+ transform: rotate(280deg);
+ }
+ .inner_b6db20 {
+ position: relative;
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ width: 32px;
+ height: 32px;
+ contain: paint;
+ } */
+}
+
+{/* <div class="loadingPopout_a8c724" role="dialog" tabindex="-1" aria-modal="true"><div class="spinner_b6db20 spinningCircle_b6db20" role="img" aria-label="Loading"><div class="spinningCircleInner_b6db20 inner_b6db20"><svg class="circular_b6db20" viewBox="25 25 50 50"><circle class="path_b6db20 path3_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20 path2_b6db20" cx="50" cy="50" r="20"></circle><circle class="path_b6db20" cx="50" cy="50" r="20"></circle></svg></div></div></div> */
+}
+// .loadingPopout_a8c724 {
+//     background-color: var(--background-secondary);
+//     display: flex;
+//     justify-content: center;
+//     padding: 8px;
+// }
+
+// .circular_b6db20 {
+//     animation: spinner-spinning-circle-rotate_b6db20 2s linear infinite;
+//     height: 100%;
+//     width: 100%;
+// }
+
+// 100% {
+//     transform: rotate(360deg);
+// }
+
+
+{/* .path3_b6db20 {
+    animation-delay: .23s;
+    stroke: var(--text-brand);
+}
+.path_b6db20 {
+    animation: spinner-spinning-circle-dash_b6db20 2s ease-in-out infinite;
+    stroke-dasharray: 1, 200;
+    stroke-dashoffset: 0;
+    fill: none;
+    stroke-width: 6;
+    stroke-miterlimit: 10;
+    stroke-linecap: round;
+    stroke: var(--brand-500);
+}
+circle[Attributes Style] {
+    cx: 50;
+    cy: 50;
+    r: 20;
+}
+user agent stylesheet
+:not(svg) {
+    transform-origin: 0px 0px;
+} */
+}
+
+
+// .path2_b6db20 {
+//     animation-delay: .15s;
+//     stroke: var(--text-brand);
+//     opacity: .6;
+// }
+// .path_b6db20 {
+//     animation: spinner-spinning-circle-dash_b6db20 2s ease-in-out infinite;
+//     stroke-dasharray: 1, 200;
+//     stroke-dashoffset: 0;
+//     fill: none;
+//     stroke-width: 6;
+//     stroke-miterlimit: 10;
+//     stroke-linecap: round;
+//     stroke: var(--brand-500);
+// }
+// circle[Attributes Style] {
+//     cx: 50;
+//     cy: 50;
+//     r: 20;
+
+
+// function throttle(func, limit) {
+//     let inThrottle;
+//     return function(...args) {
+//         if (!inThrottle) {
+//             func.apply(this, args);
+//             inThrottle = true;
+//             setTimeout(() => inThrottle = false, limit);
+//         }
+//     }
+// }
+
+// // Use it like this:
+// const throttledMouseMove = throttle((x, y) => {
+//     websocket.send(JSON.stringify({
+//         type: 'mousemove',
+//         x: x,
+//         y: y
+//     }));
+// }, 16); // ~60fps
+
+// use std::time::Instant;
+
+// // Add these to your AppState
+// struct AppState {
+//     pipeline: Arc<Mutex<gst::Pipeline>>,
+//     last_mouse_move: Arc<Mutex<(i32, i32, Instant)>>, // Add this
+// }
+
+// // Then in your MouseMove handler:
+// InputMessage::MouseMove { x, y } => {
+//     let mut last_move = state.last_mouse_move.lock().unwrap();
+//     let now = Instant::now();
+
+//     // Only process if coordinates are different or enough time has passed
+//     if (last_move.0 != x || last_move.1 != y) && 
+//        (now.duration_since(last_move.2).as_millis() > 16) { // ~60fps
+
+//         println!("Mouse moved to x: {}, y: {}", x, y);
+
+//         let structure = gst::Structure::builder("MouseMoveRelative")
+//             .field("pointer_x", x as f64)
+//             .field("pointer_y", y as f64)
+//             .build();
+
+//         let event = gst::event::CustomUpstream::new(structure);
+//         pipeline.send_event(event);
+
+//         // Update last position and time
+//         *last_move = (x, y, now);
+//     }
+// }

buf.gen.yaml

@@ -0,0 +1,17 @@
+version: v2
+inputs:
+  - directory: protobufs
+plugins:
+  # TypeScript (frontend)
+  - remote: buf.build/bufbuild/es
+    out: packages/input/src/proto
+    opt: target=ts
+
+  # Golang (relay)
+  - remote: buf.build/protocolbuffers/go
+    out: packages/relay/internal/proto
+    opt: paths=source_relative
+
+  # Rust (nestri-server)
+  - remote: buf.build/community/neoeinstein-prost
+    out: packages/server/src/proto

containers/master.Containerfile

@@ -1,206 +0,0 @@
-#! Runs the docker server that handles everything else
-#******************************************************************************
-#                                                                          base
-#******************************************************************************
-FROM archlinux:base-20241027.0.273886 AS base
-# How to run -  docker run -it --rm --device /dev/dri nestri /bin/bash - DO NOT forget the ports
-# TODO: Migrate XDG_RUNTIME_DIR to /run/user/1000
-# TODO: Add nestri-server to pulseaudio.conf
-# TODO: Add our own entrypoint, with our very own zombie ripper 🧟🏾‍♀️
-# FIXME: Add user root to `pulse-access` group as well :D
-# TODO: Test the whole damn thing
-
-# Update the pacman repo
-RUN \
-    pacman -Syu --noconfirm
-
-#******************************************************************************
-#                                                                       builder
-#******************************************************************************
-
-FROM base AS builder
-
-RUN \
-    pacman -Su --noconfirm \
-    base-devel \
-    git \
-    sudo \
-    vim
-
-WORKDIR /scratch
-
-# Allow nobody user to invoke pacman to install packages (as part of makepkg) and modify the system.
-# This should never exist in a running image, just used by *-build Docker stages.
-RUN \
-  echo "nobody ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers;
-
-ENV ARTIFACTS=/artifacts \
-    CARGO_TARGET_DIR=/build
-
-RUN \
-  mkdir -p /artifacts \
-  && mkdir -p /build
-  
-RUN \
-  chgrp nobody /scratch /artifacts /build \
-  && chmod g+ws /scratch /artifacts /build
-
-#******************************************************************************
-#                                                                  rust-builder
-#******************************************************************************
-
-FROM builder AS rust-builder
-
-RUN \
-    pacman -Su --noconfirm \
-    rustup
-
-RUN \
-    rustup default stable
-
-#******************************************************************************
-#                                                         nestri-server-builder
-#******************************************************************************
-# Builds nestri server binary
-FROM rust-builder AS nestri-server-builder
-
-RUN \
-    pacman -Su --noconfirm \
-    wayland \
-	vpl-gpu-rt \
-	gstreamer \
-	gst-plugin-va \
-	gst-plugins-base \
-	gst-plugins-good \
-	mesa-utils \
-	weston \
-	xorg-xwayland
-
-
-#******************************************************************************
-#                                                          nestri-server-build
-#******************************************************************************
-
-FROM nestri-server-builder AS nestri-server-build
-
-#Allow makepkg to be run as nobody.
-RUN chgrp -R nobody /scratch && chmod -R g+ws /scratch
-
-# USER nobody
-
-# Perform the server build.
-WORKDIR /scratch/server
-
-RUN \
-    git clone https://github.com/nestriness/nestri
-
-WORKDIR /scratch/server/nestri
-
-RUN \
-    git checkout feat/stream \
-    && cargo build -j$(nproc) --release
-
-# COPY packages/server/build/ /scratch/server/
-
-# RUN makepkg && cp *.zst "$ARTIFACTS"
-#******************************************************************************
-#                                                             runtime_base_pkgs
-#******************************************************************************
-
-FROM base AS runtime_base_pkgs
-
-COPY --from=nestri-server-build /build/release/nestri-server /usr/bin/
-
-#******************************************************************************
-#                                                                  runtime_base
-#******************************************************************************
-
-FROM runtime_base_pkgs AS runtime_base
-
-RUN \
-    pacman -Su --noconfirm \
-    weston \
-    sudo \
-    xorg-xwayland \
-    gstreamer \
-    gst-plugins-base \
-    gst-plugins-good \
-    gst-plugin-qsv \
-    gst-plugin-va \
-    gst-plugin-fmp4 \
-    mesa \
-    # Grab GPU encoding packages
-    # Intel (modern VPL + VA-API)
-    vpl-gpu-rt \
-    intel-media-driver \
-    # AMD/ATI (VA-API)
-    libva-mesa-driver \
-    # NVIDIA (proprietary)
-    nvidia-utils \
-    # Audio
-    pulseaudio \
-    # Supervisor
-    supervisor
-    
-RUN \
-    # Set up our non-root user $(nestri)
-    groupadd -g 1000 nestri \
-    && useradd -ms /bin/bash nestri -u 1000 -g 1000  \
-    && passwd -d nestri \
-    # Setup Pulseaudio
-    && useradd -d /var/run/pulse -s /usr/bin/nologin -G audio pulse \
-    && groupadd pulse-access \
-    && usermod -aG audio,input,render,video,pulse-access nestri \
-    && echo "nestri ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \
-    && echo "Users created" \
-    # Create an empty machine-id file
-    && touch /etc/machine-id
-
-ENV \
-    XDG_RUNTIME_DIR=/tmp
-
-#******************************************************************************
-#                                                                       runtime
-#******************************************************************************
-
-FROM runtime_base AS runtime
-# Setup supervisor #
-RUN <<-EOF
-    echo -e "
-        [supervisord]
-        user=root
-        nodaemon=true
-        loglevel=info
-        logfile=/tmp/supervisord.log
-        pidfile=/tmp/supervisord.pid
-
-        [program:dbus]
-        user=root
-        command=dbus-daemon --system --nofork
-        logfile=/tmp/dbus.log
-        pidfile=/tmp/dbus.pid
-        stopsignal=INT
-        autostart=true
-        autorestart=true
-        priority=1
-
-        [program:pulseaudio]
-        user=root
-        command=pulseaudio --daemonize=no --system --disallow-module-loading --disallow-exit --exit-idle-time=-1
-        logfile=/tmp/pulseaudio.log
-        pidfile=/tmp/pulseaudio.pid
-        stopsignal=INT
-        autostart=true
-        autorestart=true
-        priority=10
-    " | tee /etc/supervisord.conf
-EOF
-
-RUN \
-    chown -R nestri:nestri /tmp /etc/supervisord.conf
-
-ENV USER=nestri
-USER 1000
-
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
-# Debug - pactl list

key_ssh

@@ -0,0 +1,53 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdz
+c2gtcnNhAAAAAwEAAQAAAgEAukv1aFzn29JFNplJKQnusTysRWULax8A0u9mzM1W
+p1a07W+RfIvHja5AJRyzEOO8tqUdaSe3jsU/lBMax70d7DPRHOHhi782tSdGIsjf
+NP+0+Rf//W4wtCEqiGsDCrI+mLYRDdD6yEbKGl6N5rNS7nFYOD5xNbrnUZkzU0Q7
+XgTl4VUzPG0C3YFYVxD1Womgn8+nVP0PB21rw0OSg5fkqp6mBC0whPGWW5a75uNS
+pvuy610Etj9fs2d3+1BtlNCUFO30o+FDFbmkovvZAxfZqy8AT2E+Jpxib3LlGQmi
+TSs1EgVsYoiuG64sOw9JPat4e9LHGcddgcB7ZE/9CGuLaRcrTLS22/tojl84LkC1
+0byzRThT+Gmf4NSD1Yj7c559KCSutrul8mjP1QWxpYcxYiXGCIFb/kDB8TYX4irV
+HudeiGgnflSlREOZn8GUa4XNRSVYSV1bm4LpYcWGHH5sHsuK/21OuT/ZXRsV97fS
+MTi3D42GWvtUYEVvyhS2RiHvRq/WabZhpP+WxJmIMfCJRQUwzKqh7vqqGepbgZDY
+iIDGcHzTAVnBbygXUGvVYSoaWqaog6uDu5Voyh31lUPjXDjW0J0Mkliw/GTDHwte
+2FiEbMC2rMwm9/bqeDfaQ5PY8vSaXPKZasTfgCA/bU+6imWfjkA40gPwUNemXJFa
+GhkAAAc4zX7cDs1+3A4AAAAHc3NoLXJzYQAAAgEAukv1aFzn29JFNplJKQnusTys
+RWULax8A0u9mzM1Wp1a07W+RfIvHja5AJRyzEOO8tqUdaSe3jsU/lBMax70d7DPR
+HOHhi782tSdGIsjfNP+0+Rf//W4wtCEqiGsDCrI+mLYRDdD6yEbKGl6N5rNS7nFY
+OD5xNbrnUZkzU0Q7XgTl4VUzPG0C3YFYVxD1Womgn8+nVP0PB21rw0OSg5fkqp6m
+BC0whPGWW5a75uNSpvuy610Etj9fs2d3+1BtlNCUFO30o+FDFbmkovvZAxfZqy8A
+T2E+Jpxib3LlGQmiTSs1EgVsYoiuG64sOw9JPat4e9LHGcddgcB7ZE/9CGuLaRcr
+TLS22/tojl84LkC10byzRThT+Gmf4NSD1Yj7c559KCSutrul8mjP1QWxpYcxYiXG
+CIFb/kDB8TYX4irVHudeiGgnflSlREOZn8GUa4XNRSVYSV1bm4LpYcWGHH5sHsuK
+/21OuT/ZXRsV97fSMTi3D42GWvtUYEVvyhS2RiHvRq/WabZhpP+WxJmIMfCJRQUw
+zKqh7vqqGepbgZDYiIDGcHzTAVnBbygXUGvVYSoaWqaog6uDu5Voyh31lUPjXDjW
+0J0Mkliw/GTDHwte2FiEbMC2rMwm9/bqeDfaQ5PY8vSaXPKZasTfgCA/bU+6imWf
+jkA40gPwUNemXJFaGhkAAAADAQABAAACABU3oXOJYT6/6/IT8ykhYHX/Gv+sSmig
+108uxN5+ZGgaj/CazE8K/Gc+eT/AzQQXa97BX2Aijd1MybjrlBk7iLTmEeLgNvfb
+hzeKiYpcpk868DmOajFK43Egdd+jkG0fb9GkqryQGH/G0C3ApxinQ8K0Vf/R3MLp
+zn6UnCvXkYtbs3K+aV9wausfWBhSbh8pnYmV9NDRIW0PIT7T61PhcMGd3/qJywTZ
+Hu6/dBcPvNIXyIDVzzuSbCvalOeieViBiNv5XetsNF+9DOsjE8wuYzQD3l0sDhAH
+r45R35XCayGu0hSDWZy/RFUSuSbgzs6fQNGZZ+OfuR6V9XevRHo5BV94108cOezM
+CAqOm7hVbI/BihH22T7urbwhqj06u1ISKo3wd3bMAtKpvvekVHwMoeqRXuBIuHwI
+h/EQ/fcqyB7s51z54+Jr73zl7wljLQtAAiHncaWKKQwFbeB/rhU6YBrr/H7KsunS
+zCV+qabomUBb1iSNg0KBEtNiyRiLCEuI9/eTqy4t4G9222y0W55rjMse3grC8fK7
+oEDlR1YhzcBaeNFkCn2AGJ/B/x6TSFwuw3S1tKlo7wQenl8AH5QcuzssOh+GfCP+
+5NVn54+XOqS2Zm0n8xwCBuJOgWpHu1ZXXEkS8ZB6Nd8gdqGAloNbAIuq6dUuAqKJ
+4sxXmvr0oqghAAABAQDUtCStuzsUee6Lwb3I90RP3tJQ7g7f/rWxHma9PtOsqhfY
+hiJLncNfURM/kasd8TgERKLv9gAQmJ4gL/ray6Bwkk1ft6wkKw2gjA8Sh1VbPxvj
+13aqqJhhr7bES2hStiXpwDX10rfNeZ24PuTq5Ow8efZehqWj2JskbHWnL6Di1x6+
+p1gRSODYRlepGTi+6QZDA8NOJOYWndzN+4kwzExlserq4sBXVc3WLk9Cb52IvjUw
+DXtrKvongTOz5W/L6Q4iXwKltjqxMx9JO0nZbrcywoQ8wluoecDlvTMl8mHn1vbE
+3XR59O6XBpitaFM1Szdqa2u8u2apawf2D60j9pskAAABAQDnqJ7BIj9X6l0cicRv
+k2wG/v8y+CC8RAxBVDtGyEbn2xmXCgJjenO3qKvAjIWbcK49yV1mZOW2d+BL3Pnt
+lMk1KQCM1Ln2DK3LkI+8cvXaxFaqIzmaS6UmhPuKO5JIxf3uOIGK0xLV208tVbPx
+YL5Vuy9lELoG3cGAzElHQtScBiFnAfaYyoWsEO/m6kuBM0VKOuI93QcxJk9bqxpq
+BaD/duM25nLxEaGGkRq4pfUzZaMbs73ROGNCKfaPfj6HmEQsS3EFgnNz5DrD+ziZ
+YWSYwxZbpnA5IQ5ZgGAbKHYLi2fxSo5phl16xbLdH7JCRo5a61TJ6fzjEugWVxCV
+s+s3AAABAQDN3yYCothAAAY6gkYbmU1a0g+pVIKe7kFAYMFTMV0XVPOPYPT60uoX
+/a6B6qr+wUdpclJ+RAjdkM1icYELnL5Wn6cX7r5W2QoSpyGBNLhxx0nMB1fLV6OM
+nl2I/YtKl/P/6iFx7V07arNJJms1kCWcPomaD46yba/coXfhc8RX9/GTXiB9ZdMS
+ZQ6ZowSlR4CQqmeCju50oLiIXstDOf+kqoPHBpHjIE46Hr9bsX95mW8QzvAwtegu
+8PurDQHNmh6nHX3A2IJkFd6M2tmXOsdmeQ1246aVqvFjHhAdmXLkrhx2dmCnCfON
+81IXONa2nTAOW1IpfKLC17U5yZWH8O0vAAAAAAEC
+-----END OPENSSH PRIVATE KEY-----

packages/input/package.json

@@ -5,5 +5,12 @@
     "sideEffects": false,
     "exports": {
         ".": "./src/index.ts"
+    },
+    "devDependencies": {
+        "@bufbuild/buf": "^1.50.0",
+        "@bufbuild/protoc-gen-es": "^2.2.3"
+    },
+    "dependencies": {
+        "@bufbuild/protobuf": "^2.2.3"
     }
 }

packages/input/src/keyboard.ts

@@ -1,8 +1,17 @@
-import {type Input} from "./types"
 import {keyCodeToLinuxEventCode} from "./codes"
-import {MessageInput, encodeMessage} from "./messages";
 import {WebRTCStream} from "./webrtc-stream";
 import {LatencyTracker} from "./latency";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput,
+  ProtoInputSchema,
+  ProtoKeyDownSchema,
+  ProtoKeyUpSchema,
+  ProtoMouseMoveSchema
+} from "./proto/types_pb";
+import {create, toBinary} from "@bufbuild/protobuf";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -15,19 +24,31 @@ export class Keyboard {
   protected connected!: boolean;
 
   // Store references to event listeners
-  private keydownListener: (e: KeyboardEvent) => void;
+  private readonly keydownListener: (e: KeyboardEvent) => void;
-  private keyupListener: (e: KeyboardEvent) => void;
+  private readonly keyupListener: (e: KeyboardEvent) => void;
 
   constructor({webrtc, canvas}: Props) {
     this.wrtc = webrtc;
     this.canvas = canvas;
-    this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
+    this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyDown",
+        value: create(ProtoKeyDownSchema, {
           type: "KeyDown",
           key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
     }));
-    this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
+    this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyUp",
+        value: create(ProtoKeyUpSchema, {
           type: "KeyUp",
           key: this.keyToVirtualKeyCode(e.code)
+        }),
+      }
     }));
     this.run()
   }
@@ -59,7 +80,7 @@ export class Keyboard {
   }
 
   // Helper function to create and return mouse listeners
-  private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+  private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
@@ -67,18 +88,34 @@ export class Keyboard {
       if ((e as any).repeat)
         return;
 
-      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
+      const data = dataCreator(e as any);
-      const dataString = JSON.stringify({...data, type} as Input);
 
       // Latency tracking
       const tracker = new LatencyTracker("input-keyboard");
       tracker.addTimestamp("client_send");
-      const message: MessageInput = {
+      const protoTracker: ProtoLatencyTracker = {
-        payload_type: "input",
+        $typeName: "proto.ProtoLatencyTracker",
-        data: dataString,
+        sequenceId: tracker.sequence_id,
-        latency: tracker,
+        timestamps: [],
       };
-      this.wrtc.sendBinary(encodeMessage(message));
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 

packages/input/src/latency.ts

@@ -6,12 +6,10 @@ type TimestampEntry = {
 export class LatencyTracker {
   sequence_id: string;
   timestamps: TimestampEntry[];
-  metadata?: Record<string, any>;
 
-  constructor(sequence_id: string, timestamps: TimestampEntry[] = [], metadata: Record<string, any> = {}) {
+  constructor(sequence_id: string, timestamps: TimestampEntry[] = []) {
     this.sequence_id = sequence_id;
     this.timestamps = timestamps;
-    this.metadata = metadata;
   }
 
   addTimestamp(stage: string): void {
@@ -40,7 +38,6 @@ export class LatencyTracker {
         // Fill nanoseconds with zeros to match the expected format
         time: entry.time.toISOString().replace(/\.(\d+)Z$/, ".$1000000Z"),
       })),
-      metadata: this.metadata,
     };
   }
 
@@ -49,6 +46,6 @@ export class LatencyTracker {
       stage: ts.stage,
       time: new Date(ts.time),
     }));
-    return new LatencyTracker(json.sequence_id, timestamps, json.metadata);
+    return new LatencyTracker(json.sequence_id, timestamps);
   }
 }

packages/input/src/messages.ts

@@ -1,13 +1,7 @@
-import {gzip, ungzip} from "pako";
 import {LatencyTracker} from "./latency";
 
 export interface MessageBase {
   payload_type: string;
-}
-
-export interface MessageInput extends MessageBase {
-  payload_type: "input";
-  data: string;
   latency?: LatencyTracker;
 }
 
@@ -41,33 +35,3 @@ export interface MessageAnswer extends MessageBase {
   payload_type: "answer";
   answer_type: AnswerType;
 }
-
-function blobToUint8Array(blob: Blob): Promise<Uint8Array> {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-    reader.onloadend = () => {
-      const arrayBuffer = reader.result as ArrayBuffer;
-      resolve(new Uint8Array(arrayBuffer));
-    };
-    reader.onerror = reject;
-    reader.readAsArrayBuffer(blob);
-  });
-}
-
-export function encodeMessage<T>(message: T): Uint8Array {
-  // Convert the message to JSON string
-  const json = JSON.stringify(message);
-  // Compress the JSON string using gzip
-  return gzip(json);
-}
-
-export async function decodeMessage<T>(data: Blob): Promise<T> {
-  // Convert the Blob to Uint8Array
-  const array = await blobToUint8Array(data);
-  // Decompress the gzip data
-  const decompressed = ungzip(array);
-  // Convert the Uint8Array to JSON string
-  const json = new TextDecoder().decode(decompressed);
-  // Parse the JSON string
-  return JSON.parse(json);
-}

packages/input/src/mouse.ts

@@ -1,8 +1,18 @@
-import {type Input} from "./types"
-import {mouseButtonToLinuxEventCode} from "./codes"
-import {MessageInput, encodeMessage} from "./messages";
 import {WebRTCStream} from "./webrtc-stream";
 import {LatencyTracker} from "./latency";
+import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput, ProtoInputSchema,
+  ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
+  ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
+  ProtoMouseMove,
+  ProtoMouseMoveSchema,
+  ProtoMouseWheel, ProtoMouseWheelSchema
+} from "./proto/types_pb";
+import {mouseButtonToLinuxEventCode} from "./codes";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {create, toBinary} from "@bufbuild/protobuf";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -15,33 +25,56 @@ export class Mouse {
   protected connected!: boolean;
 
   // Store references to event listeners
-  private mousemoveListener: (e: MouseEvent) => void;
+  private readonly mousemoveListener: (e: MouseEvent) => void;
-  private mousedownListener: (e: MouseEvent) => void;
+  private readonly mousedownListener: (e: MouseEvent) => void;
-  private mouseupListener: (e: MouseEvent) => void;
+  private readonly mouseupListener: (e: MouseEvent) => void;
-  private mousewheelListener: (e: WheelEvent) => void;
+  private readonly mousewheelListener: (e: WheelEvent) => void;
 
   constructor({webrtc, canvas}: Props) {
     this.wrtc = webrtc;
     this.canvas = canvas;
 
-    this.mousemoveListener = this.createMouseListener("mousemove", (e: any) => ({
+    this.mousemoveListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseMove",
+        value: create(ProtoMouseMoveSchema, {
           type: "MouseMove",
           x: e.movementX,
           y: e.movementY
+        }),
+      }
     }));
-    this.mousedownListener = this.createMouseListener("mousedown", (e: any) => ({
+    this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyDown",
+        value: create(ProtoMouseKeyDownSchema, {
           type: "MouseKeyDown",
           key: this.keyToVirtualKeyCode(e.button)
+        }),
+      }
     }));
-
+    this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
-    this.mouseupListener = this.createMouseListener("mouseup", (e: any) => ({
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyUp",
+        value: create(ProtoMouseKeyUpSchema, {
           type: "MouseKeyUp",
           key: this.keyToVirtualKeyCode(e.button)
+        }),
+      }
     }));
-    this.mousewheelListener = this.createMouseListener("wheel", (e: any) => ({
+    this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseWheel",
+        value: create(ProtoMouseWheelSchema, {
           type: "MouseWheel",
           x: e.deltaX,
           y: e.deltaY
+        }),
+      }
     }));
 
     this.run()
@@ -59,10 +92,10 @@ export class Mouse {
 
     if (document.pointerLockElement == this.canvas) {
       this.connected = true
-      this.canvas.addEventListener("mousemove", this.mousemoveListener, { passive: false });
+      this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
-      this.canvas.addEventListener("mousedown", this.mousedownListener, { passive: false });
+      this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
-      this.canvas.addEventListener("mouseup", this.mouseupListener, { passive: false });
+      this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
-      this.canvas.addEventListener("wheel", this.mousewheelListener, { passive: false });
+      this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
 
     } else {
       if (this.connected) {
@@ -81,22 +114,38 @@ export class Mouse {
   }
 
   // Helper function to create and return mouse listeners
-  private createMouseListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+  private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
-      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
+      const data = dataCreator(e as any);
-      const dataString = JSON.stringify({...data, type} as Input);
 
       // Latency tracking
       const tracker = new LatencyTracker("input-mouse");
       tracker.addTimestamp("client_send");
-      const message: MessageInput = {
+      const protoTracker: ProtoLatencyTracker = {
-        payload_type: "input",
+        $typeName: "proto.ProtoLatencyTracker",
-        data: dataString,
+        sequenceId: tracker.sequence_id,
-        latency: tracker,
+        timestamps: [],
       };
-      this.wrtc.sendBinary(encodeMessage(message));
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 

packages/input/src/proto/latency_tracker_pb.ts

@@ -0,0 +1,60 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file latency_tracker.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { Timestamp } from "@bufbuild/protobuf/wkt";
+import { file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file latency_tracker.proto.
+ */
+export const file_latency_tracker: GenFile = /*@__PURE__*/
+  fileDesc("ChVsYXRlbmN5X3RyYWNrZXIucHJvdG8SBXByb3RvIk4KE1Byb3RvVGltZXN0YW1wRW50cnkSDQoFc3RhZ2UYASABKAkSKAoEdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiWgoTUHJvdG9MYXRlbmN5VHJhY2tlchITCgtzZXF1ZW5jZV9pZBgBIAEoCRIuCgp0aW1lc3RhbXBzGAIgAygLMhoucHJvdG8uUHJvdG9UaW1lc3RhbXBFbnRyeUIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_google_protobuf_timestamp]);
+
+/**
+ * @generated from message proto.ProtoTimestampEntry
+ */
+export type ProtoTimestampEntry = Message<"proto.ProtoTimestampEntry"> & {
+  /**
+   * @generated from field: string stage = 1;
+   */
+  stage: string;
+
+  /**
+   * @generated from field: google.protobuf.Timestamp time = 2;
+   */
+  time?: Timestamp;
+};
+
+/**
+ * Describes the message proto.ProtoTimestampEntry.
+ * Use `create(ProtoTimestampEntrySchema)` to create a new message.
+ */
+export const ProtoTimestampEntrySchema: GenMessage<ProtoTimestampEntry> = /*@__PURE__*/
+  messageDesc(file_latency_tracker, 0);
+
+/**
+ * @generated from message proto.ProtoLatencyTracker
+ */
+export type ProtoLatencyTracker = Message<"proto.ProtoLatencyTracker"> & {
+  /**
+   * @generated from field: string sequence_id = 1;
+   */
+  sequenceId: string;
+
+  /**
+   * @generated from field: repeated proto.ProtoTimestampEntry timestamps = 2;
+   */
+  timestamps: ProtoTimestampEntry[];
+};
+
+/**
+ * Describes the message proto.ProtoLatencyTracker.
+ * Use `create(ProtoLatencyTrackerSchema)` to create a new message.
+ */
+export const ProtoLatencyTrackerSchema: GenMessage<ProtoLatencyTracker> = /*@__PURE__*/
+  messageDesc(file_latency_tracker, 1);
+

packages/input/src/proto/messages_pb.ts

@@ -0,0 +1,62 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file messages.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { ProtoInput } from "./types_pb";
+import { file_types } from "./types_pb";
+import type { ProtoLatencyTracker } from "./latency_tracker_pb";
+import { file_latency_tracker } from "./latency_tracker_pb";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file messages.proto.
+ */
+export const file_messages: GenFile = /*@__PURE__*/
+  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
+
+/**
+ * @generated from message proto.ProtoMessageBase
+ */
+export type ProtoMessageBase = Message<"proto.ProtoMessageBase"> & {
+  /**
+   * @generated from field: string payload_type = 1;
+   */
+  payloadType: string;
+
+  /**
+   * @generated from field: proto.ProtoLatencyTracker latency = 2;
+   */
+  latency?: ProtoLatencyTracker;
+};
+
+/**
+ * Describes the message proto.ProtoMessageBase.
+ * Use `create(ProtoMessageBaseSchema)` to create a new message.
+ */
+export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*/
+  messageDesc(file_messages, 0);
+
+/**
+ * @generated from message proto.ProtoMessageInput
+ */
+export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
+  /**
+   * @generated from field: proto.ProtoMessageBase message_base = 1;
+   */
+  messageBase?: ProtoMessageBase;
+
+  /**
+   * @generated from field: proto.ProtoInput data = 2;
+   */
+  data?: ProtoInput;
+};
+
+/**
+ * Describes the message proto.ProtoMessageInput.
+ * Use `create(ProtoMessageInputSchema)` to create a new message.
+ */
+export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
+  messageDesc(file_messages, 1);
+

packages/input/src/proto/types_pb.ts

@@ -0,0 +1,272 @@
+// @generated by protoc-gen-es v2.2.3 with parameter "target=ts"
+// @generated from file types.proto (package proto, syntax proto3)
+/* eslint-disable */
+
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv1";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv1";
+import type { Message } from "@bufbuild/protobuf";
+
+/**
+ * Describes the file types.proto.
+ */
+export const file_types: GenFile = /*@__PURE__*/
+  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSLcAgoKUHJvdG9JbnB1dBIrCgptb3VzZV9tb3ZlGAEgASgLMhUucHJvdG8uUHJvdG9Nb3VzZU1vdmVIABIyCg5tb3VzZV9tb3ZlX2FicxgCIAEoCzIYLnByb3RvLlByb3RvTW91c2VNb3ZlQWJzSAASLQoLbW91c2Vfd2hlZWwYAyABKAsyFi5wcm90by5Qcm90b01vdXNlV2hlZWxIABIyCg5tb3VzZV9rZXlfZG93bhgEIAEoCzIYLnByb3RvLlByb3RvTW91c2VLZXlEb3duSAASLgoMbW91c2Vfa2V5X3VwGAUgASgLMhYucHJvdG8uUHJvdG9Nb3VzZUtleVVwSAASJwoIa2V5X2Rvd24YBiABKAsyEy5wcm90by5Qcm90b0tleURvd25IABIjCgZrZXlfdXAYByABKAsyES5wcm90by5Qcm90b0tleVVwSABCDAoKaW5wdXRfdHlwZUIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z");
+
+/**
+ * MouseMove message
+ *
+ * @generated from message proto.ProtoMouseMove
+ */
+export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
+  /**
+   * Fixed value "MouseMove"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseMove.
+ * Use `create(ProtoMouseMoveSchema)` to create a new message.
+ */
+export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
+  messageDesc(file_types, 0);
+
+/**
+ * MouseMoveAbs message
+ *
+ * @generated from message proto.ProtoMouseMoveAbs
+ */
+export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
+  /**
+   * Fixed value "MouseMoveAbs"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseMoveAbs.
+ * Use `create(ProtoMouseMoveAbsSchema)` to create a new message.
+ */
+export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE__*/
+  messageDesc(file_types, 1);
+
+/**
+ * MouseWheel message
+ *
+ * @generated from message proto.ProtoMouseWheel
+ */
+export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
+  /**
+   * Fixed value "MouseWheel"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
+   */
+  x: number;
+
+  /**
+   * @generated from field: int32 y = 3;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseWheel.
+ * Use `create(ProtoMouseWheelSchema)` to create a new message.
+ */
+export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
+  messageDesc(file_types, 2);
+
+/**
+ * MouseKeyDown message
+ *
+ * @generated from message proto.ProtoMouseKeyDown
+ */
+export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
+  /**
+   * Fixed value "MouseKeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseKeyDown.
+ * Use `create(ProtoMouseKeyDownSchema)` to create a new message.
+ */
+export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE__*/
+  messageDesc(file_types, 3);
+
+/**
+ * MouseKeyUp message
+ *
+ * @generated from message proto.ProtoMouseKeyUp
+ */
+export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
+  /**
+   * Fixed value "MouseKeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoMouseKeyUp.
+ * Use `create(ProtoMouseKeyUpSchema)` to create a new message.
+ */
+export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
+  messageDesc(file_types, 4);
+
+/**
+ * KeyDown message
+ *
+ * @generated from message proto.ProtoKeyDown
+ */
+export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
+  /**
+   * Fixed value "KeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoKeyDown.
+ * Use `create(ProtoKeyDownSchema)` to create a new message.
+ */
+export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
+  messageDesc(file_types, 5);
+
+/**
+ * KeyUp message
+ *
+ * @generated from message proto.ProtoKeyUp
+ */
+export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
+  /**
+   * Fixed value "KeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
+   */
+  key: number;
+};
+
+/**
+ * Describes the message proto.ProtoKeyUp.
+ * Use `create(ProtoKeyUpSchema)` to create a new message.
+ */
+export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
+  messageDesc(file_types, 6);
+
+/**
+ * Union of all Input types
+ *
+ * @generated from message proto.ProtoInput
+ */
+export type ProtoInput = Message<"proto.ProtoInput"> & {
+  /**
+   * @generated from oneof proto.ProtoInput.input_type
+   */
+  inputType: {
+    /**
+     * @generated from field: proto.ProtoMouseMove mouse_move = 1;
+     */
+    value: ProtoMouseMove;
+    case: "mouseMove";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
+     */
+    value: ProtoMouseMoveAbs;
+    case: "mouseMoveAbs";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
+     */
+    value: ProtoMouseWheel;
+    case: "mouseWheel";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
+     */
+    value: ProtoMouseKeyDown;
+    case: "mouseKeyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
+     */
+    value: ProtoMouseKeyUp;
+    case: "mouseKeyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyDown key_down = 6;
+     */
+    value: ProtoKeyDown;
+    case: "keyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyUp key_up = 7;
+     */
+    value: ProtoKeyUp;
+    case: "keyUp";
+  } | { case: undefined; value?: undefined };
+};
+
+/**
+ * Describes the message proto.ProtoInput.
+ * Use `create(ProtoInputSchema)` to create a new message.
+ */
+export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
+  messageDesc(file_types, 7);
+

packages/input/src/types.ts

@@ -1,52 +0,0 @@
-interface BaseInput {
-    timestamp?: number; // Add a timestamp for better context (optional)
-}
-
-interface MouseMove extends BaseInput {
-    type: "MouseMove";
-    x: number;
-    y: number;
-}
-
-interface MouseMoveAbs extends BaseInput {
-    type: "MouseMoveAbs";
-    x: number;
-    y: number;
-}
-
-interface MouseWheel extends BaseInput {
-    type: "MouseWheel";
-    x: number;
-    y: number;
-}
-
-interface MouseKeyDown extends BaseInput {
-    type: "MouseKeyDown";
-    key: number;
-}
-
-interface MouseKeyUp extends BaseInput {
-    type: "MouseKeyUp";
-    key: number;
-}
-
-interface KeyDown extends BaseInput {
-    type: "KeyDown";
-    key: number;
-}
-
-interface KeyUp extends BaseInput {
-    type: "KeyUp";
-    key: number;
-}
-
-
-export type Input =
-    | MouseMove
-    | MouseMoveAbs
-    | MouseWheel
-    | MouseKeyDown
-    | MouseKeyUp
-    | KeyDown
-    | KeyUp;
-

packages/input/src/webrtc-stream.ts

@@ -6,8 +6,6 @@ import {
   MessageAnswer,
   JoinerType,
   AnswerType,
-  decodeMessage,
-  encodeMessage
 } from "./messages";
 
 export class WebRTCStream {
@@ -40,16 +38,16 @@ export class WebRTCStream {
         payload_type: "join",
         joiner_type: JoinerType.JoinerClient
       };
-      this._ws!.send(encodeMessage(joinMessage));
+      this._ws!.send(JSON.stringify(joinMessage));
     }
 
     let iceHolder: RTCIceCandidateInit[] = [];
 
     this._ws.onmessage = async (e) => {
-      // allow only binary
+      // allow only JSON
-      if (typeof e.data !== "object") return;
+      if (typeof e.data === "object") return;
       if (!e.data) return;
-      const message = await decodeMessage<MessageBase>(e.data);
+      const message = JSON.parse(e.data) as MessageBase;
       switch (message.payload_type) {
         case "sdp":
           if (!this._pc) {
@@ -63,7 +61,7 @@ export class WebRTCStream {
           // Force stereo in Chromium browsers
           answer.sdp = this.forceOpusStereo(answer.sdp!);
           await this._pc!.setLocalDescription(answer);
-          this._ws!.send(encodeMessage({
+          this._ws!.send(JSON.stringify({
             payload_type: "sdp",
             sdp: answer
           }));
@@ -154,7 +152,7 @@ export class WebRTCStream {
           payload_type: "ice",
           candidate: e.candidate
         };
-        this._ws!.send(encodeMessage(message));
+        this._ws!.send(JSON.stringify(message));
       }
     }
 

packages/relay/go.mod

@@ -6,26 +6,27 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/pion/interceptor v0.1.37
-	github.com/pion/webrtc/v4 v4.0.2
+	github.com/pion/webrtc/v4 v4.0.8
+	google.golang.org/protobuf v1.36.4
 )
 
 require (
-	github.com/pion/datachannel v1.5.9 // indirect
+	github.com/pion/datachannel v1.5.10 // indirect
 	github.com/pion/dtls/v3 v3.0.4 // indirect
-	github.com/pion/ice/v4 v4.0.2 // indirect
+	github.com/pion/ice/v4 v4.0.5 // indirect
-	github.com/pion/logging v0.2.2 // indirect
+	github.com/pion/logging v0.2.3 // indirect
 	github.com/pion/mdns/v2 v2.0.7 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
-	github.com/pion/rtcp v1.2.14 // indirect
+	github.com/pion/rtcp v1.2.15 // indirect
-	github.com/pion/rtp v1.8.9 // indirect
+	github.com/pion/rtp v1.8.11 // indirect
-	github.com/pion/sctp v1.8.34 // indirect
+	github.com/pion/sctp v1.8.35 // indirect
-	github.com/pion/sdp/v3 v3.0.9 // indirect
+	github.com/pion/sdp/v3 v3.0.10 // indirect
 	github.com/pion/srtp/v3 v3.0.4 // indirect
 	github.com/pion/stun/v3 v3.0.0 // indirect
 	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pion/turn/v4 v4.0.0 // indirect
 	github.com/wlynxg/anet v0.0.5 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 )

packages/relay/go.sum

@@ -1,32 +1,33 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA=
+github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
-github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE=
+github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
 github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U=
 github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg=
-github.com/pion/ice/v4 v4.0.2 h1:1JhBRX8iQLi0+TfcavTjPjI6GO41MFn4CeTBX+Y9h5s=
+github.com/pion/ice/v4 v4.0.5 h1:6awVfa1jg9YsI9/Lep4TG/o3kwS1Oayr5b8xz50ibJ8=
-github.com/pion/ice/v4 v4.0.2/go.mod h1:DCdqyzgtsDNYN6/3U8044j3U7qsJ9KFJC92VnOWHvXg=
+github.com/pion/ice/v4 v4.0.5/go.mod h1:JJaoEIxUIlGDA9gaRZbwXYqI3j6VG/QchpjX+QmwN6A=
 github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI=
 github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
-github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
+github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
-github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
+github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
 github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
 github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
-github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE=
+github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
-github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
+github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
-github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk=
+github.com/pion/rtp v1.8.11 h1:17xjnY5WO5hgO6SD3/NTIUPvSFw/PbLsIJyz1r1yNIk=
-github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
+github.com/pion/rtp v1.8.11/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4=
-github.com/pion/sctp v1.8.34 h1:rCuD3m53i0oGxCSp7FLQKvqVx0Nf5AUAHhMRXTTQjBc=
+github.com/pion/sctp v1.8.35 h1:qwtKvNK1Wc5tHMIYgTDJhfZk7vATGVHhXbUDfHbYwzA=
-github.com/pion/sctp v1.8.34/go.mod h1:yWkCClkXlzVW7BXfI2PjrUGBwUI0CjXJBkhLt+sdo4U=
+github.com/pion/sctp v1.8.35/go.mod h1:EcXP8zCYVTRy3W9xtOF7wJm1L1aXfKRQzaM33SjQlzg=
-github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY=
+github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA=
-github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M=
+github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
 github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M=
 github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ=
 github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
@@ -35,28 +36,23 @@ github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1
 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
 github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
 github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
-github.com/pion/webrtc/v4 v4.0.2 h1:fBwm5/hqSUybrCWl0DDBSTDrpbkcgkqpeLmXw9CsBQA=
+github.com/pion/webrtc/v4 v4.0.8 h1:T1ZmnT9qxIJIt4d8XoiMOBrTClGHDDXNg9e/fh018Qc=
-github.com/pion/webrtc/v4 v4.0.2/go.mod h1:moylBT2A4dNoEaYBCdV1nThM3TLwRHzWszIG+eSPaqQ=
+github.com/pion/webrtc/v4 v4.0.8/go.mod h1:HHBeUVBAC+j4ZFnYhovEFStF02Arb1EyD4G7e7HBTJw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
 github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
-golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
+google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

packages/relay/internal/datachannel.go

@@ -2,20 +2,22 @@ package relay
 
 import (
 	"github.com/pion/webrtc/v4"
+	"google.golang.org/protobuf/proto"
 	"log"
+	gen "relay/internal/proto"
 )
 
 // NestriDataChannel is a custom data channel with callbacks
 type NestriDataChannel struct {
 	*webrtc.DataChannel
-	binaryCallbacks map[string]OnMessageCallback // MessageBase type -> callback
+	callbacks map[string]OnMessageCallback // MessageBase type -> callback
 }
 
 // NewNestriDataChannel creates a new NestriDataChannel from *webrtc.DataChannel
 func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
 	ndc := &NestriDataChannel{
 		DataChannel: dc,
-		binaryCallbacks: make(map[string]OnMessageCallback),
+		callbacks:   make(map[string]OnMessageCallback),
 	}
 
 	// Handler for incoming messages
@@ -26,14 +28,14 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
 		}
 
 		// Decode message
-		var base MessageBase
+		var base gen.ProtoMessageInput
-		if err := DecodeMessage(msg.Data, &base); err != nil {
+		if err := proto.Unmarshal(msg.Data, &base); err != nil {
 			log.Printf("Failed to decode binary DataChannel message, reason: %s\n", err)
 			return
 		}
 
 		// Handle message type callback
-		if callback, ok := ndc.binaryCallbacks[base.PayloadType]; ok {
+		if callback, ok := ndc.callbacks["input"]; ok {
 			go callback(msg.Data)
 		} // TODO: Log unknown message type?
 	})
@@ -48,16 +50,16 @@ func (ndc *NestriDataChannel) SendBinary(data []byte) error {
 
 // RegisterMessageCallback registers a callback for a given binary message type
 func (ndc *NestriDataChannel) RegisterMessageCallback(msgType string, callback OnMessageCallback) {
-	if ndc.binaryCallbacks == nil {
+	if ndc.callbacks == nil {
-		ndc.binaryCallbacks = make(map[string]OnMessageCallback)
+		ndc.callbacks = make(map[string]OnMessageCallback)
 	}
-	ndc.binaryCallbacks[msgType] = callback
+	ndc.callbacks[msgType] = callback
 }
 
 // UnregisterMessageCallback removes the callback for a given binary message type
 func (ndc *NestriDataChannel) UnregisterMessageCallback(msgType string) {
-	if ndc.binaryCallbacks != nil {
+	if ndc.callbacks != nil {
-		delete(ndc.binaryCallbacks, msgType)
+		delete(ndc.callbacks, msgType)
 	}
 }
 

packages/relay/internal/egress.go

@@ -1,8 +1,11 @@
 package relay
 
 import (
+	"encoding/json"
 	"github.com/pion/webrtc/v4"
+	"google.golang.org/protobuf/proto"
 	"log"
+	gen "relay/internal/proto"
 )
 
 func participantHandler(participant *Participant, room *Room) {
@@ -54,15 +57,22 @@ func participantHandler(participant *Participant, room *Room) {
 		if room.DataChannel != nil {
 			// If debug mode, decode and add our timestamp, otherwise just send to room
 			if GetFlags().Debug {
-				var inputMsg MessageInput
+				var inputMsg gen.ProtoMessageInput
-				if err = DecodeMessage(data, &inputMsg); err != nil {
+				if err = proto.Unmarshal(data, &inputMsg); err != nil {
 					log.Printf("Failed to decode input message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
 					return
 				}
-				inputMsg.LatencyTracker.AddTimestamp("relay_to_node")
+
-				// Encode and send
+				protoLat := inputMsg.GetMessageBase().GetLatency()
-				if data, err = EncodeMessage(inputMsg); err != nil {
+				if protoLat != nil {
-					log.Printf("Failed to encode input message for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
+					lat := LatencyTrackerFromProto(protoLat)
+					lat.AddTimestamp("relay_to_node")
+					protoLat = lat.ToProto()
+				}
+
+				// Marshal and send
+				if data, err = proto.Marshal(&inputMsg); err != nil {
+					log.Printf("Failed to marshal input message for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
 					return
 				}
 				if err = room.DataChannel.SendBinary(data); err != nil {
@@ -94,7 +104,7 @@ func participantHandler(participant *Participant, room *Room) {
 	// ICE callback
 	participant.WebSocket.RegisterMessageCallback("ice", func(data []byte) {
 		var iceMsg MessageICECandidate
-		if err = DecodeMessage(data, &iceMsg); err != nil {
+		if err = json.Unmarshal(data, &iceMsg); err != nil {
 			log.Printf("Failed to decode ICE message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
 			return
 		}
@@ -120,7 +130,7 @@ func participantHandler(participant *Participant, room *Room) {
 	// SDP answer callback
 	participant.WebSocket.RegisterMessageCallback("sdp", func(data []byte) {
 		var sdpMsg MessageSDP
-		if err = DecodeMessage(data, &sdpMsg); err != nil {
+		if err = json.Unmarshal(data, &sdpMsg); err != nil {
 			log.Printf("Failed to decode SDP message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
 			return
 		}
@@ -130,7 +140,7 @@ func participantHandler(participant *Participant, room *Room) {
 	// Log callback
 	participant.WebSocket.RegisterMessageCallback("log", func(data []byte) {
 		var logMsg MessageLog
-		if err = DecodeMessage(data, &logMsg); err != nil {
+		if err = json.Unmarshal(data, &logMsg); err != nil {
 			log.Printf("Failed to decode log message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
 			return
 		}

packages/relay/internal/http.go

@@ -1,6 +1,7 @@
 package relay
 
 import (
+	"encoding/json"
 	"github.com/gorilla/websocket"
 	"log"
 	"net/http"
@@ -81,7 +82,7 @@ func wsHandler(w http.ResponseWriter, r *http.Request) {
 	// Assign message handler for join request
 	ws.RegisterMessageCallback("join", func(data []byte) {
 		var joinMsg MessageJoin
-		if err = DecodeMessage(data, &joinMsg); err != nil {
+		if err = json.Unmarshal(data, &joinMsg); err != nil {
 			log.Printf("Failed to decode join message: %s\n", err)
 			return
 		}

packages/relay/internal/ingest.go

@@ -1,6 +1,7 @@
 package relay
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/pion/webrtc/v4"
@@ -134,7 +135,7 @@ func ingestHandler(room *Room) {
 	// ICE callback
 	room.WebSocket.RegisterMessageCallback("ice", func(data []byte) {
 		var iceMsg MessageICECandidate
-		if err = DecodeMessage(data, &iceMsg); err != nil {
+		if err = json.Unmarshal(data, &iceMsg); err != nil {
 			log.Printf("Failed to decode ICE candidate message from ingest for room: '%s' - reason: %s\n", room.Name, err)
 			return
 		}
@@ -165,7 +166,7 @@ func ingestHandler(room *Room) {
 	// SDP offer callback
 	room.WebSocket.RegisterMessageCallback("sdp", func(data []byte) {
 		var sdpMsg MessageSDP
-		if err = DecodeMessage(data, &sdpMsg); err != nil {
+		if err = json.Unmarshal(data, &sdpMsg); err != nil {
 			log.Printf("Failed to decode SDP message from ingest for room: '%s' - reason: %s\n", room.Name, err)
 			return
 		}
@@ -182,7 +183,7 @@ func ingestHandler(room *Room) {
 	// Log callback
 	room.WebSocket.RegisterMessageCallback("log", func(data []byte) {
 		var logMsg MessageLog
-		if err = DecodeMessage(data, &logMsg); err != nil {
+		if err = json.Unmarshal(data, &logMsg); err != nil {
 			log.Printf("Failed to decode log message from ingest for room: '%s' - reason: %s\n", room.Name, err)
 			return
 		}
@@ -192,7 +193,7 @@ func ingestHandler(room *Room) {
 	// Metrics callback
 	room.WebSocket.RegisterMessageCallback("metrics", func(data []byte) {
 		var metricsMsg MessageMetrics
-		if err = DecodeMessage(data, &metricsMsg); err != nil {
+		if err = json.Unmarshal(data, &metricsMsg); err != nil {
 			log.Printf("Failed to decode metrics message from ingest for room: '%s' - reason: %s\n", room.Name, err)
 			return
 		}

packages/relay/internal/latency.go

@@ -2,12 +2,14 @@ package relay
 
 import (
 	"fmt"
+	"google.golang.org/protobuf/types/known/timestamppb"
+	gen "relay/internal/proto"
 	"time"
 )
 
 type TimestampEntry struct {
 	Stage string    `json:"stage"`
-	Time  string `json:"time"` // ISO 8601 string
+	Time  time.Time `json:"time"`
 }
 
 // LatencyTracker provides a generic structure for measuring time taken at various stages in message processing.
@@ -15,7 +17,6 @@ type TimestampEntry struct {
 type LatencyTracker struct {
 	SequenceID string           `json:"sequence_id"`
 	Timestamps []TimestampEntry `json:"timestamps"`
-	Metadata   map[string]string `json:"metadata,omitempty"`
 }
 
 // NewLatencyTracker initializes a new LatencyTracker with the given sequence ID
@@ -23,7 +24,6 @@ func NewLatencyTracker(sequenceID string) *LatencyTracker {
 	return &LatencyTracker{
 		SequenceID: sequenceID,
 		Timestamps: make([]TimestampEntry, 0),
-		Metadata:   make(map[string]string),
 	}
 }
 
@@ -32,7 +32,7 @@ func (lt *LatencyTracker) AddTimestamp(stage string) {
 	lt.Timestamps = append(lt.Timestamps, TimestampEntry{
 		Stage: stage,
 		// Ensure extremely precise UTC RFC3339 timestamps (down to nanoseconds)
-		Time: time.Now().UTC().Format(time.RFC3339Nano),
+		Time: time.Now().UTC(),
 	})
 }
 
@@ -44,15 +44,11 @@ func (lt *LatencyTracker) TotalLatency() (int64, error) {
 
 	var earliest, latest time.Time
 	for _, ts := range lt.Timestamps {
-		t, err := time.Parse(time.RFC3339, ts.Time)
+		if earliest.IsZero() || ts.Time.Before(earliest) {
-		if err != nil {
+			earliest = ts.Time
-			return 0, err
 		}
-		if earliest.IsZero() || t.Before(earliest) {
+		if latest.IsZero() || ts.Time.After(latest) {
-			earliest = t
+			latest = ts.Time
-		}
-		if latest.IsZero() || t.After(latest) {
-			latest = t
 		}
 	}
 
@@ -67,14 +63,13 @@ func (lt *LatencyTracker) PainPoints(threshold time.Duration) []string {
 
 	for _, ts := range lt.Timestamps {
 		stage := ts.Stage
-		t := ts.Time
 		if lastStage == "" {
 			lastStage = stage
-			lastTime, _ = time.Parse(time.RFC3339, t)
+			lastTime = ts.Time
 			continue
 		}
 
-		currentTime, _ := time.Parse(time.RFC3339, t)
+		currentTime := ts.Time
 		if currentTime.Sub(lastTime) > threshold {
 			painPoints = append(painPoints, fmt.Sprintf("%s -> %s", lastStage, stage))
 		}
@@ -87,7 +82,7 @@ func (lt *LatencyTracker) PainPoints(threshold time.Duration) []string {
 
 // StageLatency calculates the time taken between two specific stages.
 func (lt *LatencyTracker) StageLatency(startStage, endStage string) (time.Duration, error) {
-	startTime, endTime := "", ""
+	var startTime, endTime time.Time
 	for _, ts := range lt.Timestamps {
 		if ts.Stage == startStage {
 			startTime = ts.Time
@@ -97,18 +92,41 @@ func (lt *LatencyTracker) StageLatency(startStage, endStage string) (time.Durati
 		}
 	}
 
-	if startTime == "" || endTime == "" {
+	/*if startTime == "" || endTime == "" {
 		return 0, fmt.Errorf("missing timestamps for stages: %s -> %s", startStage, endStage)
-	}
+	}*/
 
-	start, err := time.Parse(time.RFC3339, startTime)
+	return endTime.Sub(startTime), nil
-	if err != nil {
+}
-		return 0, err
+
-	}
+func LatencyTrackerFromProto(protolt *gen.ProtoLatencyTracker) *LatencyTracker {
-	end, err := time.Parse(time.RFC3339, endTime)
+	ret := &LatencyTracker{
-	if err != nil {
+		SequenceID: protolt.GetSequenceId(),
-		return 0, err
+		Timestamps: make([]TimestampEntry, 0),
-	}
+	}
-
+
-	return end.Sub(start), nil
+	for _, ts := range protolt.GetTimestamps() {
+		ret.Timestamps = append(ret.Timestamps, TimestampEntry{
+			Stage: ts.GetStage(),
+			Time:  ts.GetTime().AsTime(),
+		})
+	}
+
+	return ret
+}
+
+func (lt *LatencyTracker) ToProto() *gen.ProtoLatencyTracker {
+	ret := &gen.ProtoLatencyTracker{
+		SequenceId: lt.SequenceID,
+		Timestamps: make([]*gen.ProtoTimestampEntry, len(lt.Timestamps)),
+	}
+
+	for i, timestamp := range lt.Timestamps {
+		ret.Timestamps[i] = &gen.ProtoTimestampEntry{
+			Stage: timestamp.Stage,
+			Time:  timestamppb.New(timestamp.Time),
+		}
+	}
+
+	return ret
 }

packages/relay/internal/messages.go

@@ -1,27 +1,17 @@
 package relay
 
 import (
-	"bytes"
-	"compress/gzip"
-	"encoding/json"
-	"fmt"
 	"github.com/pion/webrtc/v4"
 	"time"
 )
 
-// OnMessageCallback is a callback for binary messages of given type
+// OnMessageCallback is a callback for messages of given type
 type OnMessageCallback func(data []byte)
 
 // MessageBase is the base type for WS/DC messages.
 type MessageBase struct {
 	PayloadType string          `json:"payload_type"`
-	LatencyTracker LatencyTracker `json:"latency_tracker,omitempty"`
+	Latency     *LatencyTracker `json:"latency,omitempty"`
-}
-
-// MessageInput represents an input message.
-type MessageInput struct {
-	MessageBase
-	Data string `json:"data"`
 }
 
 // MessageLog represents a log message.
@@ -93,50 +83,6 @@ type MessageAnswer struct {
 	AnswerType AnswerType `json:"answer_type"`
 }
 
-// EncodeMessage encodes a message to be sent with gzip compression
-func EncodeMessage(msg interface{}) ([]byte, error) {
-	// Marshal the message to JSON
-	data, err := json.Marshal(msg)
-	if err != nil {
-		return nil, fmt.Errorf("failed to encode message: %w", err)
-	}
-
-	// Gzip compress the JSON
-	var compressedData bytes.Buffer
-	writer := gzip.NewWriter(&compressedData)
-	_, err = writer.Write(data)
-	if err != nil {
-		return nil, fmt.Errorf("failed to compress message: %w", err)
-	}
-	if err := writer.Close(); err != nil {
-		return nil, fmt.Errorf("failed to finalize compression: %w", err)
-	}
-
-	return compressedData.Bytes(), nil
-}
-
-// DecodeMessage decodes a message received with gzip decompression
-func DecodeMessage(data []byte, target interface{}) error {
-	// Gzip decompress the data
-	reader, err := gzip.NewReader(bytes.NewReader(data))
-	if err != nil {
-		return fmt.Errorf("failed to initialize decompression: %w", err)
-	}
-	defer func(reader *gzip.Reader) {
-		if err = reader.Close(); err != nil {
-			fmt.Printf("failed to close reader: %v\n", err)
-		}
-	}(reader)
-
-	// Decode the JSON
-	err = json.NewDecoder(reader).Decode(target)
-	if err != nil {
-		return fmt.Errorf("failed to decode message: %w", err)
-	}
-
-	return nil
-}
-
 // SendLogMessageWS sends a log message to the given WebSocket connection.
 func (ws *SafeWebSocket) SendLogMessageWS(level, message string) error {
 	msg := MessageLog{
@@ -145,12 +91,7 @@ func (ws *SafeWebSocket) SendLogMessageWS(level, message string) error {
 		Message:     message,
 		Time:        time.Now().Format(time.RFC3339),
 	}
-	encoded, err := EncodeMessage(msg)
+	return ws.SendJSON(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode log message: %w", err)
-	}
-
-	return ws.SendBinary(encoded)
 }
 
 // SendMetricsMessageWS sends a metrics message to the given WebSocket connection.
@@ -162,12 +103,7 @@ func (ws *SafeWebSocket) SendMetricsMessageWS(usageCPU, usageMemory float64, upt
 		Uptime:          uptime,
 		PipelineLatency: pipelineLatency,
 	}
-	encoded, err := EncodeMessage(msg)
+	return ws.SendJSON(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode metrics message: %w", err)
-	}
-
-	return ws.SendBinary(encoded)
 }
 
 // SendICECandidateMessageWS sends an ICE candidate message to the given WebSocket connection.
@@ -176,12 +112,7 @@ func (ws *SafeWebSocket) SendICECandidateMessageWS(candidate webrtc.ICECandidate
 		MessageBase: MessageBase{PayloadType: "ice"},
 		Candidate:   candidate,
 	}
-	encoded, err := EncodeMessage(msg)
+	return ws.SendJSON(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode ICE candidate message: %w", err)
-	}
-
-	return ws.SendBinary(encoded)
 }
 
 // SendSDPMessageWS sends an SDP message to the given WebSocket connection.
@@ -190,12 +121,7 @@ func (ws *SafeWebSocket) SendSDPMessageWS(sdp webrtc.SessionDescription) error {
 		MessageBase: MessageBase{PayloadType: "sdp"},
 		SDP:         sdp,
 	}
-	encoded, err := EncodeMessage(msg)
+	return ws.SendJSON(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode SDP message: %w", err)
-	}
-
-	return ws.SendBinary(encoded)
 }
 
 // SendAnswerMessageWS sends an answer message to the given WebSocket connection.
@@ -204,24 +130,5 @@ func (ws *SafeWebSocket) SendAnswerMessageWS(answer AnswerType) error {
 		MessageBase: MessageBase{PayloadType: "answer"},
 		AnswerType:  answer,
 	}
-	encoded, err := EncodeMessage(msg)
+	return ws.SendJSON(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode answer message: %w", err)
-	}
-
-	return ws.SendBinary(encoded)
-}
-
-// SendInputMessageDC sends an input message to the given DataChannel connection.
-func (ndc *NestriDataChannel) SendInputMessageDC(data string) error {
-	msg := MessageInput{
-		MessageBase: MessageBase{PayloadType: "input"},
-		Data:        data,
-	}
-	encoded, err := EncodeMessage(msg)
-	if err != nil {
-		return fmt.Errorf("failed to encode input message: %w", err)
-	}
-
-	return ndc.SendBinary(encoded)
 }

packages/relay/internal/proto/latency_tracker.pb.go

@@ -0,0 +1,203 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.4
+// 	protoc        (unknown)
+// source: latency_tracker.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ProtoTimestampEntry struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Stage         string                 `protobuf:"bytes,1,opt,name=stage,proto3" json:"stage,omitempty"`
+	Time          *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=time,proto3" json:"time,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoTimestampEntry) Reset() {
+	*x = ProtoTimestampEntry{}
+	mi := &file_latency_tracker_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoTimestampEntry) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoTimestampEntry) ProtoMessage() {}
+
+func (x *ProtoTimestampEntry) ProtoReflect() protoreflect.Message {
+	mi := &file_latency_tracker_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoTimestampEntry.ProtoReflect.Descriptor instead.
+func (*ProtoTimestampEntry) Descriptor() ([]byte, []int) {
+	return file_latency_tracker_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ProtoTimestampEntry) GetStage() string {
+	if x != nil {
+		return x.Stage
+	}
+	return ""
+}
+
+func (x *ProtoTimestampEntry) GetTime() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Time
+	}
+	return nil
+}
+
+type ProtoLatencyTracker struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SequenceId    string                 `protobuf:"bytes,1,opt,name=sequence_id,json=sequenceId,proto3" json:"sequence_id,omitempty"`
+	Timestamps    []*ProtoTimestampEntry `protobuf:"bytes,2,rep,name=timestamps,proto3" json:"timestamps,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoLatencyTracker) Reset() {
+	*x = ProtoLatencyTracker{}
+	mi := &file_latency_tracker_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoLatencyTracker) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoLatencyTracker) ProtoMessage() {}
+
+func (x *ProtoLatencyTracker) ProtoReflect() protoreflect.Message {
+	mi := &file_latency_tracker_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoLatencyTracker.ProtoReflect.Descriptor instead.
+func (*ProtoLatencyTracker) Descriptor() ([]byte, []int) {
+	return file_latency_tracker_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ProtoLatencyTracker) GetSequenceId() string {
+	if x != nil {
+		return x.SequenceId
+	}
+	return ""
+}
+
+func (x *ProtoLatencyTracker) GetTimestamps() []*ProtoTimestampEntry {
+	if x != nil {
+		return x.Timestamps
+	}
+	return nil
+}
+
+var File_latency_tracker_proto protoreflect.FileDescriptor
+
+var file_latency_tracker_proto_rawDesc = string([]byte{
+	0x0a, 0x15, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x65,
+	0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f,
+	0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
+	0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22,
+	0x5b, 0x0a, 0x13, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
+	0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x67, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x04,
+	0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
+	0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
+	0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x72, 0x0a, 0x13,
+	0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x54, 0x72, 0x61, 0x63,
+	0x6b, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f,
+	0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e,
+	0x63, 0x65, 0x49, 0x64, 0x12, 0x3a, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
+	0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x45,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x73,
+	0x42, 0x16, 0x5a, 0x14, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+})
+
+var (
+	file_latency_tracker_proto_rawDescOnce sync.Once
+	file_latency_tracker_proto_rawDescData []byte
+)
+
+func file_latency_tracker_proto_rawDescGZIP() []byte {
+	file_latency_tracker_proto_rawDescOnce.Do(func() {
+		file_latency_tracker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_latency_tracker_proto_rawDesc), len(file_latency_tracker_proto_rawDesc)))
+	})
+	return file_latency_tracker_proto_rawDescData
+}
+
+var file_latency_tracker_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_latency_tracker_proto_goTypes = []any{
+	(*ProtoTimestampEntry)(nil),   // 0: proto.ProtoTimestampEntry
+	(*ProtoLatencyTracker)(nil),   // 1: proto.ProtoLatencyTracker
+	(*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp
+}
+var file_latency_tracker_proto_depIdxs = []int32{
+	2, // 0: proto.ProtoTimestampEntry.time:type_name -> google.protobuf.Timestamp
+	0, // 1: proto.ProtoLatencyTracker.timestamps:type_name -> proto.ProtoTimestampEntry
+	2, // [2:2] is the sub-list for method output_type
+	2, // [2:2] is the sub-list for method input_type
+	2, // [2:2] is the sub-list for extension type_name
+	2, // [2:2] is the sub-list for extension extendee
+	0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_latency_tracker_proto_init() }
+func file_latency_tracker_proto_init() {
+	if File_latency_tracker_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_latency_tracker_proto_rawDesc), len(file_latency_tracker_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_latency_tracker_proto_goTypes,
+		DependencyIndexes: file_latency_tracker_proto_depIdxs,
+		MessageInfos:      file_latency_tracker_proto_msgTypes,
+	}.Build()
+	File_latency_tracker_proto = out.File
+	file_latency_tracker_proto_goTypes = nil
+	file_latency_tracker_proto_depIdxs = nil
+}

packages/relay/internal/proto/messages.pb.go

@@ -0,0 +1,207 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.4
+// 	protoc        (unknown)
+// source: messages.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ProtoMessageBase struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	PayloadType   string                 `protobuf:"bytes,1,opt,name=payload_type,json=payloadType,proto3" json:"payload_type,omitempty"`
+	Latency       *ProtoLatencyTracker   `protobuf:"bytes,2,opt,name=latency,proto3" json:"latency,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMessageBase) Reset() {
+	*x = ProtoMessageBase{}
+	mi := &file_messages_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMessageBase) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMessageBase) ProtoMessage() {}
+
+func (x *ProtoMessageBase) ProtoReflect() protoreflect.Message {
+	mi := &file_messages_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMessageBase.ProtoReflect.Descriptor instead.
+func (*ProtoMessageBase) Descriptor() ([]byte, []int) {
+	return file_messages_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ProtoMessageBase) GetPayloadType() string {
+	if x != nil {
+		return x.PayloadType
+	}
+	return ""
+}
+
+func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker {
+	if x != nil {
+		return x.Latency
+	}
+	return nil
+}
+
+type ProtoMessageInput struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	MessageBase   *ProtoMessageBase      `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"`
+	Data          *ProtoInput            `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMessageInput) Reset() {
+	*x = ProtoMessageInput{}
+	mi := &file_messages_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMessageInput) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMessageInput) ProtoMessage() {}
+
+func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
+	mi := &file_messages_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead.
+func (*ProtoMessageInput) Descriptor() ([]byte, []int) {
+	return file_messages_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase {
+	if x != nil {
+		return x.MessageBase
+	}
+	return nil
+}
+
+func (x *ProtoMessageInput) GetData() *ProtoInput {
+	if x != nil {
+		return x.Data
+	}
+	return nil
+}
+
+var File_messages_proto protoreflect.FileDescriptor
+
+var file_messages_proto_rawDesc = string([]byte{
+	0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x74, 0x72,
+	0x61, 0x63, 0x6b, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6b, 0x0a, 0x10, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x12,
+	0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79,
+	0x70, 0x65, 0x12, 0x34, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x52,
+	0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0x76, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x3a, 0x0a,
+	0x0c, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x0b, 0x6d, 0x65,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x04, 0x64, 0x61, 0x74,
+	0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
+	0x50, 0x72, 0x6f, 0x74, 0x6f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61,
+	0x42, 0x16, 0x5a, 0x14, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+})
+
+var (
+	file_messages_proto_rawDescOnce sync.Once
+	file_messages_proto_rawDescData []byte
+)
+
+func file_messages_proto_rawDescGZIP() []byte {
+	file_messages_proto_rawDescOnce.Do(func() {
+		file_messages_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_messages_proto_rawDesc), len(file_messages_proto_rawDesc)))
+	})
+	return file_messages_proto_rawDescData
+}
+
+var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_messages_proto_goTypes = []any{
+	(*ProtoMessageBase)(nil),    // 0: proto.ProtoMessageBase
+	(*ProtoMessageInput)(nil),   // 1: proto.ProtoMessageInput
+	(*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker
+	(*ProtoInput)(nil),          // 3: proto.ProtoInput
+}
+var file_messages_proto_depIdxs = []int32{
+	2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker
+	0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase
+	3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_messages_proto_init() }
+func file_messages_proto_init() {
+	if File_messages_proto != nil {
+		return
+	}
+	file_types_proto_init()
+	file_latency_tracker_proto_init()
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_messages_proto_rawDesc), len(file_messages_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   2,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_messages_proto_goTypes,
+		DependencyIndexes: file_messages_proto_depIdxs,
+		MessageInfos:      file_messages_proto_msgTypes,
+	}.Build()
+	File_messages_proto = out.File
+	file_messages_proto_goTypes = nil
+	file_messages_proto_depIdxs = nil
+}

packages/relay/internal/proto/types.pb.go

@@ -0,0 +1,713 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.4
+// 	protoc        (unknown)
+// source: types.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// MouseMove message
+type ProtoMouseMove struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseMove"
+	X             int32                  `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"`
+	Y             int32                  `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMouseMove) Reset() {
+	*x = ProtoMouseMove{}
+	mi := &file_types_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMouseMove) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMouseMove) ProtoMessage() {}
+
+func (x *ProtoMouseMove) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMouseMove.ProtoReflect.Descriptor instead.
+func (*ProtoMouseMove) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ProtoMouseMove) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoMouseMove) GetX() int32 {
+	if x != nil {
+		return x.X
+	}
+	return 0
+}
+
+func (x *ProtoMouseMove) GetY() int32 {
+	if x != nil {
+		return x.Y
+	}
+	return 0
+}
+
+// MouseMoveAbs message
+type ProtoMouseMoveAbs struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseMoveAbs"
+	X             int32                  `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"`
+	Y             int32                  `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMouseMoveAbs) Reset() {
+	*x = ProtoMouseMoveAbs{}
+	mi := &file_types_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMouseMoveAbs) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMouseMoveAbs) ProtoMessage() {}
+
+func (x *ProtoMouseMoveAbs) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMouseMoveAbs.ProtoReflect.Descriptor instead.
+func (*ProtoMouseMoveAbs) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ProtoMouseMoveAbs) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoMouseMoveAbs) GetX() int32 {
+	if x != nil {
+		return x.X
+	}
+	return 0
+}
+
+func (x *ProtoMouseMoveAbs) GetY() int32 {
+	if x != nil {
+		return x.Y
+	}
+	return 0
+}
+
+// MouseWheel message
+type ProtoMouseWheel struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseWheel"
+	X             int32                  `protobuf:"varint,2,opt,name=x,proto3" json:"x,omitempty"`
+	Y             int32                  `protobuf:"varint,3,opt,name=y,proto3" json:"y,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMouseWheel) Reset() {
+	*x = ProtoMouseWheel{}
+	mi := &file_types_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMouseWheel) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMouseWheel) ProtoMessage() {}
+
+func (x *ProtoMouseWheel) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMouseWheel.ProtoReflect.Descriptor instead.
+func (*ProtoMouseWheel) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *ProtoMouseWheel) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoMouseWheel) GetX() int32 {
+	if x != nil {
+		return x.X
+	}
+	return 0
+}
+
+func (x *ProtoMouseWheel) GetY() int32 {
+	if x != nil {
+		return x.Y
+	}
+	return 0
+}
+
+// MouseKeyDown message
+type ProtoMouseKeyDown struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseKeyDown"
+	Key           int32                  `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMouseKeyDown) Reset() {
+	*x = ProtoMouseKeyDown{}
+	mi := &file_types_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMouseKeyDown) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMouseKeyDown) ProtoMessage() {}
+
+func (x *ProtoMouseKeyDown) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMouseKeyDown.ProtoReflect.Descriptor instead.
+func (*ProtoMouseKeyDown) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ProtoMouseKeyDown) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoMouseKeyDown) GetKey() int32 {
+	if x != nil {
+		return x.Key
+	}
+	return 0
+}
+
+// MouseKeyUp message
+type ProtoMouseKeyUp struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "MouseKeyUp"
+	Key           int32                  `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoMouseKeyUp) Reset() {
+	*x = ProtoMouseKeyUp{}
+	mi := &file_types_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoMouseKeyUp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoMouseKeyUp) ProtoMessage() {}
+
+func (x *ProtoMouseKeyUp) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoMouseKeyUp.ProtoReflect.Descriptor instead.
+func (*ProtoMouseKeyUp) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ProtoMouseKeyUp) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoMouseKeyUp) GetKey() int32 {
+	if x != nil {
+		return x.Key
+	}
+	return 0
+}
+
+// KeyDown message
+type ProtoKeyDown struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "KeyDown"
+	Key           int32                  `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoKeyDown) Reset() {
+	*x = ProtoKeyDown{}
+	mi := &file_types_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoKeyDown) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoKeyDown) ProtoMessage() {}
+
+func (x *ProtoKeyDown) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoKeyDown.ProtoReflect.Descriptor instead.
+func (*ProtoKeyDown) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *ProtoKeyDown) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoKeyDown) GetKey() int32 {
+	if x != nil {
+		return x.Key
+	}
+	return 0
+}
+
+// KeyUp message
+type ProtoKeyUp struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          string                 `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` // Fixed value "KeyUp"
+	Key           int32                  `protobuf:"varint,2,opt,name=key,proto3" json:"key,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoKeyUp) Reset() {
+	*x = ProtoKeyUp{}
+	mi := &file_types_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoKeyUp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoKeyUp) ProtoMessage() {}
+
+func (x *ProtoKeyUp) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoKeyUp.ProtoReflect.Descriptor instead.
+func (*ProtoKeyUp) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *ProtoKeyUp) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+func (x *ProtoKeyUp) GetKey() int32 {
+	if x != nil {
+		return x.Key
+	}
+	return 0
+}
+
+// Union of all Input types
+type ProtoInput struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to InputType:
+	//
+	//	*ProtoInput_MouseMove
+	//	*ProtoInput_MouseMoveAbs
+	//	*ProtoInput_MouseWheel
+	//	*ProtoInput_MouseKeyDown
+	//	*ProtoInput_MouseKeyUp
+	//	*ProtoInput_KeyDown
+	//	*ProtoInput_KeyUp
+	InputType     isProtoInput_InputType `protobuf_oneof:"input_type"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ProtoInput) Reset() {
+	*x = ProtoInput{}
+	mi := &file_types_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ProtoInput) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ProtoInput) ProtoMessage() {}
+
+func (x *ProtoInput) ProtoReflect() protoreflect.Message {
+	mi := &file_types_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ProtoInput.ProtoReflect.Descriptor instead.
+func (*ProtoInput) Descriptor() ([]byte, []int) {
+	return file_types_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *ProtoInput) GetInputType() isProtoInput_InputType {
+	if x != nil {
+		return x.InputType
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetMouseMove() *ProtoMouseMove {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_MouseMove); ok {
+			return x.MouseMove
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetMouseMoveAbs() *ProtoMouseMoveAbs {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_MouseMoveAbs); ok {
+			return x.MouseMoveAbs
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetMouseWheel() *ProtoMouseWheel {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_MouseWheel); ok {
+			return x.MouseWheel
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetMouseKeyDown() *ProtoMouseKeyDown {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_MouseKeyDown); ok {
+			return x.MouseKeyDown
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetMouseKeyUp() *ProtoMouseKeyUp {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_MouseKeyUp); ok {
+			return x.MouseKeyUp
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetKeyDown() *ProtoKeyDown {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_KeyDown); ok {
+			return x.KeyDown
+		}
+	}
+	return nil
+}
+
+func (x *ProtoInput) GetKeyUp() *ProtoKeyUp {
+	if x != nil {
+		if x, ok := x.InputType.(*ProtoInput_KeyUp); ok {
+			return x.KeyUp
+		}
+	}
+	return nil
+}
+
+type isProtoInput_InputType interface {
+	isProtoInput_InputType()
+}
+
+type ProtoInput_MouseMove struct {
+	MouseMove *ProtoMouseMove `protobuf:"bytes,1,opt,name=mouse_move,json=mouseMove,proto3,oneof"`
+}
+
+type ProtoInput_MouseMoveAbs struct {
+	MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,2,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"`
+}
+
+type ProtoInput_MouseWheel struct {
+	MouseWheel *ProtoMouseWheel `protobuf:"bytes,3,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"`
+}
+
+type ProtoInput_MouseKeyDown struct {
+	MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,4,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"`
+}
+
+type ProtoInput_MouseKeyUp struct {
+	MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,5,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"`
+}
+
+type ProtoInput_KeyDown struct {
+	KeyDown *ProtoKeyDown `protobuf:"bytes,6,opt,name=key_down,json=keyDown,proto3,oneof"`
+}
+
+type ProtoInput_KeyUp struct {
+	KeyUp *ProtoKeyUp `protobuf:"bytes,7,opt,name=key_up,json=keyUp,proto3,oneof"`
+}
+
+func (*ProtoInput_MouseMove) isProtoInput_InputType() {}
+
+func (*ProtoInput_MouseMoveAbs) isProtoInput_InputType() {}
+
+func (*ProtoInput_MouseWheel) isProtoInput_InputType() {}
+
+func (*ProtoInput_MouseKeyDown) isProtoInput_InputType() {}
+
+func (*ProtoInput_MouseKeyUp) isProtoInput_InputType() {}
+
+func (*ProtoInput_KeyDown) isProtoInput_InputType() {}
+
+func (*ProtoInput_KeyUp) isProtoInput_InputType() {}
+
+var File_types_proto protoreflect.FileDescriptor
+
+var file_types_proto_rawDesc = string([]byte{
+	0x0a, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x22, 0x40, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75,
+	0x73, 0x65, 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x01, 0x79, 0x22, 0x43, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d,
+	0x6f, 0x75, 0x73, 0x65, 0x4d, 0x6f, 0x76, 0x65, 0x41, 0x62, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74,
+	0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
+	0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x78, 0x12, 0x0c, 0x0a,
+	0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x79, 0x22, 0x41, 0x0a, 0x0f, 0x50,
+	0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x57, 0x68, 0x65, 0x65, 0x6c, 0x12, 0x12,
+	0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79,
+	0x70, 0x65, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x78,
+	0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x79, 0x22, 0x39,
+	0x0a, 0x11, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x44,
+	0x6f, 0x77, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x37, 0x0a, 0x0f, 0x50, 0x72, 0x6f,
+	0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x55, 0x70, 0x12, 0x12, 0x0a, 0x04,
+	0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65,
+	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b,
+	0x65, 0x79, 0x22, 0x34, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x44, 0x6f,
+	0x77, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x32, 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x74,
+	0x6f, 0x4b, 0x65, 0x79, 0x55, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
+	0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0xab, 0x03, 0x0a,
+	0x0a, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x36, 0x0a, 0x0a, 0x6d,
+	0x6f, 0x75, 0x73, 0x65, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
+	0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75,
+	0x73, 0x65, 0x4d, 0x6f, 0x76, 0x65, 0x48, 0x00, 0x52, 0x09, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x4d,
+	0x6f, 0x76, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x5f, 0x6d, 0x6f, 0x76,
+	0x65, 0x5f, 0x61, 0x62, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x4d, 0x6f,
+	0x76, 0x65, 0x41, 0x62, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x4d, 0x6f,
+	0x76, 0x65, 0x41, 0x62, 0x73, 0x12, 0x39, 0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x5f, 0x77,
+	0x68, 0x65, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x57, 0x68, 0x65,
+	0x65, 0x6c, 0x48, 0x00, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x57, 0x68, 0x65, 0x65, 0x6c,
+	0x12, 0x40, 0x0a, 0x0e, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x64, 0x6f,
+	0x77, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x44, 0x6f,
+	0x77, 0x6e, 0x48, 0x00, 0x52, 0x0c, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x44, 0x6f,
+	0x77, 0x6e, 0x12, 0x3a, 0x0a, 0x0c, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f,
+	0x75, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x55, 0x70,
+	0x48, 0x00, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x55, 0x70, 0x12, 0x30,
+	0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4b, 0x65,
+	0x79, 0x44, 0x6f, 0x77, 0x6e, 0x48, 0x00, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x44, 0x6f, 0x77, 0x6e,
+	0x12, 0x2a, 0x0a, 0x06, 0x6b, 0x65, 0x79, 0x5f, 0x75, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b,
+	0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x4b, 0x65,
+	0x79, 0x55, 0x70, 0x48, 0x00, 0x52, 0x05, 0x6b, 0x65, 0x79, 0x55, 0x70, 0x42, 0x0c, 0x0a, 0x0a,
+	0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x65,
+	0x6c, 0x61, 0x79, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+})
+
+var (
+	file_types_proto_rawDescOnce sync.Once
+	file_types_proto_rawDescData []byte
+)
+
+func file_types_proto_rawDescGZIP() []byte {
+	file_types_proto_rawDescOnce.Do(func() {
+		file_types_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_types_proto_rawDesc), len(file_types_proto_rawDesc)))
+	})
+	return file_types_proto_rawDescData
+}
+
+var file_types_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
+var file_types_proto_goTypes = []any{
+	(*ProtoMouseMove)(nil),    // 0: proto.ProtoMouseMove
+	(*ProtoMouseMoveAbs)(nil), // 1: proto.ProtoMouseMoveAbs
+	(*ProtoMouseWheel)(nil),   // 2: proto.ProtoMouseWheel
+	(*ProtoMouseKeyDown)(nil), // 3: proto.ProtoMouseKeyDown
+	(*ProtoMouseKeyUp)(nil),   // 4: proto.ProtoMouseKeyUp
+	(*ProtoKeyDown)(nil),      // 5: proto.ProtoKeyDown
+	(*ProtoKeyUp)(nil),        // 6: proto.ProtoKeyUp
+	(*ProtoInput)(nil),        // 7: proto.ProtoInput
+}
+var file_types_proto_depIdxs = []int32{
+	0, // 0: proto.ProtoInput.mouse_move:type_name -> proto.ProtoMouseMove
+	1, // 1: proto.ProtoInput.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs
+	2, // 2: proto.ProtoInput.mouse_wheel:type_name -> proto.ProtoMouseWheel
+	3, // 3: proto.ProtoInput.mouse_key_down:type_name -> proto.ProtoMouseKeyDown
+	4, // 4: proto.ProtoInput.mouse_key_up:type_name -> proto.ProtoMouseKeyUp
+	5, // 5: proto.ProtoInput.key_down:type_name -> proto.ProtoKeyDown
+	6, // 6: proto.ProtoInput.key_up:type_name -> proto.ProtoKeyUp
+	7, // [7:7] is the sub-list for method output_type
+	7, // [7:7] is the sub-list for method input_type
+	7, // [7:7] is the sub-list for extension type_name
+	7, // [7:7] is the sub-list for extension extendee
+	0, // [0:7] is the sub-list for field type_name
+}
+
+func init() { file_types_proto_init() }
+func file_types_proto_init() {
+	if File_types_proto != nil {
+		return
+	}
+	file_types_proto_msgTypes[7].OneofWrappers = []any{
+		(*ProtoInput_MouseMove)(nil),
+		(*ProtoInput_MouseMoveAbs)(nil),
+		(*ProtoInput_MouseWheel)(nil),
+		(*ProtoInput_MouseKeyDown)(nil),
+		(*ProtoInput_MouseKeyUp)(nil),
+		(*ProtoInput_KeyDown)(nil),
+		(*ProtoInput_KeyUp)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_types_proto_rawDesc), len(file_types_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   8,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_types_proto_goTypes,
+		DependencyIndexes: file_types_proto_depIdxs,
+		MessageInfos:      file_types_proto_msgTypes,
+	}.Build()
+	File_types_proto = out.File
+	file_types_proto_goTypes = nil
+	file_types_proto_depIdxs = nil
+}

packages/relay/internal/websocket.go

@@ -1,6 +1,7 @@
 package relay
 
 import (
+	"encoding/json"
 	"github.com/gorilla/websocket"
 	"log"
 	"sync"
@@ -11,7 +12,7 @@ type SafeWebSocket struct {
 	*websocket.Conn
 	sync.Mutex
 	closeCallback func()                       // OnClose callback
-	binaryCallbacks map[string]OnMessageCallback // MessageBase type -> callback
+	callbacks     map[string]OnMessageCallback // MessageBase type -> callback
 }
 
 // NewSafeWebSocket creates a new SafeWebSocket from *websocket.Conn
@@ -19,13 +20,13 @@ func NewSafeWebSocket(conn *websocket.Conn) *SafeWebSocket {
 	ws := &SafeWebSocket{
 		Conn:          conn,
 		closeCallback: nil,
-		binaryCallbacks: make(map[string]OnMessageCallback),
+		callbacks:     make(map[string]OnMessageCallback),
 	}
 
-	// Launch a goroutine to handle binary messages
+	// Launch a goroutine to handle messages
 	go func() {
 		for {
-			// Read binary message
+			// Read message
 			kind, data, err := ws.Conn.ReadMessage()
 			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure, websocket.CloseNoStatusReceived) {
 				// If unexpected close error, break
@@ -42,22 +43,23 @@ func NewSafeWebSocket(conn *websocket.Conn) *SafeWebSocket {
 
 			switch kind {
 			case websocket.TextMessage:
-				// Ignore, we use binary messages
-				continue
-			case websocket.BinaryMessage:
 				// Decode message
 				var msg MessageBase
-				if err = DecodeMessage(data, &msg); err != nil {
+				if err = json.Unmarshal(data, &msg); err != nil {
-					log.Printf("Failed to decode binary WebSocket message, reason: %s\n", err)
+					log.Printf("Failed to decode text WebSocket message, reason: %s\n", err)
 					continue
 				}
 
 				// Handle message type callback
-				if callback, ok := ws.binaryCallbacks[msg.PayloadType]; ok {
+				if callback, ok := ws.callbacks[msg.PayloadType]; ok {
 					callback(data)
 				} // TODO: Log unknown message type?
+				break
+			case websocket.BinaryMessage:
+				break
 			default:
 				log.Printf("Unknown WebSocket message type: %d\n", kind)
+				break
 			}
 		}
 
@@ -88,18 +90,18 @@ func (ws *SafeWebSocket) SendBinary(data []byte) error {
 func (ws *SafeWebSocket) RegisterMessageCallback(msgType string, callback OnMessageCallback) {
 	ws.Lock()
 	defer ws.Unlock()
-	if ws.binaryCallbacks == nil {
+	if ws.callbacks == nil {
-		ws.binaryCallbacks = make(map[string]OnMessageCallback)
+		ws.callbacks = make(map[string]OnMessageCallback)
 	}
-	ws.binaryCallbacks[msgType] = callback
+	ws.callbacks[msgType] = callback
 }
 
 // UnregisterMessageCallback removes the callback for binary message of given type
 func (ws *SafeWebSocket) UnregisterMessageCallback(msgType string) {
 	ws.Lock()
 	defer ws.Unlock()
-	if ws.binaryCallbacks != nil {
+	if ws.callbacks != nil {
-		delete(ws.binaryCallbacks, msgType)
+		delete(ws.callbacks, msgType)
 	}
 }
 
@@ -108,7 +110,7 @@ func (ws *SafeWebSocket) RegisterOnClose(callback func()) {
 	ws.closeCallback = func() {
 		// Clear our callbacks
 		ws.Lock()
-		ws.binaryCallbacks = nil
+		ws.callbacks = nil
 		ws.Unlock()
 		// Call the callback
 		callback()

packages/server/Cargo.toml

@@ -15,15 +15,15 @@ serde = {version = "1.0.214", features = ["derive"] }
 tokio = { version = "1.41.0", features = ["full"] }
 clap = { version = "4.5.20", features = ["env"] }
 serde_json = "1.0.132"
-webrtc = "0.11.0"
+webrtc = "0.12.0"
 regex = "1.11.1"
-rand = "0.8.5"
+rand = "0.9.0"
 rustls = { version = "0.23.17", features = ["ring"] }
-tokio-util = "0.7.12"
+tokio-tungstenite = { version = "0.26.1", features = ["native-tls"] }
-flate2 = "1.0.35"
-tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
 log = { version = "0.4.22", features = ["std"] }
 chrono = "0.4.38"
 futures-util = "0.3.31"
 num-derive = "0.4.2"
 num-traits = "0.2.19"
+prost = "0.13.4"
+prost-types = "0.13.4"

packages/server/src/args/app_args.rs

@@ -48,7 +48,10 @@ impl AppArgs {
                 .unwrap()
                 .parse::<u32>()
                 .unwrap_or(60),
-            relay_url: matches.get_one::<String>("relay-url").unwrap().clone(),
+            relay_url: matches
+                .get_one::<String>("relay-url")
+                .expect("relay url cannot be empty")
+                .clone(),
             // Generate random room name if not provided
             room: matches
                 .get_one::<String>("room")

packages/server/src/latency.rs

@@ -1,5 +1,5 @@
-use std::collections::HashMap;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct TimestampEntry {

packages/server/src/main.rs

@@ -5,6 +5,7 @@ mod latency;
 mod messages;
 mod nestrisink;
 mod websocket;
+mod proto;
 
 use crate::args::encoding_args;
 use crate::nestrisink::NestriSignaller;

packages/server/src/messages.rs

@@ -1,50 +1,14 @@
-use std::error::Error;
+use crate::latency::LatencyTracker;
-use std::io::{Read, Write};
-use flate2::Compression;
-use flate2::read::GzDecoder;
-use flate2::write::GzEncoder;
 use num_derive::{FromPrimitive, ToPrimitive};
 use num_traits::{FromPrimitive, ToPrimitive};
 use serde::{Deserialize, Serialize};
+use std::error::Error;
 use webrtc::ice_transport::ice_candidate::RTCIceCandidateInit;
 use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
-use crate::latency::LatencyTracker;
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(tag = "type")]
-pub enum InputMessage {
-    #[serde(rename = "mousemove")]
-    MouseMove { x: i32, y: i32 },
-
-    #[serde(rename = "mousemoveabs")]
-    MouseMoveAbs { x: i32, y: i32 },
-
-    #[serde(rename = "wheel")]
-    Wheel { x: f64, y: f64 },
-
-    #[serde(rename = "mousedown")]
-    MouseDown { key: i32 },
-    // Add other variants as needed
-    #[serde(rename = "mouseup")]
-    MouseUp { key: i32 },
-
-    #[serde(rename = "keydown")]
-    KeyDown { key: i32 },
-
-    #[serde(rename = "keyup")]
-    KeyUp { key: i32 },
-}
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct MessageBase {
     pub payload_type: String,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MessageInput {
-    #[serde(flatten)]
-    pub base: MessageBase,
-    pub data: String,
     pub latency: Option<LatencyTracker>,
 }
 
@@ -136,34 +100,21 @@ pub struct MessageAnswer {
     pub answer_type: AnswerType,
 }
 
-pub fn encode_message<T: Serialize>(message: &T) -> Result<Vec<u8>, Box<dyn Error>> {
+pub fn encode_message<T: Serialize>(message: &T) -> Result<String, Box<dyn Error>> {
     // Serialize the message to JSON
     let json = serde_json::to_string(message)?;
-
+    Ok(json)
-    // Compress the JSON using gzip
-    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
-    encoder.write_all(json.as_bytes())?;
-    let compressed_data = encoder.finish()?;
-
-    Ok(compressed_data)
 }
 
-pub fn decode_message(data: &[u8]) -> Result<MessageBase, Box<dyn Error + Send + Sync>> {
+pub fn decode_message(data: String) -> Result<MessageBase, Box<dyn Error + Send + Sync>> {
-    let mut decoder = GzDecoder::new(data);
+    println!("Data: {}", data);
-    let mut decompressed_data = String::new();
+    let base_message: MessageBase = serde_json::from_str(&data)?;
-    decoder.read_to_string(&mut decompressed_data)?;
-
-    let base_message: MessageBase = serde_json::from_str(&decompressed_data)?;
     Ok(base_message)
 }
 
 pub fn decode_message_as<T: for<'de> Deserialize<'de>>(
-    data: Vec<u8>,
+    data: String,
 ) -> Result<T, Box<dyn Error + Send + Sync>> {
-    let mut decoder = GzDecoder::new(data.as_slice());
+    let message: T = serde_json::from_str(&data)?;
-    let mut decompressed_data = String::new();
-    decoder.read_to_string(&mut decompressed_data)?;
-
-    let message: T = serde_json::from_str(&decompressed_data)?;
     Ok(message)
 }

packages/server/src/nestrisink/imp.rs

@@ -1,13 +1,18 @@
 use crate::messages::{
-    decode_message_as, encode_message, AnswerType, InputMessage, JoinerType, MessageAnswer,
+    decode_message_as, encode_message, AnswerType, JoinerType, MessageAnswer, MessageBase,
-    MessageBase, MessageICE, MessageInput, MessageJoin, MessageSDP,
+    MessageICE, MessageJoin, MessageSDP,
 };
+use crate::proto::proto::proto_input::InputType::{
+    KeyDown, KeyUp, MouseKeyDown, MouseKeyUp, MouseMove, MouseMoveAbs, MouseWheel,
+};
+use crate::proto::proto::{ProtoInput, ProtoMessageInput};
 use crate::websocket::NestriWebSocket;
 use glib::subclass::prelude::*;
 use gst::glib;
 use gst::prelude::*;
 use gst_webrtc::{gst_sdp, WebRTCSDPType, WebRTCSessionDescription};
 use gstrswebrtc::signaller::{Signallable, SignallableImpl};
+use prost::Message;
 use std::collections::HashSet;
 use std::sync::{Arc, LazyLock};
 use std::sync::{Mutex, RwLock};
@@ -200,6 +205,7 @@ impl SignallableImpl for Signaller {
                 let join_msg = MessageJoin {
                     base: MessageBase {
                         payload_type: "join".to_string(),
+                        latency: None,
                     },
                     joiner_type: JoinerType::JoinerNode,
                 };
@@ -237,6 +243,7 @@ impl SignallableImpl for Signaller {
         let join_msg = MessageJoin {
             base: MessageBase {
                 payload_type: "join".to_string(),
+                latency: None,
             },
             joiner_type: JoinerType::JoinerNode,
         };
@@ -265,6 +272,7 @@ impl SignallableImpl for Signaller {
         let sdp_message = MessageSDP {
             base: MessageBase {
                 payload_type: "sdp".to_string(),
+                latency: None,
             },
             sdp: RTCSessionDescription::offer(sdp.sdp().as_text().unwrap()).unwrap(),
         };
@@ -301,6 +309,7 @@ impl SignallableImpl for Signaller {
         let ice_message = MessageICE {
             base: MessageBase {
                 payload_type: "ice".to_string(),
+                latency: None,
             },
             candidate: candidate_init,
         };
@@ -354,11 +363,9 @@ fn setup_data_channel(data_channel: &gst_webrtc::WebRTCDataChannel, pipeline: &g
 
     data_channel.connect_on_message_data(move |_data_channel, data| {
         if let Some(data) = data {
-            match decode_message_as::<MessageInput>(data.to_vec()) {
+            match ProtoMessageInput::decode(data.to_vec().as_slice()) {
                 Ok(message_input) => {
-                    // Deserialize the input message data
+                    if let Some(input_msg) = message_input.data {
-                    if let Ok(input_msg) = serde_json::from_str::<InputMessage>(&message_input.data)
-                    {
                         // Process the input message and create an event
                         if let Some(event) =
                             handle_input_message(input_msg, &pressed_keys, &pressed_buttons)
@@ -379,88 +386,92 @@ fn setup_data_channel(data_channel: &gst_webrtc::WebRTCDataChannel, pipeline: &g
 }
 
 fn handle_input_message(
-    input_msg: InputMessage,
+    input_msg: ProtoInput,
     pressed_keys: &Arc<Mutex<HashSet<i32>>>,
     pressed_buttons: &Arc<Mutex<HashSet<i32>>>,
 ) -> Option<gst::Event> {
-    match input_msg {
+    if let Some(input_type) = input_msg.input_type {
-        InputMessage::MouseMove { x, y } => {
+        match input_type {
+            MouseMove(data) => {
                 let structure = gst::Structure::builder("MouseMoveRelative")
-                .field("pointer_x", x as f64)
+                    .field("pointer_x", data.x as f64)
-                .field("pointer_y", y as f64)
+                    .field("pointer_y", data.y as f64)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::MouseMoveAbs { x, y } => {
+            MouseMoveAbs(data) => {
                 let structure = gst::Structure::builder("MouseMoveAbsolute")
-                .field("pointer_x", x as f64)
+                    .field("pointer_x", data.x as f64)
-                .field("pointer_y", y as f64)
+                    .field("pointer_y", data.y as f64)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::KeyDown { key } => {
+            KeyDown(data) => {
                 let mut keys = pressed_keys.lock().unwrap();
                 // If the key is already pressed, return to prevent key lockup
-            if keys.contains(&key) {
+                if keys.contains(&data.key) {
                     return None;
                 }
-            keys.insert(key);
+                keys.insert(data.key);
 
                 let structure = gst::Structure::builder("KeyboardKey")
-                .field("key", key as u32)
+                    .field("key", data.key as u32)
                     .field("pressed", true)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::KeyUp { key } => {
+            KeyUp(data) => {
                 let mut keys = pressed_keys.lock().unwrap();
                 // Remove the key from the pressed state when released
-            keys.remove(&key);
+                keys.remove(&data.key);
 
                 let structure = gst::Structure::builder("KeyboardKey")
-                .field("key", key as u32)
+                    .field("key", data.key as u32)
                     .field("pressed", false)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::Wheel { x, y } => {
+            MouseWheel(data) => {
                 let structure = gst::Structure::builder("MouseAxis")
-                .field("x", x as f64)
+                    .field("x", data.x as f64)
-                .field("y", y as f64)
+                    .field("y", data.y as f64)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::MouseDown { key } => {
+            MouseKeyDown(data) => {
                 let mut buttons = pressed_buttons.lock().unwrap();
                 // If the button is already pressed, return to prevent button lockup
-            if buttons.contains(&key) {
+                if buttons.contains(&data.key) {
                     return None;
                 }
-            buttons.insert(key);
+                buttons.insert(data.key);
 
                 let structure = gst::Structure::builder("MouseButton")
-                .field("button", key as u32)
+                    .field("button", data.key as u32)
                     .field("pressed", true)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
-        InputMessage::MouseUp { key } => {
+            MouseKeyUp(data) => {
                 let mut buttons = pressed_buttons.lock().unwrap();
                 // Remove the button from the pressed state when released
-            buttons.remove(&key);
+                buttons.remove(&data.key);
 
                 let structure = gst::Structure::builder("MouseButton")
-                .field("button", key as u32)
+                    .field("button", data.key as u32)
                     .field("pressed", false)
                     .build();
 
                 Some(gst::event::CustomUpstream::new(structure))
             }
         }
+    } else {
+        None
+    }
 }

packages/server/src/proto.rs

@@ -0,0 +1 @@
+pub mod proto;

packages/server/src/proto/proto.rs

@@ -0,0 +1,139 @@
+// @generated
+// This file is @generated by prost-build.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoTimestampEntry {
+    #[prost(string, tag="1")]
+    pub stage: ::prost::alloc::string::String,
+    #[prost(message, optional, tag="2")]
+    pub time: ::core::option::Option<::prost_types::Timestamp>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoLatencyTracker {
+    #[prost(string, tag="1")]
+    pub sequence_id: ::prost::alloc::string::String,
+    #[prost(message, repeated, tag="2")]
+    pub timestamps: ::prost::alloc::vec::Vec<ProtoTimestampEntry>,
+}
+/// MouseMove message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMouseMove {
+    /// Fixed value "MouseMove"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub x: i32,
+    #[prost(int32, tag="3")]
+    pub y: i32,
+}
+/// MouseMoveAbs message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMouseMoveAbs {
+    /// Fixed value "MouseMoveAbs"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub x: i32,
+    #[prost(int32, tag="3")]
+    pub y: i32,
+}
+/// MouseWheel message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMouseWheel {
+    /// Fixed value "MouseWheel"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub x: i32,
+    #[prost(int32, tag="3")]
+    pub y: i32,
+}
+/// MouseKeyDown message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMouseKeyDown {
+    /// Fixed value "MouseKeyDown"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub key: i32,
+}
+/// MouseKeyUp message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMouseKeyUp {
+    /// Fixed value "MouseKeyUp"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub key: i32,
+}
+/// KeyDown message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoKeyDown {
+    /// Fixed value "KeyDown"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub key: i32,
+}
+/// KeyUp message
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoKeyUp {
+    /// Fixed value "KeyUp"
+    #[prost(string, tag="1")]
+    pub r#type: ::prost::alloc::string::String,
+    #[prost(int32, tag="2")]
+    pub key: i32,
+}
+/// Union of all Input types
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoInput {
+    #[prost(oneof="proto_input::InputType", tags="1, 2, 3, 4, 5, 6, 7")]
+    pub input_type: ::core::option::Option<proto_input::InputType>,
+}
+/// Nested message and enum types in `ProtoInput`.
+pub mod proto_input {
+    #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum InputType {
+        #[prost(message, tag="1")]
+        MouseMove(super::ProtoMouseMove),
+        #[prost(message, tag="2")]
+        MouseMoveAbs(super::ProtoMouseMoveAbs),
+        #[prost(message, tag="3")]
+        MouseWheel(super::ProtoMouseWheel),
+        #[prost(message, tag="4")]
+        MouseKeyDown(super::ProtoMouseKeyDown),
+        #[prost(message, tag="5")]
+        MouseKeyUp(super::ProtoMouseKeyUp),
+        #[prost(message, tag="6")]
+        KeyDown(super::ProtoKeyDown),
+        #[prost(message, tag="7")]
+        KeyUp(super::ProtoKeyUp),
+    }
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMessageBase {
+    #[prost(string, tag="1")]
+    pub payload_type: ::prost::alloc::string::String,
+    #[prost(message, optional, tag="2")]
+    pub latency: ::core::option::Option<ProtoLatencyTracker>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtoMessageInput {
+    #[prost(message, optional, tag="1")]
+    pub message_base: ::core::option::Option<ProtoMessageBase>,
+    #[prost(message, optional, tag="2")]
+    pub data: ::core::option::Option<ProtoInput>,
+}
+// @@protoc_insertion_point(module)

packages/server/src/websocket.rs

@@ -10,10 +10,10 @@ use std::time::Duration;
 use tokio::net::TcpStream;
 use tokio::sync::{mpsc, Mutex, Notify};
 use tokio::time::sleep;
-use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::tungstenite::{Message, Utf8Bytes};
 use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
 
-type Callback = Box<dyn Fn(Vec<u8>) + Send + Sync>;
+type Callback = Box<dyn Fn(String) + Send + Sync>;
 type WSRead = SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>;
 type WSWrite = SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>;
 
@@ -23,7 +23,7 @@ pub struct NestriWebSocket {
     reader: Arc<Mutex<Option<WSRead>>>,
     writer: Arc<Mutex<Option<WSWrite>>>,
     callbacks: Arc<RwLock<HashMap<String, Callback>>>,
-    message_tx: mpsc::UnboundedSender<Vec<u8>>,
+    message_tx: mpsc::UnboundedSender<String>,
     reconnected_notify: Arc<Notify>,
 }
 impl NestriWebSocket {
@@ -95,8 +95,8 @@ impl NestriWebSocket {
                 while let Some(message_result) = ws_read.next().await {
                     match message_result {
                         Ok(message) => {
-                            let data = message.into_data();
+                            let data = message.into_text().expect("failed to turn message into text");
-                            let base_message = match decode_message(&data) {
+                            let base_message = match decode_message(data.to_string()) {
                                 Ok(base_message) => base_message,
                                 Err(e) => {
                                     eprintln!("Failed to decode message: {:?}", e);
@@ -107,11 +107,14 @@ impl NestriWebSocket {
                             let callbacks_lock = callbacks.read().unwrap();
                             if let Some(callback) = callbacks_lock.get(&base_message.payload_type) {
                                 let data = data.clone();
-                                callback(data);
+                                callback(data.to_string());
                             }
                         }
                         Err(e) => {
-                            eprintln!("Error receiving message: {:?}, reconnecting in 3 seconds...", e);
+                            eprintln!(
+                                "Error receiving message: {:?}, reconnecting in 3 seconds...",
+                                e
+                            );
                             sleep(Duration::from_secs(3)).await;
                             self_clone.reconnect().await.unwrap();
                             break; // Break the inner loop to get a new ws_read
@@ -123,7 +126,7 @@ impl NestriWebSocket {
         });
     }
 
-    fn spawn_write_loop(&self, mut message_rx: mpsc::UnboundedReceiver<Vec<u8>>) {
+    fn spawn_write_loop(&self, mut message_rx: mpsc::UnboundedReceiver<String>) {
         let writer = self.writer.clone();
         let self_clone = self.clone();
 
@@ -136,7 +139,10 @@ impl NestriWebSocket {
                         let mut writer_lock = writer.lock().await;
                         if let Some(writer) = writer_lock.as_mut() {
                             // Try to send the message over the WebSocket
-                            match writer.send(Message::Binary(message.clone())).await {
+                            match writer
+                                .send(Message::Text(Utf8Bytes::from(message.clone())))
+                                .await
+                            {
                                 Ok(_) => {
                                     // Message sent successfully
                                     break;
@@ -196,7 +202,7 @@ impl NestriWebSocket {
     }
 
     /// Send a message through the WebSocket
-    pub fn send_message(&self, message: Vec<u8>) -> Result<(), Box<dyn Error>> {
+    pub fn send_message(&self, message: String) -> Result<(), Box<dyn Error>> {
         self.message_tx
             .send(message)
             .map_err(|e| format!("Failed to send message: {:?}", e).into())
@@ -205,7 +211,7 @@ impl NestriWebSocket {
     /// Register a callback for a specific response type
     pub fn register_callback<F>(&self, response_type: &str, callback: F)
     where
-        F: Fn(Vec<u8>) + Send + Sync + 'static,
+        F: Fn(String) + Send + Sync + 'static,
     {
         let mut callbacks_lock = self.callbacks.write().unwrap();
         callbacks_lock.insert(response_type.to_string(), Box::new(callback));
@@ -234,6 +240,7 @@ impl Log for NestriWebSocket {
             let log_message = MessageLog {
                 base: MessageBase {
                     payload_type: "log".to_string(),
+                    latency: None,
                 },
                 level,
                 message,

protobufs/buf.yaml

@@ -0,0 +1,7 @@
+version: v2
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - STANDARD

protobufs/latency_tracker.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "relay/internal/proto";
+
+package proto;
+
+message ProtoTimestampEntry {
+  string stage = 1;
+  google.protobuf.Timestamp time = 2;
+};
+
+message ProtoLatencyTracker {
+    string sequence_id = 1;
+    repeated ProtoTimestampEntry timestamps = 2;
+}

protobufs/messages.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+option go_package = "relay/internal/proto";
+
+import "types.proto";
+import "latency_tracker.proto";
+
+package proto;
+
+message ProtoMessageBase {
+  string payload_type = 1;
+  ProtoLatencyTracker latency = 2;
+}
+
+message ProtoMessageInput {
+    ProtoMessageBase message_base = 1;
+    ProtoInput data = 2;
+}

protobufs/types.proto

@@ -0,0 +1,63 @@
+syntax = "proto3";
+
+option go_package = "relay/internal/proto";
+
+package proto;
+
+// MouseMove message
+message ProtoMouseMove {
+  string type = 1; // Fixed value "MouseMove"
+  int32 x = 2;
+  int32 y = 3;
+}
+
+// MouseMoveAbs message
+message ProtoMouseMoveAbs {
+  string type = 1; // Fixed value "MouseMoveAbs"
+  int32 x = 2;
+  int32 y = 3;
+}
+
+// MouseWheel message
+message ProtoMouseWheel {
+  string type = 1; // Fixed value "MouseWheel"
+  int32 x = 2;
+  int32 y = 3;
+}
+
+// MouseKeyDown message
+message ProtoMouseKeyDown {
+  string type = 1; // Fixed value "MouseKeyDown"
+  int32 key = 2;
+}
+
+// MouseKeyUp message
+message ProtoMouseKeyUp {
+  string type = 1; // Fixed value "MouseKeyUp"
+  int32 key = 2;
+}
+
+// KeyDown message
+message ProtoKeyDown {
+  string type = 1; // Fixed value "KeyDown"
+  int32 key = 2;
+}
+
+// KeyUp message
+message ProtoKeyUp {
+  string type = 1; // Fixed value "KeyUp"
+  int32 key = 2;
+}
+
+// Union of all Input types
+message ProtoInput {
+  oneof input_type {
+    ProtoMouseMove mouse_move = 1;
+    ProtoMouseMoveAbs mouse_move_abs = 2;
+    ProtoMouseWheel mouse_wheel = 3;
+    ProtoMouseKeyDown mouse_key_down = 4;
+    ProtoMouseKeyUp mouse_key_up = 5;
+    ProtoKeyDown key_down = 6;
+    ProtoKeyUp key_up = 7;
+  }
+}